create-walle 0.9.21 → 0.9.23

package/template/wall-e/embeddings.js

@@ -65,39 +65,163 @@ const EMBEDDING_MODELS = {
   'voyage-3-lite':        { dimensions: 512,  provider: 'voyage', maxBatch: 128,  key: 'voyage',  label: 'Voyage AI',      cost: '$0.02/1M', desc: 'Anthropic recommended' },
   'text-embedding-3-small': { dimensions: 1536, provider: 'openai', maxBatch: 2048, key: 'openai', label: 'OpenAI',         cost: '$0.02/1M', desc: 'text-embedding-3-small' },
   'text-embedding-3-large': { dimensions: 3072, provider: 'openai', maxBatch: 2048, key: 'openai_lg', label: 'OpenAI Large', cost: '$0.13/1M', desc: 'text-embedding-3-large' },
-  'nomic-embed-text':     { dimensions: 768,  provider: 'ollama', maxBatch: 32,   key: 'ollama',  label: 'Ollama (local)', cost: 'Free', desc: 'nomic-embed-text, private' },
+  'nomic-embed-text':     { dimensions: 768,  provider: 'ollama', maxBatch: 32,   key: 'ollama',  label: 'Ollama (local)', cost: 'Free', desc: 'nomic-embed-text, private', quantize: 'int8' },
   'mxbai-embed-large':    { dimensions: 1024, provider: 'ollama', maxBatch: 32,   key: 'ollama_lg', label: 'Ollama Large', cost: 'Free', desc: 'mxbai-embed-large, private' },
 };
 
-/**
- * Determine the active embedding model.
- * Priority: explicit env var > kv_store preference > auto-detect from API keys.
- */
-function getEmbeddingModel() {
-  // Explicit override (for testing / CI)
-  const explicit = process.env.WALLE_EMBEDDING_MODEL;
-  if (explicit) return EMBEDDING_MODELS[explicit] ? explicit : null;
+const EMBEDDING_POLICY = {
+  LOCAL_FIRST: 'local_first',
+  LOCAL_ONLY: 'local_only',
+  CLOUD_ALLOWED: 'cloud_allowed',
+  OFF: 'off',
+};
+const DEFAULT_EMBEDDING_POLICY = EMBEDDING_POLICY.LOCAL_FIRST;
+const EMBEDDING_POLICY_VALUES = new Set(Object.values(EMBEDDING_POLICY));
+const EMBEDDING_LOCAL_MODELS = ['nomic-embed-text', 'mxbai-embed-large'];
+const EMBEDDING_CLOUD_MODELS = ['text-embedding-004', 'text-embedding-3-small', 'voyage-3-lite'];
+const EMBEDDING_AUTO_MODEL_ORDER = [...EMBEDDING_LOCAL_MODELS, ...EMBEDDING_CLOUD_MODELS];
+
+function _getKv(key) {
+  try { return _getAdapter().getKv(key); } catch { return null; }
+}
+
+function _setKv(key, value) {
+  try { _getAdapter().setKv(key, value); return true; } catch { return false; }
+}
+
+function _normalizeEmbeddingPolicy(value) {
+  const normalized = String(value || '').trim().toLowerCase();
+  return EMBEDDING_POLICY_VALUES.has(normalized) ? normalized : DEFAULT_EMBEDDING_POLICY;
+}
+
+function getEmbeddingPolicy() {
+  return _normalizeEmbeddingPolicy(_getKv('embedding:auto_policy'));
+}
+
+function _getEmbeddingPolicyRecord() {
+  const raw = _getKv('embedding:auto_policy');
+  return {
+    policy: _normalizeEmbeddingPolicy(raw),
+    source: raw && EMBEDDING_POLICY_VALUES.has(String(raw).trim().toLowerCase()) ? 'saved' : 'default',
+  };
+}
 
-  // User preference from setup page (stored in kv_store)
-  // IMPORTANT: Honor the user's explicit choice even if the provider is temporarily
-  // unreachable (e.g. Ollama not running). Only skip if the model is unknown.
-  // Auto-detect fallthrough should only apply when there is NO stored preference.
+function _isLocalProvider(provider) {
+  return provider === 'ollama';
+}
+
+function _isCloudProvider(provider) {
+  return Boolean(provider) && !_isLocalProvider(provider);
+}
+
+function _policyAllowsConfig(config, policy) {
+  if (!config || policy === EMBEDDING_POLICY.OFF) return false;
+  if (policy === EMBEDDING_POLICY.CLOUD_ALLOWED) return true;
+  return _isLocalProvider(config.provider);
+}
+
+function _providerReachableForModel(modelName) {
+  const config = EMBEDDING_MODELS[modelName];
+  return config ? _hasApiKey(config.provider) : false;
+}
+
+function _embeddingCountForModel(modelName) {
+  try {
+    const db = _getAdapter().getDb();
+    const row = db.prepare('SELECT count(*) as c FROM embedding_map WHERE model = ?').get(modelName);
+    return row?.c || 0;
+  } catch {
+    return 0;
+  }
+}
+
+function _findExistingIndexedModel(policy, policySource) {
   try {
-    const pref = _getAdapter().getKv('embedding:active_model');
-    if (pref && EMBEDDING_MODELS[pref]) {
-      return pref;
+    const db = _getAdapter().getDb();
+    const rows = db.prepare('SELECT model, count(*) as c FROM embedding_map GROUP BY model ORDER BY c DESC').all();
+    for (const row of rows) {
+      const config = EMBEDDING_MODELS[row.model];
+      if (!config) continue;
+      // Backward compatibility: installs that already built a cloud index before
+      // the policy setting existed should keep using it until the user changes
+      // policy or manually switches providers.
+      if (policySource === 'default' || _policyAllowsConfig(config, policy)) return row.model;
     }
   } catch {}
+  return null;
+}
 
-  // Auto-detect: prefer free providers first, check both env and brain DB
-  if (_hasApiKey('google')) return 'text-embedding-004';
-  if (_hasApiKey('voyage')) return 'voyage-3-lite';
-  if (_hasApiKey('openai')) return 'text-embedding-3-small';
-  if (_hasApiKey('ollama')) return 'nomic-embed-text';
-
+function _firstAvailableModel(models, policy) {
+  for (const model of models) {
+    const config = EMBEDDING_MODELS[model];
+    if (!config || !_policyAllowsConfig(config, policy)) continue;
+    if (_hasApiKey(config.provider)) return model;
+  }
   return null;
 }
 
+function _selectAutoCandidate(policy) {
+  if (policy === EMBEDDING_POLICY.OFF) return null;
+  return _firstAvailableModel(
+    policy === EMBEDDING_POLICY.CLOUD_ALLOWED ? EMBEDDING_AUTO_MODEL_ORDER : EMBEDDING_LOCAL_MODELS,
+    policy
+  );
+}
+
+function getActiveEmbeddingModelInfo() {
+  const explicit = process.env.WALLE_EMBEDDING_MODEL;
+  if (explicit) {
+    return EMBEDDING_MODELS[explicit]
+      ? { model: explicit, source: 'env', policy: getEmbeddingPolicy(), policySource: 'env' }
+      : { model: null, source: 'invalid_env', policy: getEmbeddingPolicy(), policySource: 'env' };
+  }
+
+  const policyRecord = _getEmbeddingPolicyRecord();
+  if (policyRecord.policy === EMBEDDING_POLICY.OFF) {
+    return { model: null, source: 'off', policy: policyRecord.policy, policySource: policyRecord.source };
+  }
+
+  const pref = _getKv('embedding:active_model');
+  const selectionMode = _getKv('embedding:selection_mode') || (pref ? 'manual' : 'auto');
+  if (pref && EMBEDDING_MODELS[pref]) {
+    const config = EMBEDDING_MODELS[pref];
+    const blockedByLocalOnly = policyRecord.source === 'saved'
+      && policyRecord.policy === EMBEDDING_POLICY.LOCAL_ONLY
+      && !_policyAllowsConfig(config, policyRecord.policy);
+    if (blockedByLocalOnly) {
+      // The explicit Local only policy is stronger than an older cloud manual
+      // choice. The saved model is left in kv_store for audit/history, but it is
+      // not effective while this policy is active.
+    } else if (selectionMode !== 'auto') {
+      return { model: pref, source: selectionMode, policy: policyRecord.policy, policySource: policyRecord.source };
+    } else if (_policyAllowsConfig(config, policyRecord.policy) && _providerReachableForModel(pref)) {
+      return { model: pref, source: 'auto', policy: policyRecord.policy, policySource: policyRecord.source };
+    }
+  }
+
+  const indexedModel = _findExistingIndexedModel(policyRecord.policy, policyRecord.source);
+  if (indexedModel) {
+    return { model: indexedModel, source: 'existing_index', policy: policyRecord.policy, policySource: policyRecord.source };
+  }
+
+  const candidate = _selectAutoCandidate(policyRecord.policy);
+  return {
+    model: candidate,
+    source: candidate ? 'auto_candidate' : 'none',
+    policy: policyRecord.policy,
+    policySource: policyRecord.source,
+  };
+}
+
+/**
+ * Determine the active embedding model.
+ * Priority: explicit env var > manual/saved preference > compatible existing
+ * index > auto policy candidate. Cloud candidates require explicit opt-in.
+ */
+function getEmbeddingModel() {
+  return getActiveEmbeddingModelInfo().model || null;
+}
+
 function _hasApiKey(provider) {
   // Check process.env first, then fall back to model_providers table in brain DB
   switch (provider) {
@@ -137,6 +261,142 @@ function _isOllamaAvailable() {
   }
 }
 
+// --- Telemetry ---
+
+const EMBEDDING_SETUP_STATUS_TELEMETRY_INTERVAL_MS = 6 * 60 * 60 * 1000;
+const EMBEDDING_USAGE_TELEMETRY_INTERVAL_MS = 60 * 60 * 1000;
+let _lastSetupStatusTelemetry = { key: '', ts: 0 };
+let _lastUsageTelemetryByKey = new Map();
+
+function _countBucket(value) {
+  const n = Number(value) || 0;
+  if (n <= 0) return '0';
+  if (n < 10) return '1-9';
+  if (n < 100) return '10-99';
+  if (n < 1000) return '100-999';
+  if (n < 10000) return '1k-9k';
+  if (n < 100000) return '10k-99k';
+  return '100k+';
+}
+
+function _safeTelemetry() {
+  try { return require('./telemetry'); } catch { return null; }
+}
+
+function _modelTelemetryFields(modelName) {
+  const cfg = modelName && EMBEDDING_MODELS[modelName] ? EMBEDDING_MODELS[modelName] : null;
+  return {
+    model: cfg ? modelName : 'none',
+    provider: cfg ? cfg.provider : 'none',
+    model_key: cfg ? cfg.key : 'none',
+    cost_tier: cfg ? (cfg.cost === 'Free' ? 'free' : 'paid') : 'none',
+  };
+}
+
+function _embeddingTelemetrySnapshot() {
+  const info = getActiveEmbeddingModelInfo();
+  let model = info.model || null;
+  let providerReachable = false;
+  let stats = { total: 0, memories: 0, knowledge: 0 };
+  try {
+    const cfg = model && EMBEDDING_MODELS[model] ? EMBEDDING_MODELS[model] : null;
+    providerReachable = cfg ? !!_hasApiKey(cfg.provider) : false;
+  } catch {}
+  try {
+    if (model) stats = getEmbeddingStats();
+  } catch {}
+
+  let state = 'ready';
+  if (!_vecLoaded) state = 'sqlite_vec_unavailable';
+  else if (!model) state = 'no_provider';
+  else if (!providerReachable) state = 'provider_unreachable';
+
+  return {
+    state,
+    enabled: state === 'ready' ? 1 : 0,
+    sqlite_vec: _vecLoaded ? 1 : 0,
+    provider_reachable: providerReachable ? 1 : 0,
+    policy: info.policy || DEFAULT_EMBEDDING_POLICY,
+    selection_source: info.source || 'none',
+    ..._modelTelemetryFields(model),
+    embedded_bucket: _countBucket(stats.total),
+    memory_bucket: _countBucket(stats.memories),
+    knowledge_bucket: _countBucket(stats.knowledge),
+  };
+}
+
+function trackEmbeddingStatus(reason = 'boot', options = {}) {
+  const telemetry = options.telemetry || _safeTelemetry();
+  if (!telemetry || typeof telemetry.track !== 'function') return;
+  telemetry.track('embedding_status', {
+    reason,
+    ..._embeddingTelemetrySnapshot(),
+  });
+}
+
+function trackEmbeddingSetupView() {
+  const snapshot = _embeddingTelemetrySnapshot();
+  const key = [
+    snapshot.state,
+    snapshot.provider,
+    snapshot.model,
+    snapshot.embedded_bucket,
+    snapshot.memory_bucket,
+  ].join('|');
+  const now = Date.now();
+  if (_lastSetupStatusTelemetry.key === key && now - _lastSetupStatusTelemetry.ts < EMBEDDING_SETUP_STATUS_TELEMETRY_INTERVAL_MS) {
+    return;
+  }
+  _lastSetupStatusTelemetry = { key, ts: now };
+  trackEmbeddingStatus('setup_view');
+}
+
+function _switchFailureReason(result) {
+  const message = String(result && result.message || '').toLowerCase();
+  if (message.includes('unknown model')) return 'unknown_model';
+  if (message.includes('no api key')) return 'missing_key';
+  return result && result.ok ? 'success' : 'failed';
+}
+
+function _trackEmbeddingProviderSwitch(previousModel, requestedModel, result) {
+  const telemetry = _safeTelemetry();
+  if (!telemetry || typeof telemetry.track !== 'function') return;
+  const fromFields = _modelTelemetryFields(previousModel);
+  const toFields = _modelTelemetryFields(requestedModel);
+  telemetry.track('embedding_provider_switch', {
+    ok: result && result.ok ? 1 : 0,
+    reason: _switchFailureReason(result),
+    from_provider: fromFields.provider,
+    from_model: fromFields.model,
+    to_provider: toFields.provider,
+    to_model: toFields.model,
+    same_provider: fromFields.provider === toFields.provider ? 1 : 0,
+  });
+  if (result && result.ok) trackEmbeddingStatus('provider_switch', { telemetry });
+}
+
+function _trackEmbeddingUsage(operation, resultCount, status = 'ok') {
+  const telemetry = _safeTelemetry();
+  if (!telemetry || typeof telemetry.track !== 'function') return;
+  const snapshot = _embeddingTelemetrySnapshot();
+  const safeOperation = String(operation || 'unknown').replace(/[^a-z0-9_-]/gi, '_').slice(0, 64) || 'unknown';
+  const safeStatus = String(status || 'ok').replace(/[^a-z0-9_-]/gi, '_').slice(0, 32) || 'ok';
+  const resultBucket = _countBucket(resultCount);
+  const key = [safeOperation, safeStatus, snapshot.provider, snapshot.model, resultBucket].join('|');
+  const now = Date.now();
+  const last = _lastUsageTelemetryByKey.get(key) || 0;
+  if (now - last < EMBEDDING_USAGE_TELEMETRY_INTERVAL_MS) return;
+  _lastUsageTelemetryByKey.set(key, now);
+  telemetry.track('embedding_usage', {
+    operation: safeOperation,
+    status: safeStatus,
+    result_bucket: resultBucket,
+    state: snapshot.state,
+    provider: snapshot.provider,
+    model: snapshot.model,
+  });
+}
+
 function getModelConfig() {
   const model = getEmbeddingModel();
   return model ? { name: model, ...EMBEDDING_MODELS[model] } : null;
@@ -154,7 +414,9 @@ function getDimensions() {
  * Used by the setup page UI.
  */
 function getProviderStatus() {
-  const activeModel = getEmbeddingModel();
+  const activeInfo = getActiveEmbeddingModelInfo();
+  const activeModel = activeInfo.model;
+  const policy = activeInfo.policy || getEmbeddingPolicy();
   const db = _getAdapter().getDb();
 
   return Object.entries(EMBEDDING_MODELS).map(([model, config]) => {
@@ -186,12 +448,171 @@ function getProviderStatus() {
       hasKey,
       isActive,
       reachable,
+      selectionSource: isActive ? activeInfo.source : undefined,
+      autoEligible: _policyAllowsConfig(config, policy),
+      cloudRequiresOptIn: _isCloudProvider(config.provider) && policy !== EMBEDDING_POLICY.CLOUD_ALLOWED,
       embeddingCount: count,
       sizeBytes: sizeEstimate,
     };
   });
 }
 
+function _resetBackfillForModelChange() {
+  try {
+    _setKv('backfill:embed:cursor', '0');
+    _setKv('backfill:done', null);
+  } catch {}
+}
+
+function _ensureVecTableForModel(modelName) {
+  const config = EMBEDDING_MODELS[modelName];
+  if (!config || !ensureVecLoaded()) return false;
+  const db = _getAdapter().getDb();
+  const vecTable = _vecTableName(config.key);
+  const existing = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name=?`).get(vecTable);
+  if (!existing) {
+    db.prepare(`CREATE VIRTUAL TABLE "${vecTable}" USING vec0(embedding ${_vecColType(config)})`).run();
+  }
+  return true;
+}
+
+function _trackEmbeddingAutoSelect(reason, result) {
+  const telemetry = _safeTelemetry();
+  if (!telemetry || typeof telemetry.track !== 'function') return;
+  telemetry.track('embedding_auto_select', {
+    reason,
+    ok: result.ok ? 1 : 0,
+    status: result.status || '',
+    policy: result.policy || getEmbeddingPolicy(),
+    model: result.model || 'none',
+    provider: result.provider || 'none',
+    selection_source: result.selectionSource || 'none',
+  });
+}
+
+function autoSelectProvider(reason = 'auto') {
+  const info = getActiveEmbeddingModelInfo();
+  const config = info.model ? EMBEDDING_MODELS[info.model] : null;
+  if (!info.model || !config) {
+    const result = {
+      ok: false,
+      status: info.source === 'off' ? 'off' : 'waiting',
+      reason: info.source,
+      policy: info.policy || getEmbeddingPolicy(),
+      selectionSource: info.source || 'none',
+    };
+    _trackEmbeddingAutoSelect(reason, result);
+    return result;
+  }
+  if (!['auto_candidate', 'auto'].includes(info.source)) {
+    return {
+      ok: false,
+      status: 'preserved',
+      reason: info.source,
+      model: info.model,
+      provider: config.provider,
+      policy: info.policy,
+      selectionSource: info.source,
+    };
+  }
+
+  const previous = _getKv('embedding:active_model');
+  _setKv('embedding:active_model', info.model);
+  _setKv('embedding:selection_mode', 'auto');
+  _ensureVecTableForModel(info.model);
+  if (previous !== info.model) _resetBackfillForModelChange();
+
+  const result = {
+    ok: true,
+    status: previous === info.model ? 'unchanged' : 'selected',
+    model: info.model,
+    provider: config.provider,
+    policy: info.policy,
+    selectionSource: 'auto',
+  };
+  _trackEmbeddingAutoSelect(reason, result);
+  if (previous !== info.model) trackEmbeddingStatus('auto_select');
+  return result;
+}
+
+function getEmbeddingSetupState() {
+  const policyRecord = _getEmbeddingPolicyRecord();
+  const activeInfo = getActiveEmbeddingModelInfo();
+  const activeConfig = activeInfo.model ? EMBEDDING_MODELS[activeInfo.model] : null;
+  const localAvailable = Boolean(_firstAvailableModel(EMBEDDING_LOCAL_MODELS, EMBEDDING_POLICY.LOCAL_ONLY));
+  const cloudAvailableModel = _firstAvailableModel(EMBEDDING_CLOUD_MODELS, EMBEDDING_POLICY.CLOUD_ALLOWED);
+  const cloudAvailable = Boolean(cloudAvailableModel);
+  const cloudOptInAvailable = !activeInfo.model
+    && cloudAvailable
+    && policyRecord.policy !== EMBEDDING_POLICY.CLOUD_ALLOWED
+    && policyRecord.policy !== EMBEDDING_POLICY.OFF;
+
+  let status = 'waiting_for_provider';
+  let message = 'Memory Search will turn on automatically when a local embedding provider is available.';
+  if (policyRecord.policy === EMBEDDING_POLICY.OFF) {
+    status = 'off';
+    message = 'Memory Search is off.';
+  } else if (activeInfo.model) {
+    status = activeInfo.source === 'manual' ? 'manual_active' : 'active';
+    message = `${activeConfig.label} is powering Memory Search.`;
+  } else if (cloudOptInAvailable) {
+    status = 'cloud_opt_in_available';
+    message = 'A cloud embedding provider is available. Opt in to enable semantic Memory Search.';
+  } else if (!localAvailable && !cloudAvailable) {
+    status = 'no_provider';
+    message = 'Add a local embedding provider or cloud API key to enable semantic Memory Search.';
+  }
+
+  return {
+    policy: policyRecord.policy,
+    policySource: policyRecord.source,
+    status,
+    message,
+    activeModel: activeInfo.model || null,
+    activeProvider: activeConfig ? activeConfig.provider : null,
+    activeLabel: activeConfig ? activeConfig.label : null,
+    selectionSource: activeInfo.source || 'none',
+    localAvailable,
+    cloudAvailable,
+    cloudAvailableModel,
+    cloudOptInAvailable,
+    totalIndexed: activeInfo.model ? _embeddingCountForModel(activeInfo.model) : 0,
+  };
+}
+
+function setEmbeddingPolicy(policy) {
+  const normalized = String(policy || '').trim().toLowerCase();
+  if (!EMBEDDING_POLICY_VALUES.has(normalized)) {
+    return { ok: false, message: `Unknown Memory Search policy: ${policy}` };
+  }
+  const previousPolicy = getEmbeddingPolicy();
+  const previousModel = _getKv('embedding:active_model');
+  const previousConfig = previousModel && EMBEDDING_MODELS[previousModel] ? EMBEDDING_MODELS[previousModel] : null;
+  _setKv('embedding:auto_policy', normalized);
+  if (normalized !== EMBEDDING_POLICY.OFF && _getKv('embedding:selection_mode') === 'auto') {
+    _setKv('embedding:active_model', null);
+  }
+  if ((normalized === EMBEDDING_POLICY.LOCAL_ONLY || normalized === EMBEDDING_POLICY.OFF)
+      && previousConfig
+      && _isCloudProvider(previousConfig.provider)) {
+    _setKv('embedding:active_model', null);
+    _setKv('embedding:selection_mode', 'auto');
+    _resetBackfillForModelChange();
+  }
+  const auto = autoSelectProvider('policy_update');
+  const setup = getEmbeddingSetupState();
+  const telemetry = _safeTelemetry();
+  try {
+    telemetry?.track?.('embedding_policy_update', {
+      from: previousPolicy,
+      to: normalized,
+      status: setup.status,
+      active_provider: setup.activeProvider || 'none',
+    });
+  } catch {}
+  return { ok: true, policy: normalized, auto, setup };
+}
+
 // --- Embedding computation ---
 
 async function computeEmbedding(text) {
@@ -425,6 +846,33 @@ function _vecTableName(modelKey) {
   return `embedding_vec_${modelKey}`;
 }
 
+// vec0 column type for a model: int8[N] for models that opt into quantization
+// (config.quantize==='int8'), float[N] otherwise. int8 stores 1 byte/dim vs 4 — ~6x
+// smaller on disk + faster KNN, at ~95% recall@10 for unit-normalized vectors (nomic).
+function _vecColType(config) {
+  return `${config.quantize === 'int8' ? 'int8' : 'float'}[${config.dimensions}]`;
+}
+
+// The bind expression that turns a JSON-array param into the stored/queried vector.
+// Derived from the ACTUAL declared table type (not just config) so the code is correct
+// whether the table is still float32 (pre-migration) or already int8 (post-migration) —
+// no fragile coupling between the schema and a flag. 'unit' assumes inputs in [-1,1]
+// (true for L2-normalized models like nomic-embed-text), the only mode sqlite-vec supports.
+function _declaredVecType(db, table) {
+  try {
+    const r = db.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name=?").get(table);
+    if (!r || !r.sql) return null;
+    if (/\bint8\s*\[/.test(r.sql)) return 'int8';
+    if (/\bfloat\s*\[/.test(r.sql)) return 'float';
+  } catch {}
+  return null;
+}
+function _vecBindExpr(db, config) {
+  return _declaredVecType(db, _vecTableName(config.key)) === 'int8'
+    ? "vec_quantize_int8(vec_f32(?),'unit')"
+    : 'vec_f32(?)';
+}
+
 /**
  * Create embedding tables for the active model.
  * Each model gets its own vec table. The mapping table is shared.
@@ -454,8 +902,8 @@ function createEmbeddingTables() {
   const vecTable = _vecTableName(config.key);
   const existing = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name=?`).get(vecTable);
   if (!existing) {
-    db.prepare(`CREATE VIRTUAL TABLE "${vecTable}" USING vec0(embedding float[${config.dimensions}])`).run();
-    console.log(`[embeddings] Created vec table: ${vecTable} (${config.dimensions}d)`);
+    db.prepare(`CREATE VIRTUAL TABLE "${vecTable}" USING vec0(embedding ${_vecColType(config)})`).run();
+    console.log(`[embeddings] Created vec table: ${vecTable} (${_vecColType(config)})`);
   }
 
   // Migrate from old single-table layout if it exists
@@ -543,7 +991,7 @@ function storeEmbedding(entityId, entityType, embedding) {
         rowid = info.lastInsertRowid;
       }
 
-      db.prepare(`INSERT INTO "${vecTable}"(rowid, embedding) VALUES (?, vec_f32(?))`).run(
+      db.prepare(`INSERT INTO "${vecTable}"(rowid, embedding) VALUES (?, ${_vecBindExpr(db, config)})`).run(
         BigInt(rowid), JSON.stringify(embedding)
       );
     });
@@ -632,6 +1080,75 @@ function deleteEmbeddingsForModel(modelName) {
   }
 }
 
+// Delete all embeddings for one entity (its embedding_map rows across models + the
+// matching vec-table rows keyed by rowid). Call this whenever an entity is hard-deleted
+// so we don't leak orphaned vectors (the source of the audited 47,649 orphans — skills
+// that re-sync by `DELETE FROM memories WHERE id = ?` left the embeddings behind).
+function deleteEmbeddingsForEntity(entityId) {
+  if (!entityId) return { deleted: 0 };
+  // Load vec0 into this connection if not already (skill subprocesses may not have
+  // touched the embeddings module yet, but their brain.initDb() opened the DB).
+  if (!ensureVecLoaded()) return { deleted: 0 };
+  const db = _getAdapter().getDb();
+  try {
+    const rows = db.prepare('SELECT rowid, model FROM embedding_map WHERE entity_id = ?').all(entityId);
+    if (!rows.length) return { deleted: 0 };
+    db.transaction(() => {
+      for (const r of rows) {
+        const cfg = EMBEDDING_MODELS[r.model];
+        if (cfg) {
+          try { db.prepare(`DELETE FROM "${_vecTableName(cfg.key)}" WHERE rowid = ?`).run(BigInt(r.rowid)); } catch {}
+        }
+      }
+      db.prepare('DELETE FROM embedding_map WHERE entity_id = ?').run(entityId);
+    })();
+    return { deleted: rows.length };
+  } catch (err) {
+    return { deleted: 0, error: err.message };
+  }
+}
+
+// One-time / periodic sweep: delete embeddings whose entity no longer exists. Only sweeps
+// entity types whose source table we can verify (memory→memories, knowledge→knowledge);
+// other types are left untouched. Batched per-transaction so each holds the cross-process
+// write lock only briefly (lock-friendly, per the CTM lesson). entity_type is parameterized;
+// the source-table names are internal constants (no user input → no injection).
+const _ORPHAN_ENTITY_TABLES = { memory: 'memories', knowledge: 'knowledge' };
+function pruneOrphanEmbeddings({ batchSize = 200 } = {}) {
+  if (!ensureVecLoaded()) return { deleted: 0 };
+  const db = _getAdapter().getDb();
+  const batch = Math.max(50, Math.min(2000, Number(batchSize) || 200));
+  let deleted = 0;
+  for (const [etype, table] of Object.entries(_ORPHAN_ENTITY_TABLES)) {
+    // Guard: only sweep if the source table exists, else skip (never mass-delete blind).
+    const hasTable = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").get(table);
+    if (!hasTable) continue;
+    for (let i = 0; i < 5000; i += 1) { // batch backstop
+      const orphans = db.prepare(
+        `SELECT em.rowid AS rowid, em.model AS model
+           FROM embedding_map em
+          WHERE em.entity_type = ?
+            AND NOT EXISTS (SELECT 1 FROM ${table} t WHERE t.id = em.entity_id)
+          LIMIT ?`
+      ).all(etype, batch);
+      if (!orphans.length) break;
+      db.transaction(() => {
+        for (const o of orphans) {
+          const cfg = EMBEDDING_MODELS[o.model];
+          if (cfg) {
+            try { db.prepare(`DELETE FROM "${_vecTableName(cfg.key)}" WHERE rowid = ?`).run(BigInt(o.rowid)); } catch {}
+          }
+          db.prepare('DELETE FROM embedding_map WHERE rowid = ?').run(o.rowid);
+        }
+      })();
+      deleted += orphans.length;
+      if (orphans.length < batch) break;
+    }
+  }
+  if (deleted) console.log(`[embeddings] pruneOrphanEmbeddings removed ${deleted} orphaned embeddings`);
+  return { deleted };
+}
+
 // --- Vector search ---
 
 function searchSimilarMemories(queryEmbedding, limit = 10, filters = {}) {
@@ -649,7 +1166,7 @@ function searchSimilarMemories(queryEmbedding, limit = 10, filters = {}) {
       FROM "${vecTable}" v
       JOIN embedding_map em ON em.rowid = v.rowid
       JOIN memories m ON m.id = em.entity_id
-      WHERE v.embedding MATCH vec_f32(?) AND k = ?
+      WHERE v.embedding MATCH ${_vecBindExpr(db, config)} AND k = ?
         AND em.entity_type = 'memory' AND em.model = ?
     `;
     const params = [JSON.stringify(queryEmbedding), kFetch, config.name];
@@ -661,9 +1178,12 @@ function searchSimilarMemories(queryEmbedding, limit = 10, filters = {}) {
     sql += ' ORDER BY v.distance LIMIT ?';
     params.push(limit);
 
-    return db.prepare(sql).all(...params);
+    const rows = db.prepare(sql).all(...params);
+    _trackEmbeddingUsage('memory_search', rows.length);
+    return rows;
   } catch (err) {
     console.error('[embeddings] Vector search failed:', err.message);
+    _trackEmbeddingUsage('memory_search', 0, 'error');
     return [];
   }
 }
@@ -678,19 +1198,22 @@ function searchSimilarKnowledge(queryEmbedding, limit = 10) {
     const vecTable = _vecTableName(config.key);
     const kFetch = Math.min(limit * 3, 100);
 
-    return db.prepare(`
+    const rows = db.prepare(`
       SELECT kn.*, v.distance
       FROM "${vecTable}" v
       JOIN embedding_map em ON em.rowid = v.rowid
       JOIN knowledge kn ON kn.id = em.entity_id
-      WHERE v.embedding MATCH vec_f32(?) AND k = ?
+      WHERE v.embedding MATCH ${_vecBindExpr(db, config)} AND k = ?
         AND em.entity_type = 'knowledge' AND em.model = ?
         AND kn.status = 'active'
       ORDER BY v.distance
       LIMIT ?
     `).all(JSON.stringify(queryEmbedding), kFetch, config.name, limit);
+    _trackEmbeddingUsage('knowledge_search', rows.length);
+    return rows;
   } catch (err) {
     console.error('[embeddings] Knowledge vector search failed:', err.message);
+    _trackEmbeddingUsage('knowledge_search', 0, 'error');
     return [];
   }
 }
@@ -714,7 +1237,7 @@ function searchSimilarEntities(queryEmbedding, limit = 10, entityType = null) {
       SELECT em.entity_id, em.entity_type, v.distance
       FROM "${vecTable}" v
       JOIN embedding_map em ON em.rowid = v.rowid
-      WHERE v.embedding MATCH vec_f32(?) AND k = ?
+      WHERE v.embedding MATCH ${_vecBindExpr(db, config)} AND k = ?
         AND em.model = ?
     `;
     const params = [JSON.stringify(queryEmbedding), kFetch, config.name];
@@ -724,9 +1247,12 @@ function searchSimilarEntities(queryEmbedding, limit = 10, entityType = null) {
     sql += ' ORDER BY v.distance LIMIT ?';
     params.push(limit);
 
-    return db.prepare(sql).all(...params);
+    const rows = db.prepare(sql).all(...params);
+    _trackEmbeddingUsage(entityType ? `${entityType}_entity_search` : 'entity_search', rows.length);
+    return rows;
   } catch (err) {
     console.error('[embeddings] Entity vector search failed:', err.message);
+    _trackEmbeddingUsage(entityType ? `${entityType}_entity_search` : 'entity_search', 0, 'error');
     return [];
   }
 }
@@ -765,51 +1291,57 @@ function reciprocalRankFusion(lexicalResults, vectorResults, limit, k = 60) {
  * @returns {{ ok: boolean, message: string }}
  */
 function switchProvider(modelName) {
+  const previousModel = getEmbeddingModel();
   const config = EMBEDDING_MODELS[modelName];
-  if (!config) return { ok: false, message: `Unknown model: ${modelName}` };
-  if (!_hasApiKey(config.provider)) return { ok: false, message: `No API key for ${config.label}` };
+  if (!config) {
+    const result = { ok: false, message: `Unknown model: ${modelName}` };
+    _trackEmbeddingProviderSwitch(previousModel, modelName, result);
+    return result;
+  }
+  if (!_hasApiKey(config.provider)) {
+    const result = { ok: false, message: `No API key for ${config.label}` };
+    _trackEmbeddingProviderSwitch(previousModel, modelName, result);
+    return result;
+  }
 
   // Save preference
-  _getAdapter().setKv('embedding:active_model', modelName);
+  _setKv('embedding:active_model', modelName);
+  _setKv('embedding:selection_mode', 'manual');
 
   // Reset cached dimensions so they're recalculated
   _dimensions = null;
 
   // Create the new vec table if needed
-  if (ensureVecLoaded()) {
-    const db = _getAdapter().getDb();
-    const vecTable = _vecTableName(config.key);
-    const existing = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name=?`).get(vecTable);
-    if (!existing) {
-      db.prepare(`CREATE VIRTUAL TABLE "${vecTable}" USING vec0(embedding float[${config.dimensions}])`).run();
-    }
-  }
+  _ensureVecTableForModel(modelName);
 
   // Reset backfill cursors so the new model gets backfilled
-  try {
-    _getAdapter().setKv('backfill:embed:cursor', '0');
-    _getAdapter().setKv('backfill:done', null);
-  } catch {}
+  _resetBackfillForModelChange();
 
   console.log(`[embeddings] Switched to ${config.label} (${modelName}, ${config.dimensions}d)`);
-  return { ok: true, message: `Switched to ${config.label}` };
+  const result = { ok: true, message: `Switched to ${config.label}` };
+  _trackEmbeddingProviderSwitch(previousModel, modelName, result);
+  return result;
 }
 
 // --- Initialization ---
 
 function init(adapter) {
+  // Sub-phase timing so the boot profile pinpoints what's slow inside embeddings.init
+  // (on the live brain this whole call was ~15s; isolate which step). No-op if profiler absent.
+  let _boot; try { _boot = require('./lib/boot-profile'); } catch { _boot = { measure: (_l, f) => f() }; }
   if (adapter) _adapter = adapter;
-  if (!ensureVecLoaded()) {
+  if (!_boot.measure('embeddings.ensureVecLoaded', () => ensureVecLoaded())) {
     console.log('[embeddings] sqlite-vec not available — vector search disabled');
     return false;
   }
+  _boot.measure('embeddings.autoSelectProvider', () => autoSelectProvider('boot'));
   if (!getEmbeddingModel()) {
-    console.log('[embeddings] No embedding API key found (set GOOGLE_API_KEY, VOYAGE_API_KEY, OPENAI_API_KEY, or OLLAMA_BASE_URL) — vector search disabled');
+    console.log('[embeddings] No embedding provider active — vector search disabled until local provider is available or cloud embeddings are allowed');
     return false;
   }
-  const ok = createEmbeddingTables();
+  const ok = _boot.measure('embeddings.createEmbeddingTables', () => createEmbeddingTables());
   if (ok) {
-    const stats = getEmbeddingStats();
+    const stats = _boot.measure('embeddings.getEmbeddingStats', () => getEmbeddingStats());
     const config = getModelConfig();
     console.log(`[embeddings] Initialized (${stats.total} embeddings, model: ${config.name}, dims: ${config.dimensions}, provider: ${config.label})`);
   }
@@ -822,14 +1354,20 @@ function _resetForTesting() {
   _adapter = null;
   _defaultAdapter = null;
   _ollamaAvailableCache = { value: false, ts: 0 };
+  _lastSetupStatusTelemetry = { key: '', ts: 0 };
+  _lastUsageTelemetryByKey = new Map();
 }
 
 module.exports = {
   init, ensureVecLoaded, isAvailable, createEmbeddingTables,
-  getEmbeddingModel, getModelConfig, getDimensions, EMBEDDING_MODELS,
+  getEmbeddingModel, getActiveEmbeddingModelInfo, getEmbeddingPolicy, setEmbeddingPolicy,
+  getEmbeddingSetupState, autoSelectProvider,
+  getModelConfig, getDimensions, EMBEDDING_MODELS, EMBEDDING_POLICY,
+  trackEmbeddingStatus, trackEmbeddingSetupView,
   computeEmbedding, batchComputeEmbeddings,
   storeEmbedding, hasEmbedding, getEmbeddingStats, getAllEmbeddingStats,
   getProviderStatus, switchProvider, deleteEmbeddingsForModel,
+  deleteEmbeddingsForEntity, pruneOrphanEmbeddings,
   searchSimilarMemories, searchSimilarKnowledge, searchSimilarEntities, reciprocalRankFusion,
   _resetForTesting,
 };