@geekbeer/minion 3.53.1 → 3.57.0

package/core/stores/variable-store.js

@@ -1,34 +1,69 @@
 /**
  * Variable Store
  *
- * Manages minion-local secrets and variables stored in .env-style files.
- * - Secrets: ~/.minion/.env.secrets (or DATA_DIR/.env.secrets)
- * - Variables: ~/.minion/.env.variables (or DATA_DIR/.env.variables)
+ * Manages minion-local variables and secrets — both workspace-scoped.
  *
- * Files use standard .env format: KEY=value (one per line, # for comments).
- * Secrets never leave the minion; variables are non-sensitive configuration.
+ * Storage: SQLite tables `variables(workspace_id, key, value, updated_at)`
+ * and `secrets(workspace_id, key, value, updated_at)`. `workspace_id=''` is
+ * the minion-wide bucket; any other value is a workspace-specific bucket.
+ *
+ * At skill execution time the runner / template-expander merges
+ * minion-wide + current-workspace entries for the relevant scope, with the
+ * WS-scoped value winning on conflict.
+ *
+ * Migration: on first read after the SQLite tables are created, any legacy
+ * `.env.variables` / `.env.secrets` flat files are imported into the
+ * minion-wide bucket and renamed to `.env.variables.migrated` /
+ * `.env.secrets.migrated` (kept as one-time backups).
+ *
+ * Secrets never leave the minion and are never persisted in HQ DB. Variables
+ * are non-sensitive and may also be defined at HQ scopes (workspace /
+ * project / workflow); see packages/docs-internal/src/content/docs/design/
+ * variables-and-secrets.md for the full resolution model.
  */
 
 const fs = require('fs')
 const path = require('path')
 const { DATA_DIR } = require('../lib/platform')
 const { config } = require('../config')
+const { getDb } = require('../db')
 
 /**
- * Resolve file path for a given store type.
- * @param {'secrets' | 'variables'} type
+ * Resolve the legacy `.env.variables` path (for one-time migration).
  * @returns {string}
  */
-function getFilePath(type) {
-  const filename = type === 'secrets' ? '.env.secrets' : '.env.variables'
+function getLegacyVariablesFilePath() {
   try {
     fs.accessSync(DATA_DIR, fs.constants.W_OK)
-    return path.join(DATA_DIR, filename)
+    return path.join(DATA_DIR, '.env.variables')
   } catch {
-    return path.join(config.HOME_DIR, '.minion', filename)
+    return path.join(config.HOME_DIR, '.minion', '.env.variables')
   }
 }
 
+/**
+ * Resolve the legacy `.env.secrets` path (for one-time migration).
+ * @returns {string}
+ */
+function getLegacySecretsFilePath() {
+  try {
+    fs.accessSync(DATA_DIR, fs.constants.W_OK)
+    return path.join(DATA_DIR, '.env.secrets')
+  } catch {
+    return path.join(config.HOME_DIR, '.minion', '.env.secrets')
+  }
+}
+
+/**
+ * Normalize a workspace identifier to its canonical string form.
+ * '' (empty string) represents the minion-wide bucket.
+ * @param {string|null|undefined} workspaceId
+ * @returns {string}
+ */
+function normalizeWorkspaceId(workspaceId) {
+  return workspaceId == null ? '' : String(workspaceId)
+}
+
 /**
  * Parse a .env file into a key-value object.
  * @param {string} filePath
@@ -54,105 +89,272 @@ function parseEnvFile(filePath) {
   return result
 }
 
+// ─── Legacy flat-file → SQLite migration ────────────────────────────────────
+
+let _legacyVariablesMigrationDone = false
+let _legacySecretsMigrationDone = false
+
+function migrateLegacyFile(legacyPath, table, doneFlagRef) {
+  if (doneFlagRef.done) return
+  doneFlagRef.done = true
+
+  let entries
+  try {
+    entries = parseEnvFile(legacyPath)
+  } catch {
+    return
+  }
+  const keys = Object.keys(entries)
+  if (keys.length === 0) return
+
+  const db = getDb()
+  const now = Date.now()
+  const insert = db.prepare(
+    `INSERT OR IGNORE INTO ${table} (workspace_id, key, value, updated_at) VALUES (?, ?, ?, ?)`
+  )
+  const tx = db.transaction(() => {
+    for (const k of keys) {
+      insert.run('', k, String(entries[k] ?? ''), now)
+    }
+  })
+  tx()
+
+  try {
+    fs.renameSync(legacyPath, `${legacyPath}.migrated`)
+    console.log(`[VariableStore] Migrated ${keys.length} legacy ${table} entry/ies from ${legacyPath} into SQLite (minion-wide scope)`)
+  } catch (err) {
+    console.error(`[VariableStore] Migrated ${table} to SQLite but failed to rename ${legacyPath}: ${err.message}`)
+  }
+}
+
+const _variablesFlag = { done: false }
+const _secretsFlag = { done: false }
+
+function migrateLegacyVariablesFile() {
+  migrateLegacyFile(getLegacyVariablesFilePath(), 'variables', _variablesFlag)
+}
+
+function migrateLegacySecretsFile() {
+  migrateLegacyFile(getLegacySecretsFilePath(), 'secrets', _secretsFlag)
+}
+
+// ─── Generic table operations (variables and secrets share the shape) ───────
+
+function getRowsForScope(table, workspaceId) {
+  const wsId = normalizeWorkspaceId(workspaceId)
+  const db = getDb()
+  return db.prepare(`SELECT key, value FROM ${table} WHERE workspace_id = ? ORDER BY key`).all(wsId)
+}
+
+function getScopeAsObject(table, workspaceId) {
+  const rows = getRowsForScope(table, workspaceId)
+  const out = {}
+  for (const r of rows) out[r.key] = r.value
+  return out
+}
+
+function getEffective(table, workspaceId) {
+  const wide = getScopeAsObject(table, '')
+  if (!workspaceId) return wide
+  const scoped = getScopeAsObject(table, workspaceId)
+  return { ...wide, ...scoped }
+}
+
+function upsertEntry(table, workspaceId, key, value) {
+  const wsId = normalizeWorkspaceId(workspaceId)
+  const db = getDb()
+  db.prepare(`
+    INSERT INTO ${table} (workspace_id, key, value, updated_at) VALUES (?, ?, ?, ?)
+    ON CONFLICT(workspace_id, key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at
+  `).run(wsId, key, String(value ?? ''), Date.now())
+  return wsId
+}
+
+function deleteEntry(table, workspaceId, key) {
+  const wsId = normalizeWorkspaceId(workspaceId)
+  const db = getDb()
+  const info = db.prepare(`DELETE FROM ${table} WHERE workspace_id = ? AND key = ?`).run(wsId, key)
+  return { removed: info.changes > 0, wsId }
+}
+
+function listScopes(table) {
+  const db = getDb()
+  const rows = db.prepare(`SELECT DISTINCT workspace_id FROM ${table}`).all()
+  return rows.map(r => r.workspace_id)
+}
+
+// ─── Variables (workspace-scoped) ────────────────────────────────────────────
+
+function getVariablesForScope(workspaceId) {
+  migrateLegacyVariablesFile()
+  return getScopeAsObject('variables', workspaceId)
+}
+
 /**
- * Write a key-value object to a .env file.
- * @param {string} filePath
- * @param {Record<string, string>} data
+ * Merge minion-wide and workspace-scoped variables. Workspace-scoped values
+ * override minion-wide values when the key is the same.
  */
-function writeEnvFile(filePath, data) {
-  const dir = path.dirname(filePath)
-  fs.mkdirSync(dir, { recursive: true })
+function getEffectiveVariables(workspaceId) {
+  migrateLegacyVariablesFile()
+  return getEffective('variables', workspaceId)
+}
+
+function getVariable(workspaceId, key) {
+  const all = getVariablesForScope(workspaceId)
+  return all[key] ?? null
+}
 
-  const lines = Object.entries(data).map(([key, value]) => `${key}=${value}`)
-  fs.writeFileSync(filePath, lines.join('\n') + '\n', 'utf-8')
+function setVariable(workspaceId, key, value) {
+  migrateLegacyVariablesFile()
+  const wsId = upsertEntry('variables', workspaceId, key, value)
+  console.log(`[VariableStore] Set variable (scope=${wsId || 'minion-wide'}): ${key}`)
+}
+
+function removeVariable(workspaceId, key) {
+  migrateLegacyVariablesFile()
+  const { removed, wsId } = deleteEntry('variables', workspaceId, key)
+  if (removed) {
+    console.log(`[VariableStore] Removed variable (scope=${wsId || 'minion-wide'}): ${key}`)
+  }
+  return removed
+}
+
+function listVariableKeys(workspaceId) {
+  migrateLegacyVariablesFile()
+  const rows = getRowsForScope('variables', workspaceId)
+  return rows.map(r => r.key)
+}
+
+function listVariableScopes() {
+  migrateLegacyVariablesFile()
+  return listScopes('variables')
+}
+
+// ─── Secrets (workspace-scoped, sensitive) ───────────────────────────────────
+
+function getSecretsForScope(workspaceId) {
+  migrateLegacySecretsFile()
+  return getScopeAsObject('secrets', workspaceId)
+}
+
+function getEffectiveSecrets(workspaceId) {
+  migrateLegacySecretsFile()
+  return getEffective('secrets', workspaceId)
+}
+
+function listSecretKeys(workspaceId) {
+  migrateLegacySecretsFile()
+  const rows = getRowsForScope('secrets', workspaceId)
+  return rows.map(r => r.key)
 }
 
 /**
- * Get all key-value pairs for a store type.
- * @param {'secrets' | 'variables'} type
- * @returns {Record<string, string>}
+ * Set (insert or update) a secret value in the given scope.
+ *
+ * For minion-wide secrets (workspace_id=''), the value is also synced into
+ * `process.env` so in-process consumers (chat plugin sessions, ad-hoc Bash
+ * commands) see the change immediately without restarting the server.
+ *
+ * WS-scoped secrets are NEVER written to `process.env` to prevent cross-WS
+ * leakage; runners must call `getEffectiveSecrets(workspaceId)` and inject
+ * them explicitly into the session env.
  */
+function setSecret(workspaceId, key, value) {
+  migrateLegacySecretsFile()
+  const wsId = upsertEntry('secrets', workspaceId, key, value)
+  if (wsId === '') {
+    process.env[key] = String(value ?? '')
+  }
+  console.log(`[VariableStore] Set secret (scope=${wsId || 'minion-wide'}): ${key}`)
+}
+
+function removeSecret(workspaceId, key) {
+  migrateLegacySecretsFile()
+  const { removed, wsId } = deleteEntry('secrets', workspaceId, key)
+  if (removed) {
+    if (wsId === '') {
+      delete process.env[key]
+    }
+    console.log(`[VariableStore] Removed secret (scope=${wsId || 'minion-wide'}): ${key}`)
+  }
+  return removed
+}
+
+function listSecretScopes() {
+  migrateLegacySecretsFile()
+  return listScopes('secrets')
+}
+
+// ─── Generic / legacy dispatchers (back-compat) ─────────────────────────────
+//
+// The original API used `getAll('variables')` / `getAll('secrets')` etc. with
+// no workspace awareness. We retain the same names but route them to the
+// minion-wide bucket so any straggler caller keeps working.
+
 function getAll(type) {
-  return parseEnvFile(getFilePath(type))
+  if (type === 'secrets') return getSecretsForScope('')
+  return getVariablesForScope('')
 }
 
-/**
- * Get a single value by key.
- * @param {'secrets' | 'variables'} type
- * @param {string} key
- * @returns {string | null}
- */
 function get(type, key) {
-  const data = getAll(type)
-  return data[key] ?? null
+  if (type === 'secrets') {
+    const all = getSecretsForScope('')
+    return all[key] ?? null
+  }
+  return getVariable('', key)
 }
 
-/**
- * Set a key-value pair (creates or updates).
- * Only secrets are synced to process.env (for child process inheritance).
- * Variables use {{VAR}} template expansion in skill content instead.
- * @param {'secrets' | 'variables'} type
- * @param {string} key
- * @param {string} value
- */
 function set(type, key, value) {
-  const filePath = getFilePath(type)
-  const data = parseEnvFile(filePath)
-  data[key] = value
-  writeEnvFile(filePath, data)
-  // Only sync secrets to process.env; variables use template expansion instead
   if (type === 'secrets') {
-    process.env[key] = value
+    setSecret('', key, value)
+    return
   }
-  console.log(`[VariableStore] Set ${type} key: ${key}`)
+  setVariable('', key, value)
 }
 
-/**
- * Remove a key.
- * @param {'secrets' | 'variables'} type
- * @param {string} key
- * @returns {boolean} true if key existed
- */
 function remove(type, key) {
-  const filePath = getFilePath(type)
-  const data = parseEnvFile(filePath)
-  if (!(key in data)) return false
-  delete data[key]
-  writeEnvFile(filePath, data)
-  // Only sync secrets to process.env; variables use template expansion instead
-  if (type === 'secrets') {
-    delete process.env[key]
-  }
-  console.log(`[VariableStore] Removed ${type} key: ${key}`)
-  return true
+  if (type === 'secrets') return removeSecret('', key)
+  return removeVariable('', key)
 }
 
-/**
- * List all keys for a store type.
- * @param {'secrets' | 'variables'} type
- * @returns {string[]}
- */
 function listKeys(type) {
-  return Object.keys(getAll(type))
+  if (type === 'secrets') return listSecretKeys('')
+  return listVariableKeys('')
 }
 
 /**
- * Build environment object from minion secrets only.
- * Variables are no longer injected as env vars; they use {{VAR}} template
- * expansion in skill content (same mechanism as project/workflow variables).
- *
- * @returns {Record<string, string>} Secret key-value pairs for process.env
+ * @deprecated Use `getEffectiveSecrets(workspaceId)` instead. Retained so any
+ * stragglers continue compiling; returns minion-wide secrets only.
  */
 function buildEnv() {
-  return getAll('secrets')
+  return getSecretsForScope('')
 }
 
 module.exports = {
+  // Generic (back-compat)
   getAll,
   get,
   set,
   remove,
   listKeys,
   buildEnv,
-  getFilePath,
+
+  // Variables (workspace-scoped)
+  getVariablesForScope,
+  getEffectiveVariables,
+  getVariable,
+  setVariable,
+  removeVariable,
+  listVariableKeys,
+  listVariableScopes,
+  migrateLegacyVariablesFile,
+
+  // Secrets (workspace-scoped)
+  getSecretsForScope,
+  getEffectiveSecrets,
+  listSecretKeys,
+  setSecret,
+  removeSecret,
+  listSecretScopes,
+  migrateLegacySecretsFile,
 }