@shadowforge0/aquifer-memory 1.8.1 → 1.9.0

package/consumers/openclaw-ext/index.js

@@ -6,8 +6,10 @@
 //   $OPENCLAW_HOME/extensions/aquifer-memory/  ← symlink to this directory
 //
 // Behavior:
-//   - Loads $OPENCLAW_HOME/.env so DATABASE_URL / EMBED_PROVIDER /
-//     AQUIFER_LLM_PROVIDER etc. are visible to the plugin.
+//   - Reads $OPENCLAW_HOME/.env and then mcp.servers.aquifer.env from
+//     $OPENCLAW_HOME/openclaw.json so ingest uses the same DB/schema/env as
+//     the Aquifer MCP recall tools, without leaving those values in the
+//     shared OpenClaw process environment.
 //   - Delegates to consumers/openclaw-plugin.js. If AQUIFER_PERSONA is set
 //     (pluginConfig.persona or env), the plugin loads the persona module
 //     and hands off mountOnOpenClaw(api); otherwise the default generic
@@ -21,17 +23,73 @@ const os = require('os');
 
 const OPENCLAW_HOME = process.env.OPENCLAW_HOME || path.join(os.homedir(), '.openclaw');
 
-function loadEnvFile(envPath) {
+function mergeEnvObject(target, values, opts = {}) {
+  if (!values || typeof values !== 'object') return;
+  for (const [key, value] of Object.entries(values)) {
+    if (!/^[A-Z_][A-Z0-9_]*$/.test(key)) continue;
+    if (value === undefined || value === null) continue;
+    if (!opts.override && (process.env[key] || target[key])) continue;
+    target[key] = String(value);
+  }
+}
+
+function readEnvFile(envPath) {
+  const values = {};
   try {
     const text = fs.readFileSync(envPath, 'utf8');
     for (const line of text.split('\n')) {
       const m = line.match(/^([A-Z_][A-Z0-9_]*)=(.*)$/);
-      if (m && !process.env[m[1]]) process.env[m[1]] = m[2].trim();
+      if (m) values[m[1]] = m[2].trim();
     }
   } catch { /* .env missing — ok */ }
+  return values;
+}
+
+function readOpenClawAquiferMcpEnv(configPath) {
+  try {
+    const cfg = JSON.parse(fs.readFileSync(configPath, 'utf8'));
+    return cfg.mcp?.servers?.aquifer?.env || {};
+  } catch { /* openclaw.json missing or malformed — plugin config/env may still work */ }
+  return {};
+}
+
+function buildAquiferEnv() {
+  const env = {};
+  mergeEnvObject(env, readEnvFile(path.join(OPENCLAW_HOME, '.env')));
+  mergeEnvObject(env, readOpenClawAquiferMcpEnv(path.join(OPENCLAW_HOME, 'openclaw.json')), { override: true });
+  return env;
 }
 
-loadEnvFile(path.join(OPENCLAW_HOME, '.env'));
+function withEnvOverlay(env, fn) {
+  const previous = new Map();
+  for (const [key, value] of Object.entries(env || {})) {
+    previous.set(key, Object.prototype.hasOwnProperty.call(process.env, key) ? process.env[key] : undefined);
+    process.env[key] = value;
+  }
+  try {
+    return fn();
+  } finally {
+    for (const [key, value] of previous.entries()) {
+      if (value === undefined) delete process.env[key];
+      else process.env[key] = value;
+    }
+  }
+}
 
 // Re-export the plugin as-is. OpenClaw expects { id, name, register }.
-module.exports = require('../openclaw-plugin');
+const plugin = require('../openclaw-plugin');
+
+module.exports = {
+  ...plugin,
+  register(api) {
+    const aquiferEnv = buildAquiferEnv();
+    const pluginConfig = { ...(api.pluginConfig || {}) };
+    if (!pluginConfig.persona && aquiferEnv.AQUIFER_PERSONA) {
+      pluginConfig.persona = aquiferEnv.AQUIFER_PERSONA;
+    }
+    return withEnvOverlay(aquiferEnv, () => plugin.register({
+      ...api,
+      pluginConfig,
+    }));
+  },
+};

package/consumers/openclaw-ext/openclaw.plugin.json

@@ -1,7 +1,7 @@
 {
   "id": "aquifer-memory",
   "name": "Aquifer Memory",
-  "version": "1.2.0",
+  "version": "1.9.0",
   "description": "Session ingest + recall + feedback. Reads DATABASE_URL / EMBED_PROVIDER / AQUIFER_LLM_PROVIDER from host env; delegates to AQUIFER_PERSONA module if set.",
   "main": "index.js",
   "hooks": ["before_reset"],

package/consumers/openclaw-ext/package.json

@@ -1,6 +1,6 @@
 {
   "name": "aquifer-openclaw-ext",
-  "version": "1.2.0",
+  "version": "1.9.0",
   "private": true,
   "main": "index.js",
   "description": "Drop-in OpenClaw extension for Aquifer Memory. Symlink into $OPENCLAW_HOME/extensions/aquifer-memory/ — no host-side boilerplate required.",

package/consumers/openclaw-install.js

@@ -0,0 +1,326 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const { version: packageVersion } = require('../package.json');
+
+const OPENCLAW_PLUGIN_ID = 'aquifer-memory';
+
+function nowStamp() {
+  return new Date().toISOString().replace(/[:.]/g, '-');
+}
+
+function uniqueBackupPath(filePath) {
+  const base = `${filePath}.bak-${nowStamp()}`;
+  if (!fs.existsSync(base)) return base;
+  for (let i = 1; i < 1000; i += 1) {
+    const candidate = `${base}-${i}`;
+    if (!fs.existsSync(candidate)) return candidate;
+  }
+  throw new Error(`Could not allocate unique backup path for ${filePath}`);
+}
+
+function valueFlag(flags = {}, name, fallback = null) {
+  const value = flags[name];
+  if (value === undefined || value === null || value === true || value === '') return fallback;
+  return String(value);
+}
+
+function resolveOpenClawHome(flags = {}, env = process.env) {
+  return path.resolve(valueFlag(flags, 'openclaw-home', env.OPENCLAW_HOME || path.join(os.homedir(), '.openclaw')));
+}
+
+function readJson(filePath) {
+  return JSON.parse(fs.readFileSync(filePath, 'utf8'));
+}
+
+function writeJsonWithBackup(filePath, value, opts = {}) {
+  if (opts.dryRun) return null;
+  const backupPath = uniqueBackupPath(filePath);
+  if (fs.existsSync(filePath)) fs.copyFileSync(filePath, backupPath);
+  fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, 'utf8');
+  return backupPath;
+}
+
+function samePath(a, b) {
+  return path.resolve(a) === path.resolve(b);
+}
+
+function ensureOpenClawHome(openclawHome) {
+  if (!fs.existsSync(openclawHome) || !fs.statSync(openclawHome).isDirectory()) {
+    throw new Error(`OPENCLAW_HOME not found: ${openclawHome}`);
+  }
+}
+
+function ensureExtensionLink({ openclawHome, packageRoot, dryRun = false, force = false }) {
+  const source = path.join(packageRoot, 'consumers', 'openclaw-ext');
+  const dest = path.join(openclawHome, 'extensions', 'aquifer-memory');
+  if (!fs.existsSync(source)) throw new Error(`OpenClaw extension source not found: ${source}`);
+
+  const result = {
+    source,
+    dest,
+    action: 'link',
+    backupPath: null,
+  };
+
+  if (fs.existsSync(dest)) {
+    const stat = fs.lstatSync(dest);
+    if (stat.isSymbolicLink()) {
+      const target = fs.readlinkSync(dest);
+      const resolvedTarget = path.resolve(path.dirname(dest), target);
+      if (samePath(resolvedTarget, source)) {
+        return { ...result, action: 'already-linked' };
+      }
+    }
+
+    if (!force) {
+      throw new Error(`${dest} already exists; rerun with --force to move it aside and relink`);
+    }
+
+    result.backupPath = `${dest}.bak-${nowStamp()}`;
+    result.action = 'replace-link';
+    if (!dryRun) fs.renameSync(dest, result.backupPath);
+  }
+
+  if (!dryRun) {
+    fs.mkdirSync(path.dirname(dest), { recursive: true });
+    fs.symlinkSync(source, dest, 'dir');
+  }
+  return result;
+}
+
+function normalizeMcpServer(existing = {}, packageRoot) {
+  return {
+    ...existing,
+    command: 'node',
+    args: [path.join(packageRoot, 'consumers', 'mcp.js')],
+    env: {
+      ...(existing.env && typeof existing.env === 'object' ? existing.env : {}),
+    },
+  };
+}
+
+function ensureMcpConfig({ openclawHome, packageRoot, dryRun = false }) {
+  const configPath = path.join(openclawHome, 'openclaw.json');
+  const mcpPath = path.join(packageRoot, 'consumers', 'mcp.js');
+  const result = {
+    configPath,
+    mcpPath,
+    action: 'skipped',
+    backupPath: null,
+    previousArgs: null,
+    nextArgs: [mcpPath],
+  };
+
+  if (!fs.existsSync(configPath)) {
+    return { ...result, reason: 'openclaw.json not found' };
+  }
+
+  const config = readJson(configPath);
+  const existing = config.mcp?.servers?.aquifer || {};
+  result.previousArgs = Array.isArray(existing.args) ? existing.args : null;
+
+  const nextServer = normalizeMcpServer(existing, packageRoot);
+  const previousText = JSON.stringify(existing);
+  const nextText = JSON.stringify(nextServer);
+  if (previousText === nextText) {
+    return { ...result, action: 'already-configured' };
+  }
+
+  config.mcp = config.mcp && typeof config.mcp === 'object' ? config.mcp : {};
+  config.mcp.servers = config.mcp.servers && typeof config.mcp.servers === 'object'
+    ? config.mcp.servers
+    : {};
+  config.mcp.servers.aquifer = nextServer;
+  result.action = 'write-config';
+  result.backupPath = writeJsonWithBackup(configPath, config, { dryRun });
+  return result;
+}
+
+function normalizePluginEntry(existing = {}) {
+  const entry = existing && typeof existing === 'object' ? existing : {};
+  return {
+    ...entry,
+    enabled: true,
+    config: entry.config && typeof entry.config === 'object' ? entry.config : {},
+  };
+}
+
+function ensurePluginConfig({ openclawHome, extensionPath, dryRun = false }) {
+  const configPath = path.join(openclawHome, 'openclaw.json');
+  const result = {
+    configPath,
+    pluginId: OPENCLAW_PLUGIN_ID,
+    extensionPath,
+    action: 'skipped',
+    backupPath: null,
+    previousLoadPaths: null,
+    nextLoadPaths: null,
+  };
+
+  if (!fs.existsSync(configPath)) {
+    return { ...result, reason: 'openclaw.json not found' };
+  }
+
+  const config = readJson(configPath);
+  const plugins = config.plugins && typeof config.plugins === 'object' ? config.plugins : {};
+  const load = plugins.load && typeof plugins.load === 'object' ? plugins.load : {};
+  const previousLoadPaths = Array.isArray(load.paths) ? load.paths : [];
+  const nextLoadPaths = previousLoadPaths.includes(extensionPath)
+    ? previousLoadPaths
+    : [...previousLoadPaths, extensionPath];
+  const entries = plugins.entries && typeof plugins.entries === 'object' ? plugins.entries : {};
+  const previousEntry = entries[OPENCLAW_PLUGIN_ID];
+  const nextEntry = normalizePluginEntry(previousEntry);
+  const previousAllow = Array.isArray(plugins.allow) ? plugins.allow : null;
+  const nextAllow = previousAllow && !previousAllow.includes(OPENCLAW_PLUGIN_ID)
+    ? [...previousAllow, OPENCLAW_PLUGIN_ID]
+    : previousAllow;
+
+  result.previousLoadPaths = previousLoadPaths;
+  result.nextLoadPaths = nextLoadPaths;
+
+  const unchanged = JSON.stringify(previousLoadPaths) === JSON.stringify(nextLoadPaths)
+    && JSON.stringify(previousEntry) === JSON.stringify(nextEntry)
+    && JSON.stringify(previousAllow) === JSON.stringify(nextAllow);
+  if (unchanged) {
+    return { ...result, action: 'already-configured' };
+  }
+
+  config.plugins = {
+    ...plugins,
+    entries: {
+      ...entries,
+      [OPENCLAW_PLUGIN_ID]: nextEntry,
+    },
+    load: {
+      ...load,
+      paths: nextLoadPaths,
+    },
+  };
+  if (nextAllow) config.plugins.allow = nextAllow;
+
+  result.action = 'write-config';
+  result.backupPath = writeJsonWithBackup(configPath, config, { dryRun });
+  return result;
+}
+
+function inspectInstalledPackage(openclawHome) {
+  const packagePath = path.join(openclawHome, 'node_modules', '@shadowforge0', 'aquifer-memory', 'package.json');
+  if (!fs.existsSync(packagePath)) {
+    return { packagePath, installed: false, version: null };
+  }
+  const pkg = readJson(packagePath);
+  return { packagePath, installed: true, version: pkg.version || null };
+}
+
+function buildInstallReport(args = {}, opts = {}) {
+  const flags = args.flags || {};
+  const env = opts.env || process.env;
+  const openclawHome = resolveOpenClawHome(flags, env);
+  const packageRoot = opts.packageRoot || path.resolve(__dirname, '..');
+  const dryRun = flags['dry-run'] === true;
+  const force = flags.force === true;
+  const linkCurrentPackage = flags['link-current-package'] === true;
+  const skipExtension = flags['skip-extension'] === true;
+  const skipMcp = flags['skip-mcp'] === true;
+
+  ensureOpenClawHome(openclawHome);
+
+  const installedPackage = inspectInstalledPackage(openclawHome);
+  const expectedPackageRoot = path.join(openclawHome, 'node_modules', '@shadowforge0', 'aquifer-memory');
+  const targetPackageRoot = linkCurrentPackage || !installedPackage.installed
+    ? packageRoot
+    : expectedPackageRoot;
+  const report = {
+    ok: true,
+    dryRun,
+    openclawHome,
+    packageRoot,
+    targetPackageRoot,
+    linkCurrentPackage,
+    packageVersion,
+    expectedPackageRoot,
+    installedPackage,
+    extension: skipExtension ? { action: 'skipped', reason: '--skip-extension' } : null,
+    plugin: skipExtension ? { action: 'skipped', reason: '--skip-extension' } : null,
+    mcp: skipMcp ? { action: 'skipped', reason: '--skip-mcp' } : null,
+    warnings: [],
+  };
+
+  if (installedPackage.installed && installedPackage.version && installedPackage.version !== packageVersion) {
+    report.warnings.push(
+      `OpenClaw node_modules has @shadowforge0/aquifer-memory ${installedPackage.version}; installer package is ${packageVersion}.`
+    );
+  }
+  if (!samePath(targetPackageRoot, expectedPackageRoot)) {
+    report.warnings.push(
+      `MCP will point outside OpenClaw node_modules: ${targetPackageRoot}. For a durable package install, run npm install --prefix "${openclawHome}" @shadowforge0/aquifer-memory@${packageVersion}.`
+    );
+  }
+  if (!samePath(packageRoot, expectedPackageRoot) && samePath(targetPackageRoot, expectedPackageRoot)) {
+    report.warnings.push(
+      `Installer is running from ${packageRoot}, but durable OpenClaw wiring targets ${expectedPackageRoot}. Use --link-current-package only for source-checkout development.`
+    );
+  }
+
+  if (!skipExtension) {
+    report.extension = ensureExtensionLink({ openclawHome, packageRoot: targetPackageRoot, dryRun, force });
+    report.plugin = ensurePluginConfig({
+      openclawHome,
+      extensionPath: report.extension.dest,
+      dryRun,
+    });
+  }
+
+  if (!skipMcp) {
+    report.mcp = ensureMcpConfig({ openclawHome, packageRoot: targetPackageRoot, dryRun });
+  }
+
+  return report;
+}
+
+function formatInstallReport(report) {
+  const lines = [];
+  lines.push(`Aquifer package: ${report.packageVersion} at ${report.packageRoot}`);
+  if (report.installedPackage.installed) {
+    lines.push(`OpenClaw package: ${report.installedPackage.version} at ${report.installedPackage.packagePath}`);
+  } else {
+    lines.push(`OpenClaw package: not installed at ${report.installedPackage.packagePath}`);
+  }
+  lines.push(`Extension: ${report.extension.action}${report.extension.dest ? ` (${report.extension.dest})` : ''}`);
+  if (report.extension.backupPath) lines.push(`Extension backup: ${report.extension.backupPath}`);
+  lines.push(`Plugin config: ${report.plugin.action}${report.plugin.configPath ? ` (${report.plugin.configPath})` : ''}`);
+  if (report.plugin.backupPath) lines.push(`Plugin config backup: ${report.plugin.backupPath}`);
+  if (report.plugin.reason) lines.push(`Plugin config note: ${report.plugin.reason}`);
+  lines.push(`MCP config: ${report.mcp.action}${report.mcp.configPath ? ` (${report.mcp.configPath})` : ''}`);
+  if (report.mcp.backupPath) lines.push(`MCP config backup: ${report.mcp.backupPath}`);
+  if (report.mcp.reason) lines.push(`MCP config note: ${report.mcp.reason}`);
+  for (const warning of report.warnings) lines.push(`Warning: ${warning}`);
+  if (report.dryRun) lines.push('Dry run: no files were changed.');
+  return lines.join('\n');
+}
+
+async function cmdInstallOpenClaw(args = {}, opts = {}) {
+  const report = buildInstallReport(args, opts);
+  if (args.flags?.json) {
+    console.log(JSON.stringify(report, null, 2));
+  } else {
+    console.log(formatInstallReport(report));
+  }
+  return report;
+}
+
+module.exports = {
+  buildInstallReport,
+  cmdInstallOpenClaw,
+  ensureExtensionLink,
+  ensurePluginConfig,
+  ensureMcpConfig,
+  formatInstallReport,
+  inspectInstalledPackage,
+  normalizeMcpServer,
+  resolveOpenClawHome,
+};

package/consumers/openclaw-plugin.js

@@ -33,6 +33,43 @@ function coerceRawEntries(messages) {
   });
 }
 
+function normalizeTimestamp(value) {
+  if (value === null || value === undefined || value === '') return null;
+
+  if (value instanceof Date) {
+    const time = value.getTime();
+    return Number.isFinite(time) ? value.toISOString() : null;
+  }
+
+  if (typeof value === 'number') {
+    if (!Number.isFinite(value)) return null;
+    const millis = Math.abs(value) < 1e12 ? value * 1000 : value;
+    const date = new Date(millis);
+    return Number.isFinite(date.getTime()) ? date.toISOString() : null;
+  }
+
+  if (typeof value === 'string') {
+    const trimmed = value.trim();
+    if (!trimmed) return null;
+    if (/^-?\d+(?:\.\d+)?$/.test(trimmed)) {
+      return normalizeTimestamp(Number(trimmed));
+    }
+    const parsed = Date.parse(trimmed);
+    if (!Number.isFinite(parsed)) return null;
+    return new Date(parsed).toISOString();
+  }
+
+  return null;
+}
+
+function selectTimestamp(...values) {
+  for (const value of values) {
+    const ts = normalizeTimestamp(value);
+    if (ts) return ts;
+  }
+  return null;
+}
+
 function normalizeEntries(rawEntries) {
   const normalized = [];
   let userCount = 0, assistantCount = 0;
@@ -55,7 +92,7 @@ function normalizeEntries(rawEntries) {
         .join('\n');
     }
 
-    const ts = entry.timestamp || msg.timestamp || null;
+    const ts = selectTimestamp(entry.timestamp, msg.timestamp);
     if (ts && !startedAt) startedAt = ts;
     if (ts) lastMessageAt = ts;
 
@@ -117,6 +154,7 @@ module.exports = buildPlugin();
 // contract; OpenClaw reads { id, name, register } only.
 module.exports.normalizeEntries = normalizeEntries;
 module.exports.coerceRawEntries = coerceRawEntries;
+module.exports.normalizeTimestamp = normalizeTimestamp;
 
 function register(api) {
     const pluginConfig = api.pluginConfig || {};

package/consumers/shared/config.js

@@ -55,6 +55,7 @@ const DEFAULTS = {
     servingMode: 'legacy',   // 'legacy' | 'curated'
     activeScopeKey: null,
     activeScopePath: null,
+    allowedScopeKeys: null,
   },
   codex: {
     checkpoint: {
@@ -120,6 +121,7 @@ const ENV_MAP = [
   ['AQUIFER_MEMORY_SERVING_MODE', 'memory.servingMode'],
   ['AQUIFER_MEMORY_ACTIVE_SCOPE_KEY', 'memory.activeScopeKey'],
   ['AQUIFER_MEMORY_ACTIVE_SCOPE_PATH', 'memory.activeScopePath'],
+  ['AQUIFER_MEMORY_ALLOWED_SCOPE_KEYS', 'memory.allowedScopeKeys'],
   ['AQUIFER_CODEX_CHECKPOINT_CHECK_INTERVAL_MS', 'codex.checkpoint.checkIntervalMs', Number],
   ['AQUIFER_CODEX_CHECKPOINT_CHECK_INTERVAL_MINUTES', 'codex.checkpoint.checkIntervalMinutes', Number],
   ['AQUIFER_CODEX_CHECKPOINT_EVERY_MESSAGES', 'codex.checkpoint.everyMessages', Number],