@geravant/sinain 1.13.0 → 1.15.0

package/sinain-core/src/agents-loader.ts

@@ -0,0 +1,254 @@
+// agents-loader.ts
+//
+// sinain-core's view of sinain-agent/agents.json. Loaded once at startup,
+// gives config.ts a typed snapshot of the bare-agent + OpenClaw config that
+// used to live in .env.
+//
+// Why sinain-core reads it at all: the OpenClaw WS client is constructed
+// before the bare agent ever registers, and stays connected across the
+// session for heartbeat / situation push / feedback. So sinain-core needs
+// the gateway URL/token at startup — it can't wait for run.sh to forward
+// them via /bareagent/register.
+//
+// Path resolution:
+//   1. AGENTS_CONFIG_PATH env var (explicit override)
+//   2. <repo-root>/sinain-agent/agents.json (default)
+//   3. <repo-root>/sinain-agent/agents.example.json (fresh-checkout fallback)
+//
+// Returns null if no candidate exists. config.ts treats null as "use env
+// defaults" so the migration is non-breaking.
+//
+// Field expansion happens in two passes (shell-like ordering):
+//   1. Cross-field literal expansion: ${allowedTools} inside escAllowedTools
+//      resolves against other top-level fields. Lets users compose
+//      whitelists by reference.
+//   2. Env-var indirection: ${OPENROUTER_API_KEY}, ${HOME}, etc. resolve
+//      against process.env. Mirrors run.sh's apply_profile_env regex
+//      (\$\{[A-Za-z_][A-Za-z0-9_]*\}).
+
+import { readFileSync, existsSync } from "node:fs";
+import { resolve, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import { homedir } from "node:os";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+// Regex shared with run.sh's apply_profile_env. Keep the two implementations
+// in lockstep when changing either side.
+const VAR_REF_RE = /\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g;
+
+export interface OpenClawProfileFields {
+  type: "openclaw";
+  wsUrl?: string;
+  wsToken?: string;
+  httpUrl?: string;
+  httpToken?: string;
+  sessionKey?: string;
+  phase1TimeoutMs?: number;
+  phase2TimeoutMs?: number;
+  pingIntervalMs?: number;
+}
+
+export interface AgentProfile {
+  type?: string;
+  bin?: string;
+  settings?: string;
+  model?: string;
+  env?: Record<string, string>;
+  // openclaw-specific fields (typed loose since profiles are heterogeneous)
+  wsUrl?: string;
+  wsToken?: string;
+  httpUrl?: string;
+  httpToken?: string;
+  sessionKey?: string;
+  phase1TimeoutMs?: number;
+  phase2TimeoutMs?: number;
+  pingIntervalMs?: number;
+}
+
+export interface AnalyzerBlock {
+  debounceMs?: number;
+  maxIntervalMs?: number;
+}
+
+export interface EscalationBlock {
+  mode?: string;
+  cooldownMs?: number;
+  staleMs?: number;
+}
+
+export interface AgentsConfig {
+  default?: string;
+  pollIntervalSec?: number;
+  agentMaxTurns?: number;
+  spawnMaxTurns?: number;
+  allowedTools?: string;
+  escAllowedTools?: string;
+  spawnAllowedTools?: string;
+  autoApproveTools?: string;
+  analyzer?: AnalyzerBlock;
+  escalation?: EscalationBlock;
+  profiles?: Record<string, AgentProfile>;
+  /** Absolute path of the file actually loaded (for debug logging). */
+  loadedFrom?: string;
+}
+
+function expandVars(value: string): string {
+  return value.replace(VAR_REF_RE, (_, name: string) => process.env[name] ?? "");
+}
+
+/** Recursively expand ${env} references in any string field of `obj`. */
+function expandEnvDeep<T>(obj: T): T {
+  if (obj === null || obj === undefined) return obj;
+  if (typeof obj === "string") return expandVars(obj) as unknown as T;
+  if (Array.isArray(obj)) return obj.map((x) => expandEnvDeep(x)) as unknown as T;
+  if (typeof obj === "object") {
+    const out: Record<string, unknown> = {};
+    for (const [k, v] of Object.entries(obj as Record<string, unknown>)) {
+      out[k] = expandEnvDeep(v);
+    }
+    return out as T;
+  }
+  return obj;
+}
+
+/**
+ * Cross-field literal expansion at the top level. Lets users write
+ *   "escAllowedTools": "${allowedTools} Bash(git:*) Edit"
+ * and have it compose with the value of `allowedTools`. Only string
+ * top-level fields are eligible to be referenced; nested blocks aren't.
+ */
+function expandTopLevelRefs(cfg: AgentsConfig): void {
+  const refTargets: Record<string, string | undefined> = {
+    allowedTools: cfg.allowedTools,
+    escAllowedTools: cfg.escAllowedTools,
+    spawnAllowedTools: cfg.spawnAllowedTools,
+    autoApproveTools: cfg.autoApproveTools,
+    default: cfg.default,
+  };
+  const expand = (s?: string): string | undefined => {
+    if (!s) return s;
+    return s.replace(VAR_REF_RE, (m, name: string) => {
+      const ref = refTargets[name];
+      // Only substitute when the referenced field is a top-level field
+      // *that has a value*. Otherwise leave the placeholder untouched so
+      // the env-var pass can still pick it up (e.g. ${HOME}).
+      return ref !== undefined ? ref : m;
+    });
+  };
+  cfg.escAllowedTools = expand(cfg.escAllowedTools);
+  cfg.spawnAllowedTools = expand(cfg.spawnAllowedTools);
+}
+
+function candidatePaths(): string[] {
+  const explicit = process.env.AGENTS_CONFIG_PATH;
+  if (explicit) return [resolve(explicit)];
+  // ~/.sinain/agents.json is the wizard's write target — works on npm
+  // installs where the package dir is read-only. Highest priority after
+  // an explicit env override so user-home config wins over the shipped
+  // template.
+  const userHome = resolve(homedir(), ".sinain", "agents.json");
+  // sinain-core compiled output sits at sinain-core/dist/, source at
+  // sinain-core/src/. From either, sinain-agent is two levels up.
+  const repoRoot = resolve(__dirname, "..", "..");
+  return [
+    userHome,
+    resolve(repoRoot, "sinain-agent", "agents.json"),
+    resolve(repoRoot, "sinain-agent", "agents.example.json"),
+  ];
+}
+
+/**
+ * Load agents.json (or fall back to agents.example.json). Returns null if
+ * neither candidate file exists or parsing fails. Caller (config.ts) is
+ * expected to treat null as "use env defaults" so the migration is
+ * non-breaking.
+ */
+export function loadAgentsConfig(): AgentsConfig | null {
+  for (const path of candidatePaths()) {
+    if (!existsSync(path)) continue;
+    try {
+      const raw = readFileSync(path, "utf-8");
+      const parsed = JSON.parse(raw) as AgentsConfig;
+      // Pass 1: cross-field literal substitution at top level. Has to run
+      // before env-var expansion so `${allowedTools}` doesn't accidentally
+      // resolve to process.env.allowedTools (which doesn't exist anyway,
+      // but the principle of layered substitution is what we want).
+      expandTopLevelRefs(parsed);
+      // Pass 2: ${ENV_VAR} expansion across the whole tree. Profile-level
+      // env blocks are deliberately skipped here — those are applied by
+      // run.sh in a subshell at invocation time, not by sinain-core.
+      const profiles = parsed.profiles;
+      delete parsed.profiles;
+      const expanded = expandEnvDeep(parsed);
+      // Restore profiles (with their own env blocks expanded for the few
+      // openclaw-specific fields that sinain-core does read).
+      if (profiles) {
+        const expandedProfiles: Record<string, AgentProfile> = {};
+        for (const [name, profile] of Object.entries(profiles)) {
+          // Don't expand the per-profile `env` block — those are evaluated
+          // at agent invocation time inside run.sh. But DO expand other
+          // profile fields (wsToken, etc.) since sinain-core reads them.
+          const { env: profileEnv, ...rest } = profile;
+          expandedProfiles[name] = {
+            ...expandEnvDeep(rest),
+            ...(profileEnv !== undefined ? { env: profileEnv } : {}),
+          };
+        }
+        expanded.profiles = expandedProfiles;
+      }
+      expanded.loadedFrom = path;
+      return expanded;
+    } catch (err) {
+      console.warn(`[agents-loader] failed to parse ${path}: ${(err as Error).message}`);
+    }
+  }
+  return null;
+}
+
+/** Convenience: pull the openclaw profile if present, else null.
+ *  @deprecated Prefer findGatewayProfile() — it doesn't assume the name. */
+export function getOpenClawProfile(cfg: AgentsConfig | null): AgentProfile | null {
+  return cfg?.profiles?.openclaw ?? null;
+}
+
+/**
+ * Find a gateway-style profile (any profile with `type: "openclaw"`,
+ * regardless of the profile's name). Used by sinain-core to:
+ *   - load WS connection params at startup (config.ts)
+ *   - decide whether a lane choice should route via WS or HTTP (escalator.ts)
+ *
+ * Preference order: the canonical name "openclaw" wins if present, otherwise
+ * the first openclaw-typed profile found in the JSON. This lets users add
+ * `nemoclaw`, `nanoclaw-prod`, etc. with their own URLs as drop-in replacements.
+ */
+export function findGatewayProfile(
+  cfg: AgentsConfig | null,
+): { name: string; profile: AgentProfile } | null {
+  const profiles = cfg?.profiles;
+  if (!profiles) return null;
+  if (profiles.openclaw?.type === "openclaw") {
+    return { name: "openclaw", profile: profiles.openclaw };
+  }
+  for (const [name, profile] of Object.entries(profiles)) {
+    if (profile?.type === "openclaw") return { name, profile };
+  }
+  return null;
+}
+
+/** Is the given profile name a gateway-style (WS-dispatched) profile? */
+export function isGatewayProfile(
+  cfg: AgentsConfig | null,
+  name: string,
+): boolean {
+  return cfg?.profiles?.[name]?.type === "openclaw";
+}
+
+/** All openclaw-typed profile names in the config. */
+export function gatewayProfileNames(cfg: AgentsConfig | null): string[] {
+  const profiles = cfg?.profiles;
+  if (!profiles) return [];
+  return Object.entries(profiles)
+    .filter(([_, p]) => p?.type === "openclaw")
+    .map(([name]) => name);
+}

package/sinain-core/src/buffers/feed-buffer.ts

@@ -48,12 +48,14 @@ export class FeedBuffer {
     this.items.push(item);
     if (this.items.length > this._hwm) this._hwm = this.items.length;
 
-    // Fire onFull when buffer is at capacity AND enough new items have arrived
-    // since the last distillation (at least half the buffer replaced)
+    // Fire when enough new items have arrived since last distillation.
+    // 20 items ≈ 1.7 min of audio at ~12 items/min transcription rate.
+    // Distillation takes ~7s, so 20-item threshold gives 100s gap — safe margin.
+    // This means ~35 passes/hour, leaving <20 items undistilled at shutdown.
     const newSinceRearm = this._version - this._onFullVersion;
-    if (this.items.length >= this.maxSize
+    if (this.items.length >= 20
         && this._onFullCb && this._onFullArmed
-        && newSinceRearm >= Math.floor(this.maxSize / 2)) {
+        && newSinceRearm >= 20) {
       this._onFullArmed = false;
       const snapshot = [...this.items];
       queueMicrotask(() => this._onFullCb!(snapshot));

package/sinain-core/src/config.ts

@@ -2,8 +2,9 @@ import { readFileSync, existsSync } from "node:fs";
 import { resolve, dirname } from "node:path";
 import { fileURLToPath } from "node:url";
 import os from "node:os";
-import type { CoreConfig, AudioPipelineConfig, TranscriptionConfig, AnalysisConfig, EscalationConfig, OpenClawConfig, EscalationMode, EscalationTransport, LearningConfig, PrivacyConfig, PrivacyMatrix, PrivacyLevel, PrivacyRow } from "./types.js";
+import type { CoreConfig, AudioPipelineConfig, TranscriptionConfig, AnalysisConfig, EscalationConfig, OpenClawConfig, EscalationMode, LearningConfig, PrivacyConfig, PrivacyMatrix, PrivacyLevel, PrivacyRow } from "./types.js";
 import { PRESETS } from "./privacy/presets.js";
+import { loadAgentsConfig, findGatewayProfile, type AgentsConfig } from "./agents-loader.js";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
@@ -140,6 +141,26 @@ function loadPrivacyConfig(): PrivacyConfig {
 }
 
 export function loadConfig(): CoreConfig {
+  // sinain-agent/agents.json is the new single source of truth for the
+  // bare-agent + OpenClaw config that used to live in .env. Loading
+  // happens once at startup; null means the file isn't there (fresh
+  // checkout, custom layout, etc.) and we fall back to env defaults.
+  const agentsCfg = loadAgentsConfig();
+  if (agentsCfg) {
+    console.log(`[config] loaded agents config from ${agentsCfg.loadedFrom}`);
+  }
+
+  // Helpers that prefer agents.json values, then env, then a hardcoded
+  // default. Keeps .env working as a safety net during the migration.
+  const fromCfgStr = (cfgVal: string | undefined, envKey: string, def: string): string => {
+    if (cfgVal !== undefined && cfgVal !== "") return cfgVal;
+    return env(envKey, def);
+  };
+  const fromCfgInt = (cfgVal: number | undefined, envKey: string, def: number): number => {
+    if (cfgVal !== undefined) return cfgVal;
+    return intEnv(envKey, def);
+  };
+
   const audioConfig: AudioPipelineConfig = {
     device: env("AUDIO_DEVICE", "BlackHole 2ch"),
     sampleRate: intEnv("AUDIO_SAMPLE_RATE", 16000),
@@ -196,30 +217,57 @@ export function loadConfig(): CoreConfig {
       .split(",").map(s => s.trim()).filter(Boolean),
     timeout: intEnv("ANALYSIS_TIMEOUT", 15000),
     pushToFeed: boolEnv("AGENT_PUSH_TO_FEED", true),
-    debounceMs: intEnv("AGENT_DEBOUNCE_MS", 3000),
-    maxIntervalMs: intEnv("AGENT_MAX_INTERVAL_MS", 30000),
+    // analyzer pacing: agents.json `analyzer` block, fall back to env
+    debounceMs: fromCfgInt(agentsCfg?.analyzer?.debounceMs, "AGENT_DEBOUNCE_MS", 3000),
+    maxIntervalMs: fromCfgInt(agentsCfg?.analyzer?.maxIntervalMs, "AGENT_MAX_INTERVAL_MS", 30000),
     cooldownMs: intEnv("AGENT_COOLDOWN_MS", 10000),
     maxAgeMs: intEnv("AGENT_MAX_AGE_MS", 120000),
     historyLimit: intEnv("AGENT_HISTORY_LIMIT", 50),
   };
 
-  const escalationMode = env("ESCALATION_MODE", "rich") as EscalationMode;
+  // escalation policy: agents.json `escalation` block, fall back to env.
+  // Mode is runtime-mutable via the overlay's flash-icon selector; this only
+  // sets the boot-time default. (Transport is no longer a setting — per-lane
+  // agent choice in the overlay determines WS vs HTTP dispatch.)
+  const escalationMode = fromCfgStr(agentsCfg?.escalation?.mode, "ESCALATION_MODE", "rich") as EscalationMode;
   const escalationConfig: EscalationConfig = {
     mode: escalationMode,
-    cooldownMs: intEnv("ESCALATION_COOLDOWN_MS", 30000),
-    staleMs: intEnv("ESCALATION_STALE_MS", 90000),
-    transport: env("ESCALATION_TRANSPORT", "auto") as EscalationTransport,
+    cooldownMs: fromCfgInt(agentsCfg?.escalation?.cooldownMs, "ESCALATION_COOLDOWN_MS", 30000),
+    staleMs: fromCfgInt(agentsCfg?.escalation?.staleMs, "ESCALATION_STALE_MS", 90000),
   };
 
+  // OpenClaw gateway config: pick the first openclaw-TYPED profile
+  // (preferring the canonical name "openclaw"). This lets users add
+  // server-side variants — `nemoclaw`, `nanoclaw-prod`, etc. — as full
+  // peers; the WS client connects to whichever shows up first in the
+  // profiles list. Removing all openclaw-typed profiles (and unsetting
+  // the env vars) leaves gatewayWsUrl empty → registerBareAgent skips
+  // injecting any gateway entry into the roster, and escalator.start()
+  // skips the WS connect.
+  //
+  // Limitation: only ONE gateway URL gets a live WS client today. Adding
+  // multiple openclaw-typed profiles with different URLs lets the user
+  // pick them in the overlay roster, but dispatch always uses the first
+  // one's connection params. True per-profile WS clients is a follow-up.
+  const gateway = findGatewayProfile(agentsCfg);
+  const openclawProfile = gateway?.profile;
+  // wsUrl/hookUrl default to empty so "delete the openclaw profile" genuinely
+  // disables the gateway. The shipped agents.example.json includes the
+  // profile with localhost defaults, so fresh installs still work out of
+  // the box; removal is an explicit opt-out.
   const openclawConfig: OpenClawConfig = {
-    gatewayWsUrl: envAllowEmpty("OPENCLAW_WS_URL", "OPENCLAW_GATEWAY_WS_URL", "ws://localhost:18789"),
-    gatewayToken: env("OPENCLAW_WS_TOKEN", env("OPENCLAW_GATEWAY_TOKEN", "")),
-    hookUrl: envAllowEmpty("OPENCLAW_HTTP_URL", "OPENCLAW_HOOK_URL", "http://localhost:18789/hooks/agent"),
-    hookToken: env("OPENCLAW_HTTP_TOKEN", env("OPENCLAW_HOOK_TOKEN", "")),
-    sessionKey: env("OPENCLAW_SESSION_KEY", "agent:main:sinain"),
-    phase1TimeoutMs: intEnv("OPENCLAW_PHASE1_TIMEOUT_MS", 30_000),
-    phase2TimeoutMs: intEnv("OPENCLAW_PHASE2_TIMEOUT_MS", 120_000),
-    pingIntervalMs: intEnv("OPENCLAW_PING_INTERVAL_MS", 30_000),
+    gatewayWsUrl: fromCfgStr(openclawProfile?.wsUrl, "OPENCLAW_WS_URL",
+      envAllowEmpty("OPENCLAW_GATEWAY_WS_URL", undefined, "")),
+    gatewayToken: fromCfgStr(openclawProfile?.wsToken, "OPENCLAW_WS_TOKEN",
+      env("OPENCLAW_GATEWAY_TOKEN", "")),
+    hookUrl: fromCfgStr(openclawProfile?.httpUrl, "OPENCLAW_HTTP_URL",
+      envAllowEmpty("OPENCLAW_HOOK_URL", undefined, "")),
+    hookToken: fromCfgStr(openclawProfile?.httpToken, "OPENCLAW_HTTP_TOKEN",
+      env("OPENCLAW_HOOK_TOKEN", "")),
+    sessionKey: fromCfgStr(openclawProfile?.sessionKey, "OPENCLAW_SESSION_KEY", "agent:main:sinain"),
+    phase1TimeoutMs: fromCfgInt(openclawProfile?.phase1TimeoutMs, "OPENCLAW_PHASE1_TIMEOUT_MS", 30_000),
+    phase2TimeoutMs: fromCfgInt(openclawProfile?.phase2TimeoutMs, "OPENCLAW_PHASE2_TIMEOUT_MS", 120_000),
+    pingIntervalMs: fromCfgInt(openclawProfile?.pingIntervalMs, "OPENCLAW_PING_INTERVAL_MS", 30_000),
   };
 
   const situationDir = env("OPENCLAW_WORKSPACE_DIR", "~/.openclaw/workspace");
@@ -236,6 +284,19 @@ export function loadConfig(): CoreConfig {
 
   const privacyConfig = loadPrivacyConfig();
 
+  // autoApproveTools: agents.json top-level field, falls back to
+  // SINAIN_AUTO_APPROVE_TOOLS env. Space-separated tool names or prefix
+  // patterns (suffix *) that sinain-core auto-approves at /spawn/approve
+  // without routing to the overlay.
+  const autoApproveRaw = fromCfgStr(
+    agentsCfg?.autoApproveTools,
+    "SINAIN_AUTO_APPROVE_TOOLS",
+    "Read Glob Grep Ls Cat mcp__sinain*",
+  );
+  const permissionsConfig = {
+    autoApproveTools: autoApproveRaw.split(/\s+/).filter((t) => t.length > 0),
+  };
+
   return {
     port: intEnv("PORT", 9500),
     audioConfig,
@@ -251,5 +312,6 @@ export function loadConfig(): CoreConfig {
     costDisplayEnabled: boolEnv("COST_DISPLAY_ENABLED", false),
     learningConfig,
     privacyConfig,
+    permissionsConfig,
   };
 }