@totalreclaw/totalreclaw 3.3.1-rc.8 → 3.3.1

package/dist/tool-gating.js

@@ -0,0 +1,58 @@
+/**
+ * Tool gating predicate for 3.2.0 — the `before_tool_call` hook in index.ts
+ * delegates to this module so the logic is testable without standing up a
+ * full OpenClaw plugin host.
+ *
+ * Scope: the 3.2.0 state machine has two states (`fresh`, `active`). Memory
+ * tools are blocked when state is anything other than `active`. Billing +
+ * setup-adjacent tools remain usable — users need to be able to upgrade,
+ * migrate, and start onboarding before their vault is active.
+ *
+ * This module imports ONLY types + the state resolver. No I/O beyond what
+ * `resolveOnboardingState` already does; no network; no env reads.
+ */
+/**
+ * Tool names gated on `state=active`. Keep in sync with the actual
+ * `registerTool` calls in `index.ts`. Anything NOT in this set is always
+ * callable (e.g. totalreclaw_upgrade, totalreclaw_migrate,
+ * totalreclaw_onboarding_start, totalreclaw_setup).
+ */
+export const GATED_TOOL_NAMES = Object.freeze([
+    'totalreclaw_remember',
+    'totalreclaw_recall',
+    'totalreclaw_forget',
+    'totalreclaw_export',
+    'totalreclaw_status',
+    'totalreclaw_consolidate',
+    'totalreclaw_pin',
+    'totalreclaw_unpin',
+    'totalreclaw_retype',
+    'totalreclaw_set_scope',
+    'totalreclaw_import_from',
+    'totalreclaw_import_batch',
+]);
+/**
+ * Decide whether a specific tool call should be blocked given the current
+ * onboarding state. Does not read any files — caller resolves state first
+ * (that lets tests stub state without touching disk).
+ */
+export function decideToolGate(toolName, state) {
+    if (!toolName)
+        return { block: false };
+    if (!GATED_TOOL_NAMES.includes(toolName))
+        return { block: false };
+    if (state?.onboardingState === 'active')
+        return { block: false };
+    return {
+        block: true,
+        blockReason: 'TotalReclaw onboarding required. Run `openclaw totalreclaw onboard` ' +
+            'in a terminal (or call the `totalreclaw_onboarding_start` tool for ' +
+            'details). Memory tools are gated until the user completes setup.',
+    };
+}
+/**
+ * Convenience predicate — useful for tests + documentation.
+ */
+export function isGatedToolName(toolName) {
+    return GATED_TOOL_NAMES.includes(toolName);
+}

package/download-ux.ts

@@ -0,0 +1,91 @@
+/**
+ * download-ux.ts — Wrapper for heavy first-call downloads (rc.16, fixes #92).
+ *
+ * Wraps a download promise with:
+ *   - per-attempt timeout (default 600s, override via TOTALRECLAW_ONNX_INSTALL_TIMEOUT in seconds)
+ *   - 60s keep-alive log so slow-bandwidth users don't think it's frozen
+ *   - 3-attempt exponential-backoff retry (per-attempt timeout grows 1x/2x/4x)
+ *   - loud actionable error after exhaustion
+ *
+ * No third-party imports here — pure stdlib so the unit test can exercise it
+ * without pulling the heavy `@huggingface/transformers` chain.
+ */
+
+const DEFAULT_DOWNLOAD_TIMEOUT_MS = 600_000;
+const KEEPALIVE_INTERVAL_MS = 60_000;
+const MAX_DOWNLOAD_ATTEMPTS = 3;
+
+export function getDownloadTimeoutMs(): number {
+  const raw = process.env.TOTALRECLAW_ONNX_INSTALL_TIMEOUT;
+  if (!raw) return DEFAULT_DOWNLOAD_TIMEOUT_MS;
+  const parsed = Number(raw);
+  if (!Number.isFinite(parsed) || parsed <= 0) return DEFAULT_DOWNLOAD_TIMEOUT_MS;
+  // Spec accepts seconds; convert to ms.
+  return Math.floor(parsed * 1000);
+}
+
+export interface DownloadWithUXOpts {
+  /** Override the per-attempt base timeout in ms (env var takes precedence by default). */
+  timeoutMs?: number;
+  /** Override the keep-alive cadence in ms. */
+  keepaliveMs?: number;
+  /** Override the max attempts. */
+  maxAttempts?: number;
+  /** Logger override (defaults to console.error). */
+  log?: (msg: string) => void;
+  /** Sleep override for tests; defaults to setTimeout. */
+  sleep?: (ms: number) => Promise<void>;
+}
+
+export async function downloadWithUX<T>(
+  label: string,
+  download: () => Promise<T>,
+  opts?: DownloadWithUXOpts,
+): Promise<T> {
+  const baseTimeoutMs = opts?.timeoutMs ?? getDownloadTimeoutMs();
+  const keepaliveMs = opts?.keepaliveMs ?? KEEPALIVE_INTERVAL_MS;
+  const maxAttempts = opts?.maxAttempts ?? MAX_DOWNLOAD_ATTEMPTS;
+  const log = opts?.log ?? ((msg: string) => console.error(msg));
+  const sleep = opts?.sleep ?? ((ms: number) => new Promise(r => setTimeout(r, ms)));
+
+  let lastErr: unknown = null;
+
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    const attemptTimeoutMs = baseTimeoutMs * Math.pow(2, attempt - 1);
+    const startedAt = Date.now();
+    const keepaliveTimer = setInterval(() => {
+      const elapsedSec = Math.floor((Date.now() - startedAt) / 1000);
+      log(`[TotalReclaw] ${label}: still downloading… (${elapsedSec}s elapsed, attempt ${attempt}/${maxAttempts})`);
+    }, keepaliveMs);
+
+    try {
+      const result = await Promise.race([
+        download(),
+        new Promise<never>((_, reject) =>
+          setTimeout(
+            () => reject(new Error(`Download timeout after ${Math.floor(attemptTimeoutMs / 1000)}s (attempt ${attempt}/${maxAttempts})`)),
+            attemptTimeoutMs,
+          ),
+        ),
+      ]);
+      clearInterval(keepaliveTimer);
+      return result;
+    } catch (err) {
+      clearInterval(keepaliveTimer);
+      lastErr = err;
+      const msg = err instanceof Error ? err.message : String(err);
+      if (attempt < maxAttempts) {
+        const backoffMs = Math.min(5_000 * Math.pow(2, attempt - 1), 30_000);
+        log(`[TotalReclaw] ${label}: attempt ${attempt} failed (${msg}). Retrying in ${Math.floor(backoffMs / 1000)}s…`);
+        await sleep(backoffMs);
+      }
+    }
+  }
+
+  const finalMsg = lastErr instanceof Error ? lastErr.message : String(lastErr);
+  throw new Error(
+    `[TotalReclaw] Embedding model download failed after ${maxAttempts} attempts (last error: ${finalMsg}). ` +
+      `Check your network connection and retry: \`openclaw plugins install totalreclaw\`. ` +
+      `On slow connections, set TOTALRECLAW_ONNX_INSTALL_TIMEOUT=1200 (in seconds) to extend the per-attempt timeout.`,
+  );
+}

package/embedder-cache.ts

@@ -0,0 +1,230 @@
+/**
+ * embedder-cache.ts — pure-FS reader for the lazy embedder bundle (rc.22+).
+ *
+ * Scanner-isolation note: this module reads from disk AND verifies SHA-256
+ * hashes. It MUST NOT contain any of the network-trigger substrings the
+ * OpenClaw skill scanner gates on — see `skill/scripts/check-scanner.mjs`
+ * for the rule list. The network side of the lazy-retrieval flow lives in a
+ * sibling module (the downloader), and the orchestrator imports both.
+ *
+ * Responsibilities:
+ *   - Resolve the on-disk cache layout (`<root>/v1/`, with `manifest.json`
+ *     + `node_modules/` + `model/`).
+ *   - Synchronously load + parse the manifest JSON.
+ *   - Verify the cache is intact: every file listed in `manifest.files`
+ *     exists at the expected path with the SHA-256 hash declared in the
+ *     manifest. Any mismatch invalidates the cache so the loader rebuilds.
+ *
+ * The manifest format is the contract between this file and the bundle
+ * generation script (`scripts/build-embedder-bundle.mjs`):
+ *   {
+ *     "version": "v1",                          // bundle format version
+ *     "model_id": "harrier-oss-270m-q4",        // semantic model identifier
+ *     "dimension": 640,                          // output vector size
+ *     "tarball_sha256": "<hex>",                 // informational only here
+ *     "tarball_size_bytes": <int>,               // informational only here
+ *     "files": [
+ *       { "path": "node_modules/.../foo.js", "sha256": "<hex>", "size": <int> },
+ *       ...
+ *     ]
+ *   }
+ *
+ * Hard rule for this file: stdlib only — `node:fs` + `node:crypto` +
+ * `node:path`. No env reads, no remote retrievals.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import crypto from 'node:crypto';
+
+/** Bundle format version — bump only when the on-disk layout changes. */
+export const BUNDLE_FORMAT_VERSION = 'v1' as const;
+
+/**
+ * Layout: `<cacheRoot>/<BUNDLE_FORMAT_VERSION>/`. The version subdirectory
+ * lets us ship `v2/` side-by-side with `v1/` later (e.g. for a distilled
+ * model) without invalidating active vaults.
+ */
+export interface CacheLayout {
+  /** Top-level embedder cache directory (e.g. `~/.totalreclaw/embedder/`). */
+  root: string;
+  /** Versioned bundle root (e.g. `~/.totalreclaw/embedder/v1/`). */
+  versionRoot: string;
+  /** Path to the manifest JSON file. */
+  manifestPath: string;
+  /** Path to the extracted node_modules tree (transformers + onnxruntime). */
+  nodeModulesPath: string;
+  /** Path to the extracted ONNX model directory. */
+  modelPath: string;
+}
+
+export function resolveCacheLayout(cacheRoot: string): CacheLayout {
+  const versionRoot = path.join(cacheRoot, BUNDLE_FORMAT_VERSION);
+  return {
+    root: cacheRoot,
+    versionRoot,
+    manifestPath: path.join(versionRoot, 'manifest.json'),
+    nodeModulesPath: path.join(versionRoot, 'node_modules'),
+    modelPath: path.join(versionRoot, 'model'),
+  };
+}
+
+export interface BundleManifestFileEntry {
+  /** Path RELATIVE to the version-root directory (e.g. `node_modules/foo/bar.js`). */
+  path: string;
+  /** Lowercase hex SHA-256 of the file's content. */
+  sha256: string;
+  /** Byte size — informational; not load-bearing for verification. */
+  size: number;
+}
+
+export interface BundleManifest {
+  /** Bundle format version. MUST match `BUNDLE_FORMAT_VERSION`. */
+  version: string;
+  /** Semantic model id, e.g. `"harrier-oss-270m-q4"`. */
+  model_id: string;
+  /** Output vector dimensionality. */
+  dimension: number;
+  /** Lowercase hex SHA-256 of the entire .tar.gz tarball. */
+  tarball_sha256: string;
+  /** Tarball size in bytes. */
+  tarball_size_bytes: number;
+  /** Per-file integrity table — used by the loader after extraction. */
+  files: BundleManifestFileEntry[];
+}
+
+/**
+ * Synchronously read + parse the manifest. Returns `null` when the file
+ * is missing, unreadable, or malformed JSON — callers treat any of those
+ * as a cache miss.
+ */
+export function readManifest(layout: CacheLayout): BundleManifest | null {
+  let raw: string;
+  try {
+    raw = fs.readFileSync(layout.manifestPath, 'utf8');
+  } catch {
+    return null;
+  }
+  try {
+    const parsed = JSON.parse(raw) as Partial<BundleManifest>;
+    if (!isValidManifestShape(parsed)) return null;
+    return parsed as BundleManifest;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Shape guard for a parsed manifest. Strict on every required field; lax
+ * on extras so bundle-generation tools may add diagnostic fields without
+ * tripping verification.
+ */
+export function isValidManifestShape(obj: unknown): obj is BundleManifest {
+  if (!obj || typeof obj !== 'object') return false;
+  const m = obj as Record<string, unknown>;
+  if (typeof m.version !== 'string' || m.version.length === 0) return false;
+  if (typeof m.model_id !== 'string' || m.model_id.length === 0) return false;
+  if (typeof m.dimension !== 'number' || !Number.isFinite(m.dimension) || m.dimension <= 0) return false;
+  if (typeof m.tarball_sha256 !== 'string' || !/^[0-9a-f]{64}$/.test(m.tarball_sha256)) return false;
+  if (typeof m.tarball_size_bytes !== 'number' || m.tarball_size_bytes < 0) return false;
+  if (!Array.isArray(m.files)) return false;
+  for (const entry of m.files as unknown[]) {
+    if (!entry || typeof entry !== 'object') return false;
+    const e = entry as Record<string, unknown>;
+    if (typeof e.path !== 'string' || e.path.length === 0) return false;
+    if (typeof e.sha256 !== 'string' || !/^[0-9a-f]{64}$/.test(e.sha256)) return false;
+    if (typeof e.size !== 'number' || e.size < 0) return false;
+    // Block path-traversal up front — any `..` segment, absolute path,
+    // or backslash makes the entry untrusted.
+    if (e.path.includes('..') || e.path.startsWith('/') || e.path.includes('\\')) return false;
+  }
+  return true;
+}
+
+/**
+ * Compute the SHA-256 of a file's contents. Returns null on any IO error.
+ * Synchronous + buffered — files are small (<10 MB each in the bundle).
+ */
+export function sha256OfFile(filePath: string): string | null {
+  try {
+    const buf = fs.readFileSync(filePath);
+    return crypto.createHash('sha256').update(buf).digest('hex');
+  } catch {
+    return null;
+  }
+}
+
+export interface VerifyResult {
+  ok: boolean;
+  /** First failure reason — empty when ok is true. */
+  reason: string;
+  /** When ok=false, the file that failed (relative path) or `''`. */
+  offendingPath: string;
+}
+
+/**
+ * Verify that every file listed in `manifest.files` exists at the
+ * expected path under `layout.versionRoot` with the declared hash.
+ *
+ * Returns ok=true only when:
+ *   - every entry's file exists,
+ *   - file size matches,
+ *   - SHA-256 matches.
+ *
+ * Bails on the FIRST failure — the loader's only branch on this is
+ * "discard cache + re-build", so we don't need to enumerate every fault.
+ */
+export function verifyCache(
+  layout: CacheLayout,
+  manifest: BundleManifest,
+): VerifyResult {
+  if (manifest.version !== BUNDLE_FORMAT_VERSION) {
+    return {
+      ok: false,
+      reason: `cache manifest version "${manifest.version}" does not match expected "${BUNDLE_FORMAT_VERSION}"`,
+      offendingPath: 'manifest.json',
+    };
+  }
+  for (const entry of manifest.files) {
+    const abs = path.join(layout.versionRoot, entry.path);
+    let stat: fs.Stats;
+    try {
+      stat = fs.statSync(abs);
+    } catch {
+      return { ok: false, reason: `cache missing file: ${entry.path}`, offendingPath: entry.path };
+    }
+    if (!stat.isFile()) {
+      return { ok: false, reason: `cache entry not a regular file: ${entry.path}`, offendingPath: entry.path };
+    }
+    if (stat.size !== entry.size) {
+      return {
+        ok: false,
+        reason: `cache size mismatch for ${entry.path}: expected ${entry.size}, got ${stat.size}`,
+        offendingPath: entry.path,
+      };
+    }
+    const actualHash = sha256OfFile(abs);
+    if (actualHash !== entry.sha256) {
+      return {
+        ok: false,
+        reason: `cache hash mismatch for ${entry.path}`,
+        offendingPath: entry.path,
+      };
+    }
+  }
+  return { ok: true, reason: '', offendingPath: '' };
+}
+
+/**
+ * Cheap pre-flight before a full verifyCache pass: does the manifest
+ * exist and parse to the expected shape with the expected version?
+ */
+export function quickCacheProbe(layout: CacheLayout): {
+  hasManifest: boolean;
+  manifest: BundleManifest | null;
+} {
+  const m = readManifest(layout);
+  if (!m) return { hasManifest: false, manifest: null };
+  if (m.version !== BUNDLE_FORMAT_VERSION) return { hasManifest: false, manifest: m };
+  return { hasManifest: true, manifest: m };
+}

package/embedder-loader.ts

@@ -0,0 +1,189 @@
+/**
+ * embedder-loader.ts — orchestrator for the lazy embedder bundle (rc.22+).
+ *
+ * Splits the work between the cache-reader sibling (pure FS + manifest
+ * verify) and the downloader sibling (HTTPS + tar extraction). This file
+ * imports from both; scanner-wise it stays away from env-reads and the
+ * scanner's network-trigger substrings, since merely importing the
+ * downloader does not trip either rule.
+ *
+ * Lifecycle:
+ *   1. `loadEmbedder(opts)` is called on first call to embed().
+ *   2. Probe the cache via `quickCacheProbe`. If a manifest with the
+ *      expected version is present and the cache verifies, skip to step 5.
+ *   3. Pull the manifest JSON from the GitHub Release pinned to the
+ *      caller's RC tag (via the downloader sibling).
+ *   4. Stream-download the bundle tarball, verify its SHA-256 against
+ *      the manifest, untar into the cache dir, then re-verify per-file
+ *      hashes. Refuse to use the cache on any mismatch.
+ *   5. `createRequire` from inside the cache's `node_modules/` and lazy-
+ *      load the bundled embedder + model.
+ */
+
+import path from 'node:path';
+import { Module, createRequire } from 'node:module';
+import {
+  resolveCacheLayout,
+  quickCacheProbe,
+  verifyCache,
+  isValidManifestShape,
+  BUNDLE_FORMAT_VERSION,
+  type BundleManifest,
+  type CacheLayout,
+} from './embedder-cache.js';
+import {
+  buildBundleUrl,
+  buildManifestUrl,
+  downloadAndExtractTarGz,
+  fetchManifestJson,
+  DEFAULT_BUNDLE_URL_TEMPLATE,
+  DEFAULT_MANIFEST_URL_TEMPLATE,
+} from './embedder-network.js';
+
+export interface LoadEmbedderOptions {
+  /** Top-level cache directory (e.g. `~/.totalreclaw/embedder/`). */
+  cacheRoot: string;
+  /** RC tag for URL templating, e.g. `"3.3.1-rc.22"`. */
+  rcTag: string;
+  /** Optional override for the bundle URL template (test injection). */
+  bundleUrlTemplate?: string;
+  /** Optional override for the manifest URL template (test injection). */
+  manifestUrlTemplate?: string;
+  /** Optional remote-loader override (test injection). */
+  fetchImpl?: typeof globalThis.fetch;
+  /** Optional logger. */
+  log?: (msg: string) => void;
+  /** Optional per-attempt timeout for the bundle download (ms). */
+  bundleTimeoutMs?: number;
+  /** Optional per-attempt timeout for the manifest pull (ms). */
+  manifestTimeoutMs?: number;
+}
+
+export interface LoadedEmbedder {
+  /** Path to the cache directory used. */
+  layout: CacheLayout;
+  /** Verified manifest. */
+  manifest: BundleManifest;
+  /** A `require` function bound to the embedder's node_modules tree. */
+  cacheRequire: NodeRequire;
+  /** True when the bundle was downloaded this call (vs. cache hit). */
+  wasFetched: boolean;
+}
+
+const DEFAULT_LOG = (msg: string) => console.error(msg);
+
+/**
+ * Top-level entry point. Idempotent: caching is by `cacheRoot` so repeat
+ * calls with a hot cache return immediately.
+ */
+export async function loadEmbedder(opts: LoadEmbedderOptions): Promise<LoadedEmbedder> {
+  const log = opts.log ?? DEFAULT_LOG;
+  const layout = resolveCacheLayout(opts.cacheRoot);
+
+  // --- Cache hit path -------------------------------------------------------
+  const probe = quickCacheProbe(layout);
+  if (probe.hasManifest && probe.manifest) {
+    const verify = verifyCache(layout, probe.manifest);
+    if (verify.ok) {
+      log(`[TotalReclaw] embedder: cache hit at ${layout.versionRoot} (model=${probe.manifest.model_id})`);
+      return {
+        layout,
+        manifest: probe.manifest,
+        cacheRequire: makeCacheRequire(layout),
+        wasFetched: false,
+      };
+    }
+    log(`[TotalReclaw] embedder: cache present but failed verify (${verify.reason}); rebuilding`);
+  } else {
+    log(`[TotalReclaw] embedder: no cache at ${layout.versionRoot}; pulling from GitHub Releases`);
+  }
+
+  // --- Build path -----------------------------------------------------------
+  const manifestUrl = buildManifestUrl(
+    { rcTag: opts.rcTag, bundleVersion: BUNDLE_FORMAT_VERSION },
+    opts.manifestUrlTemplate ?? DEFAULT_MANIFEST_URL_TEMPLATE,
+  );
+  const bundleUrl = buildBundleUrl(
+    { rcTag: opts.rcTag, bundleVersion: BUNDLE_FORMAT_VERSION },
+    opts.bundleUrlTemplate ?? DEFAULT_BUNDLE_URL_TEMPLATE,
+  );
+
+  const rawManifest = await fetchManifestJson(manifestUrl, {
+    fetchImpl: opts.fetchImpl,
+    log,
+    timeoutMs: opts.manifestTimeoutMs ?? 60_000,
+  });
+  if (!isValidManifestShape(rawManifest)) {
+    throw new Error(`embedder manifest at ${manifestUrl} failed shape validation`);
+  }
+  const manifest = rawManifest as BundleManifest;
+  if (manifest.version !== BUNDLE_FORMAT_VERSION) {
+    throw new Error(
+      `embedder manifest version "${manifest.version}" does not match plugin's expected "${BUNDLE_FORMAT_VERSION}"`,
+    );
+  }
+
+  await downloadAndExtractTarGz(bundleUrl, layout.versionRoot, manifest.tarball_sha256, {
+    fetchImpl: opts.fetchImpl,
+    log,
+    timeoutMs: opts.bundleTimeoutMs ?? 600_000,
+  });
+
+  // Persist the verified manifest alongside the extracted tree so the
+  // cache layout is self-describing on the next boot. Plain stdlib write.
+  const fs = await import('node:fs');
+  fs.writeFileSync(layout.manifestPath, JSON.stringify(manifest, null, 2), { encoding: 'utf8', mode: 0o644 });
+
+  // Re-run the integrity check against the on-disk tree.
+  const postVerify = verifyCache(layout, manifest);
+  if (!postVerify.ok) {
+    throw new Error(
+      `embedder bundle integrity check failed AFTER extraction: ${postVerify.reason}. ` +
+        `Cache at ${layout.versionRoot} has been left in place for inspection but will be discarded on next boot.`,
+    );
+  }
+
+  log(
+    `[TotalReclaw] embedder: bundle ready at ${layout.versionRoot} (model=${manifest.model_id}, files=${manifest.files.length})`,
+  );
+
+  return {
+    layout,
+    manifest,
+    cacheRequire: makeCacheRequire(layout),
+    wasFetched: true,
+  };
+}
+
+/**
+ * Build a `require` function rooted at the embedder cache's
+ * `node_modules/`. We anchor it on a synthetic `package.json` at the
+ * version-root so `require('@huggingface/transformers')` resolves
+ * normally inside that tree.
+ */
+export function makeCacheRequire(layout: CacheLayout): NodeRequire {
+  // Anchor on the version-root so node-module resolution starts inside
+  // the bundle's node_modules.
+  const anchor = path.join(layout.versionRoot, 'package.json');
+  // Append the cache node_modules to the global resolution path as a
+  // belt-and-braces guarantee that modules outside the bundle that might
+  // be transitively required still resolve from the host's tree.
+  if (!Module.globalPaths.includes(layout.nodeModulesPath)) {
+    Module.globalPaths.push(layout.nodeModulesPath);
+  }
+  return createRequire(anchor);
+}
+
+/**
+ * Destructive: remove the entire on-disk cache. Useful only as an
+ * escape hatch for repair flows. Returns true on success, false on error.
+ */
+export async function destroyCache(layout: CacheLayout): Promise<boolean> {
+  try {
+    const fs = await import('node:fs');
+    fs.rmSync(layout.versionRoot, { recursive: true, force: true });
+    return true;
+  } catch {
+    return false;
+  }
+}