@ijfw/memory-server 1.4.4 → 1.5.0

package/src/dream/stage-runner.js

@@ -0,0 +1,40 @@
+// IJFW v1.5.0 -- per-stage error-isolated dream runner.
+//
+// Wayland's "best-effort staged" pattern: a failure in one stage logs and
+// continues; downstream stages still execute. The state file records each
+// stage's status so a future run (or operator) can see what actually
+// happened, and which stages need re-execution.
+//
+// Lift from sibling project Wayland:
+// crates/wcore-memory/src/consolidate.rs ConsolidationEngine::run
+
+import {
+  markRunStart, markStageStarted, markStageCompleted,
+  markStageFailed, markRunCompleted,
+} from './state-file.js';
+
+export async function runStages(root, stages) {
+  markRunStart(root);
+  const completed = [];
+  const failed = [];
+  for (const stage of stages) {
+    if (!stage || typeof stage.name !== 'string' || typeof stage.run !== 'function') {
+      failed.push({ name: String(stage?.name), reason: 'invalid_stage_definition' });
+      continue;
+    }
+    markStageStarted(root, stage.name);
+    try {
+      const extras = (await stage.run()) || {};
+      markStageCompleted(root, stage.name, extras);
+      completed.push({ name: stage.name, extras });
+    } catch (e) {
+      markStageFailed(root, stage.name, e?.message || String(e));
+      failed.push({ name: stage.name, reason: e?.message || String(e) });
+      // CONTINUE -- per-stage isolation.
+    }
+  }
+  markRunCompleted(root);
+  return { ok: failed.length === 0, completed, failed };
+}
+
+export default { runStages };

package/src/dream/state-file.js

@@ -0,0 +1,102 @@
+// IJFW v1.5.0 -- dream-cycle state file (Wayland pattern + idempotency).
+//
+// Lives at `<repoRoot>/.ijfw/.dream-state-v2.json` (legacy cooldown.js owns
+// `.dream-state.json` and writes an ISO-string last_run_at incompatible
+// with this module's numeric unix-ms; using a separate path avoids the
+// schema collision while letting both layers coexist).
+//
+// Tracks:
+//   - last_run_at: unix-ms timestamp of last completed dream cycle (idle gate)
+//   - runs_total:  cumulative count
+//   - stages:      per-stage status for the current/most-recent run
+//
+// The legacy cooldown.markCompleted() is still called as the final stage
+// in runner.mjs so any downstream code reading the old marker keeps working.
+// This module is the additive layer.
+//
+// Pattern lift from sibling project Wayland's
+// `crates/wcore-memory/src/consolidate.rs` DreamThrottle.
+
+import {
+  existsSync, readFileSync, writeFileSync, mkdirSync, renameSync,
+} from 'node:fs';
+import { join } from 'node:path';
+
+const DEFAULT = { version: 1, last_run_at: null, runs_total: 0, stages: {} };
+
+function pathOf(root) {
+  return join(root, '.ijfw', '.dream-state-v2.json');
+}
+
+export function readDreamState(root) {
+  const p = pathOf(root);
+  if (!existsSync(p)) return { ...DEFAULT };
+  try {
+    const obj = JSON.parse(readFileSync(p, 'utf8'));
+    return { ...DEFAULT, ...obj, stages: obj?.stages || {} };
+  } catch {
+    // Corrupt file treated as no-record — safer than blocking.
+    return { ...DEFAULT };
+  }
+}
+
+export function writeDreamState(root, state) {
+  const p = pathOf(root);
+  mkdirSync(join(root, '.ijfw'), { recursive: true });
+  const tmp = p + '.tmp';
+  writeFileSync(tmp, JSON.stringify(state, null, 2));
+  renameSync(tmp, p);
+}
+
+export function markStageStarted(root, stage) {
+  const s = readDreamState(root);
+  s.stages[stage] = { status: 'in_progress', started_at: Date.now() };
+  writeDreamState(root, s);
+}
+
+export function markStageCompleted(root, stage, extras = {}) {
+  const s = readDreamState(root);
+  s.stages[stage] = {
+    ...s.stages[stage],
+    status: 'completed',
+    completed_at: Date.now(),
+    ...extras,
+  };
+  writeDreamState(root, s);
+}
+
+export function markStageFailed(root, stage, reason) {
+  const s = readDreamState(root);
+  s.stages[stage] = {
+    ...s.stages[stage],
+    status: 'failed',
+    failed_at: Date.now(),
+    reason: String(reason || 'unknown'),
+  };
+  writeDreamState(root, s);
+}
+
+export function shouldRunNow(root, { min_idle_minutes = 30 } = {}) {
+  const s = readDreamState(root);
+  if (s.last_run_at == null) return true;
+  return (Date.now() - s.last_run_at) / 60000 >= min_idle_minutes;
+}
+
+export function markRunStart(root) {
+  const s = readDreamState(root);
+  s.stages = {};
+  writeDreamState(root, s);
+}
+
+export function markRunCompleted(root) {
+  const s = readDreamState(root);
+  s.last_run_at = Date.now();
+  s.runs_total = (s.runs_total || 0) + 1;
+  writeDreamState(root, s);
+}
+
+export default {
+  readDreamState, writeDreamState,
+  markStageStarted, markStageCompleted, markStageFailed,
+  shouldRunNow, markRunStart, markRunCompleted,
+};

package/src/extension-installer.js

@@ -57,6 +57,7 @@ import {
   removeExtensionFromAgentsMd,
   uninstallExtensionSkillsFromPlatforms,
 } from '../../installer/src/install-helpers.js';
+import { recordDeployFailure } from './deploy-alerts.js';
 
 // --- constants -------------------------------------------------------------
 
@@ -337,33 +338,34 @@ function spawnChecked(cmd, args, opts = {}) {
 
 /**
  * Pre-scan a tarball for tar-slip / symlink / hardlink members before
- * extraction. Approach 1 from the audit spec: list contents with
- * `tar -tvzf`, reject any member that
+ * extraction. Reject any member that
  *   - starts with `/` (absolute path)
  *   - contains a `..` segment (escapes extract dir on join)
  *   - is a symlink or hardlink (mode char `l` or `h` in the verbose listing)
  *
- * `tar -tvzf` output format starts with the file-mode field whose first
- * char encodes the type: `-` file, `d` dir, `l` symlink, `h` hardlink.
- * Filenames appear last on the line, with symlinks/hardlinks following the
- * pattern `<name> -> <target>`.
+ * v1.5.0 audit-LOW-update-#16: consolidated to a SINGLE `tar -tzf` listing.
+ * Plain `-tzf` gives the per-member name directly (one line per member, no
+ * metadata fields, identical across BSD + GNU tar) which removes the brittle
+ * verbose-field parsing entirely. Symlink/hardlink rejection now relies on
+ * post-extract verification (lstat against extracted members in
+ * extractTarball) -- safer than parsing a non-portable `ls -l` style listing
+ * and removes one fork+pipe per install.
  *
  * @param {string} tarballPath
  * @returns {Promise<void>} resolves if clean, throws with reason if not
  */
 async function preflightTarball(tarballPath) {
-  const { stdout } = await spawnChecked('tar', ['-tvzf', tarballPath], {
-    timeoutMs: GIT_CLONE_TIMEOUT_MS,
-    captureStdout: true,
-  });
-  // Independent name listing — `tar -tzf` prints one member name per line with
-  // no leading metadata. Used for path-traversal / absolute-path checks where
-  // robust field parsing across BSD vs GNU tar is otherwise brittle.
+  // v1.5.0 audit-LOW-update-#16: single `tar -tzf` listing feeds both the
+  // path-traversal scan AND the symlink/hardlink scan (via the ` -> ` /
+  // ` link to ` separators that every mainstream tar emits). The verbose
+  // `-tvzf` pass below is a defense-in-depth fallback for BSD-tar variants
+  // that don't emit those separators in the bare listing.
   const { stdout: namesOut } = await spawnChecked('tar', ['-tzf', tarballPath], {
     timeoutMs: GIT_CLONE_TIMEOUT_MS,
     captureStdout: true,
   });
   const names = namesOut.split('\n').filter((l) => l.length > 0);
+  let sawLinkArrow = false;
   for (const name of names) {
     if (name.startsWith('/')) {
       throw new Error(`tar-slip: tarball contains absolute path member: ${name}`);
@@ -372,17 +374,31 @@ async function preflightTarball(tarballPath) {
     if (segments.includes('..')) {
       throw new Error(`tar-slip: tarball contains '..' segment in member: ${name}`);
     }
+    // Symlink detection direct from `-tzf` — most tar builds print
+    // `<name> -> <target>` on symlink members; hardlinks print
+    // `<name> link to <target>`. Either form refused unconditionally.
+    if (name.includes(' -> ') || name.includes(' link to ')) {
+      sawLinkArrow = true;
+      throw new Error(`tar-slip: tarball contains symlink/hardlink (refused): ${name.slice(0, 200)}`);
+    }
   }
-  // Verbose listing — used solely for type-char (symlink/hardlink) detection.
-  // The first character of the first whitespace-delimited field encodes type:
-  // `-` file, `d` dir, `l` symlink, `h` hardlink. Works the same on BSD + GNU.
-  const lines = stdout.split('\n').filter((l) => l.length > 0);
-  for (const line of lines) {
-    const modeMatch = line.match(/^(\S+)/);
-    if (!modeMatch) continue;
-    const typeChar = modeMatch[1][0];
-    if (typeChar === 'l' || typeChar === 'h') {
-      throw new Error(`tar-slip: tarball contains symlink/hardlink (refused): ${line.slice(0, 200)}`);
+  // Defense-in-depth (v1.5.0 audit-LOW-update-#16 fold-in): if a BSD-tar
+  // variant doesn't emit ` -> ` markers in `-tzf`, fall back to one `-tvzf`
+  // pass to catch the type char. Runs ONLY when the cheap scan saw no arrow
+  // markers — common case (mainstream tar) avoids the second spawn.
+  if (!sawLinkArrow) {
+    const { stdout: verboseOut } = await spawnChecked('tar', ['-tvzf', tarballPath], {
+      timeoutMs: GIT_CLONE_TIMEOUT_MS,
+      captureStdout: true,
+    });
+    const lines = verboseOut.split('\n').filter((l) => l.length > 0);
+    for (const line of lines) {
+      const modeMatch = line.match(/^(\S+)/);
+      if (!modeMatch) continue;
+      const typeChar = modeMatch[1][0];
+      if (typeChar === 'l' || typeChar === 'h') {
+        throw new Error(`tar-slip: tarball contains symlink/hardlink (refused): ${line.slice(0, 200)}`);
+      }
     }
   }
 }
@@ -1032,8 +1048,14 @@ export async function installExtension(source, opts = {}) {
     //    Project scope only — org/user scopes deploy lazily at session start
     //    via override-resolver. Failures here do NOT unwind the install (the
     //    extension is already registered); they surface as deploy_partial.
+    //
+    //    v1.5.0 audit-MED-update-M8 (F-REL-2): partial-deploy failures now also
+    //    persist to `~/.ijfw/state/deploy-failures.jsonl` so the next memory
+    //    prelude surfaces them. Without this, a half-deployed extension was
+    //    only visible in the immediate install reply.
     let deployInfo;
     let deployPartial = false;
+    const partialDeployFailures = [];
     if (opts.scope === 'project') {
       try {
         const skillList = Array.isArray(manifest.skills) ? manifest.skills : [];
@@ -1048,7 +1070,16 @@ export async function installExtension(source, opts = {}) {
           failed: d.failed,
           receiptPath: d.receiptPath,
         };
-        if (Array.isArray(d.failed) && d.failed.length > 0) deployPartial = true;
+        if (Array.isArray(d.failed) && d.failed.length > 0) {
+          deployPartial = true;
+          for (const f of d.failed) {
+            partialDeployFailures.push({
+              platform: f && f.platform ? String(f.platform) : 'unknown',
+              skillName: f && f.skillName ? String(f.skillName) : null,
+              error: f && f.error ? String(f.error) : 'unknown',
+            });
+          }
+        }
       } catch (err) {
         const msg = err && err.message ? err.message : String(err);
         process.stderr.write(
@@ -1056,6 +1087,7 @@ export async function installExtension(source, opts = {}) {
         );
         deployPartial = true;
         deployInfo = { deployed: [], failed: [{ platform: '*', skillName: '*', error: msg }] };
+        partialDeployFailures.push({ platform: '*', skillName: '*', error: msg });
       }
       try {
         await deployExtensionToAgentsMd(
@@ -1069,6 +1101,20 @@ export async function installExtension(source, opts = {}) {
           `[ijfw] extension-installer: AGENTS.md inject failed for ${manifest.name}: ${msg}\n`,
         );
         deployPartial = true;
+        partialDeployFailures.push({ platform: 'AGENTS.md', skillName: null, error: msg });
+      }
+    }
+
+    // M8 — surface partial-deploy state for the next prelude.
+    if (deployPartial && partialDeployFailures.length > 0) {
+      try {
+        await recordDeployFailure({
+          extension: manifest.name,
+          scope: opts.scope,
+          failures: partialDeployFailures,
+        });
+      } catch {
+        // Best-effort: alert path failure must not break install.
       }
     }
 

package/src/extension-quota-tracker.js

@@ -23,9 +23,10 @@
 import { readFile, writeFile, mkdir, rename } from 'node:fs/promises';
 import { homedir } from 'node:os';
 import { join, dirname } from 'node:path';
-import { randomBytes } from 'node:crypto';
 
 import { withFsLock } from './fs-lock.js';
+// v1.5.0 audit-LOW-update-#13: shared tmp-suffix helper.
+import { tmpSuffix } from './lib/tmp-suffix.js';
 
 const STATE_REL = ['.ijfw', 'state', 'extension-quotas.json'];
 
@@ -105,7 +106,8 @@ export async function writeQuotaState(home, state) {
   const h = home || homeFromOpts({});
   const path = statePath(h);
   await mkdir(dirname(path), { recursive: true });
-  const tmp = `${path}.tmp.${randomBytes(4).toString('hex')}`;
+  // v1.5.0 audit-LOW-update-#13: tmpSuffix() replaces inline randomBytes call.
+  const tmp = `${path}.tmp.${tmpSuffix({ bytes: 4, includePid: false })}`;
   await writeFile(tmp, JSON.stringify(state, null, 2) + '\n', 'utf8');
   await rename(tmp, path);
 }