npm - create-walle - Versions diffs - 0.9.30 → 0.9.32 - Mend

create-walle 0.9.30 → 0.9.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (112) hide show

package/template/wall-e/docs/skill-self-heal-design.html ADDED Viewed

@@ -0,0 +1,429 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>Wall-E Skill Self-Heal — Design Review</title>
+<style>
+  :root{
+    --bg:#0d1117; --panel:#161b22; --panel2:#1c2230; --border:#2a3240;
+    --ink:#e6edf3; --muted:#8b97a7; --faint:#5c6675;
+    --blue:#4c8dff; --green:#3fb950; --amber:#d29922; --red:#f85149;
+    --purple:#a371f7; --teal:#39c5cf;
+    --mono:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;
+  }
+  *{box-sizing:border-box}
+  body{margin:0;background:var(--bg);color:var(--ink);
+    font:15px/1.65 -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Helvetica,Arial,sans-serif;}
+  .wrap{max-width:960px;margin:0 auto;padding:40px 28px 96px;}
+  h1{font-size:30px;font-weight:700;letter-spacing:-.02em;margin:0 0 6px;}
+  h2{font-size:21px;font-weight:650;margin:46px 0 14px;padding-bottom:8px;
+    border-bottom:1px solid var(--border);letter-spacing:-.01em;}
+  h3{font-size:16px;font-weight:650;margin:26px 0 8px;color:var(--ink);}
+  p{margin:10px 0;color:#cdd6e0;}
+  a{color:var(--blue);text-decoration:none;} a:hover{text-decoration:underline;}
+  .sub{color:var(--muted);font-size:15px;margin:0 0 4px;}
+  .meta{color:var(--faint);font-size:13px;margin-top:8px;}
+  code{font-family:var(--mono);font-size:.86em;background:#222b38;color:#c8d3e0;
+    padding:1.5px 5px;border-radius:5px;border:1px solid #2c3645;}
+  pre{background:#0a0e14;border:1px solid var(--border);border-radius:10px;
+    padding:14px 16px;overflow:auto;font-family:var(--mono);font-size:13px;line-height:1.55;color:#c8d3e0;}
+  pre .c{color:var(--faint);} pre .k{color:var(--purple);} pre .s{color:var(--green);} pre .f{color:var(--teal);}
+  .panel{background:var(--panel);border:1px solid var(--border);border-radius:12px;padding:18px 20px;margin:16px 0;}
+  .grid{display:grid;gap:14px;}
+  .cols-2{grid-template-columns:1fr 1fr;} .cols-3{grid-template-columns:1fr 1fr 1fr;}
+  @media(max-width:720px){.cols-2,.cols-3{grid-template-columns:1fr;}}
+  table{width:100%;border-collapse:collapse;margin:14px 0;font-size:13.5px;}
+  th,td{text-align:left;padding:9px 11px;border-bottom:1px solid var(--border);}
+  th{color:var(--muted);font-weight:600;font-size:12px;text-transform:uppercase;letter-spacing:.04em;}
+  td code{font-size:.92em;}
+  .tag{display:inline-block;font-size:11px;font-weight:600;padding:2px 8px;border-radius:20px;
+    border:1px solid;vertical-align:middle;}
+  .t-red{color:var(--red);border-color:#5a2027;background:#2a1418;}
+  .t-amber{color:var(--amber);border-color:#5a4416;background:#241d0c;}
+  .t-green{color:var(--green);border-color:#1d4427;background:#0e2415;}
+  .t-blue{color:var(--blue);border-color:#1d3a66;background:#0d1d36;}
+  .t-purple{color:var(--purple);border-color:#3d2a66;background:#1a1230;}
+  .t-gray{color:var(--muted);border-color:#333b48;background:#1a212c;}
+  .callout{border-left:3px solid var(--blue);background:var(--panel2);border-radius:0 10px 10px 0;
+    padding:12px 16px;margin:16px 0;}
+  .callout.warn{border-color:var(--amber);} .callout.bad{border-color:var(--red);}
+  .callout.good{border-color:var(--green);}
+  .callout b{color:var(--ink);}
+  .lead{font-size:13px;font-weight:700;text-transform:uppercase;letter-spacing:.05em;color:var(--muted);margin-bottom:4px;}
+  ul{margin:8px 0;padding-left:22px;} li{margin:5px 0;color:#cdd6e0;}
+  .kpill{font-family:var(--mono);font-size:12px;color:var(--amber);}
+  .num{font-variant-numeric:tabular-nums;}
+  .step{display:flex;gap:12px;align-items:flex-start;margin:10px 0;}
+  .step .n{flex:none;width:26px;height:26px;border-radius:50%;background:var(--panel2);
+    border:1px solid var(--border);display:grid;place-items:center;font-size:13px;font-weight:700;color:var(--blue);}
+  .mock{background:#10151d;border:1px solid var(--border);border-radius:10px;padding:14px 16px;font-size:13px;}
+  .mock .row{display:flex;align-items:center;gap:10px;padding:7px 0;border-bottom:1px solid #1d2430;}
+  .mock .row:last-child{border-bottom:none;}
+  .mock .nm{font-weight:650;} .mock .desc{color:var(--muted);font-size:12px;}
+  .dot{width:8px;height:8px;border-radius:50%;flex:none;}
+  .legend{display:flex;flex-wrap:wrap;gap:14px;margin:10px 0;font-size:12.5px;color:var(--muted);}
+  .legend span{display:inline-flex;align-items:center;gap:6px;}
+  footer{margin-top:60px;padding-top:20px;border-top:1px solid var(--border);color:var(--faint);font-size:13px;}
+  .toc{columns:2;font-size:13.5px;color:var(--muted);} .toc a{display:block;margin:3px 0;}
+  @media(max-width:720px){.toc{columns:1;}}
+  svg{max-width:100%;height:auto;display:block;margin:8px auto;}
+</style>
+</head>
+<body>
+<div class="wrap">
+  <p class="sub">Wall-E · Background Skills · Reliability Design</p>
+  <h1>Self-Healing Skills</h1>
+  <p class="sub">A half-open circuit breaker so auto-disabled background skills recover on their own
+  — instead of dying silently and staying dead for months.</p>
+  <p class="meta">Design review · 2026-06-24 · status: <span class="tag t-amber">awaiting your sign-off</span>
+  · scope: <code>wall-e/skills</code>, <code>brain.js</code>, <code>walle.js</code></p>
+  <div class="panel">
+    <div class="lead">On this page</div>
+    <div class="toc">
+      <a href="#problem">1 · The problem (live data)</a>
+      <a href="#root">2 · Root cause</a>
+      <a href="#goal">3 · What "self-heal" should mean</a>
+      <a href="#design">4 · Design: half-open breaker</a>
+      <a href="#machine">5 · State machine</a>
+      <a href="#code">6 · Where it plugs in</a>
+      <a href="#diag">7 · Fixing blind failures</a>
+      <a href="#setup">8 · "Needs setup" vs "failure"</a>
+      <a href="#retire">9 · Retiring google-calendar</a>
+      <a href="#ui">10 · UI surfacing</a>
+      <a href="#safety">11 · Safety, flags, tests</a>
+      <a href="#sources">12 · Research basis</a>
+    </div>
+  </div>
+  <!-- ───────────────────────── 1 ───────────────────────── -->
+  <h2 id="problem">1 · The problem, in live numbers</h2>
+  <p>You flagged two skills showing <span class="tag t-red">skipped: auto disabled</span> in the Skills tab.
+  A read-only query of the live brain DB (<code>~/.walle/live-data/wall-e-brain.db</code>) shows it's not
+  two — it's <b>four</b>, all tripped by the same <code>consecutive_failures</code> breaker:</p>
+  <table>
+    <thead><tr><th>Skill</th><th>Auto-disabled</th><th>Dead for</th><th class="num">ok / fail</th><th>Last captured error</th></tr></thead>
+    <tbody>
+      <tr><td><code>google-calendar</code></td><td>2026-06-17</td><td>7 days</td><td class="num">11678 / 156</td><td><code>exited code 1</code> · ~10.15s</td></tr>
+      <tr><td><code>gws-workspace</code></td><td>2026-04-27</td><td><b style="color:var(--red)">58 days</b></td><td class="num">1674 / 10</td><td><span class="meta">history pruned</span></td></tr>
+      <tr><td><code>slack-sync</code></td><td>2026-04-27</td><td>58 days</td><td class="num">0 / 12</td><td><span class="meta">never once succeeded</span></td></tr>
+      <tr><td><code>mcp-scan</code></td><td>2026-04-27</td><td>58 days</td><td class="num">2350 / 12</td><td><span class="meta">history pruned</span></td></tr>
+    </tbody>
+  </table>
+  <div class="callout bad">
+    <b>The headline:</b> three of these have been dead for <b>58 days</b> — across many Wall-E restarts —
+    and never came back on their own. The breaker only ever <b>opens</b>. It has no path to ever <b>close</b>.
+  </div>
+  <div class="grid cols-2">
+    <div class="panel">
+      <div class="lead">Why "99% success" is a lie here</div>
+      <p style="margin:6px 0">The UI's success rate is a <b>lifetime</b> average:
+      <code>success / (success + failure)</code>. <code>google-calendar</code> shows 99% because 11.6k old
+      successes drown out a <b>recent unbroken streak of failures</b>. The breaker fired correctly; the
+      headline metric just can't see it.</p>
+    </div>
+    <div class="panel">
+      <div class="lead">Why a human can't debug it either</div>
+      <p style="margin:6px 0">The only stored error is <code>bundled skill "google-calendar" exited
+      code 1</code>. The skill's real stderr — token expired? <code>gws</code> CLI gone? Calendar
+      permission revoked? — is <b>thrown away</b>. The flat ~10.15s runtime even hints at a silent
+      timeout. You can't fix what you can't see.</p>
+    </div>
+  </div>
+  <!-- ───────────────────────── 2 ───────────────────────── -->
+  <h2 id="root">2 · Root cause — the recovery path is hardcoded to one skill</h2>
+  <p>Wall-E <i>does</i> have an auto-recovery path. It just isn't general. On startup,
+  <code>syncBundledSkills()</code> clears the auto-disabled flag — but only inside an
+  <code>if</code> that names a single skill:</p>
+<pre><span class="c">// agent.js:276  — the recovery gate</span>
+<span class="k">if</span> (skill.name === <span class="s">'scan-ctm-sessions'</span> &amp;&amp; skill.execution === <span class="s">'script'</span> &amp;&amp; existing) {
+  <span class="c">// ...</span>
+  <span class="k">if</span> (existing.auto_disabled_at) {       <span class="c">// agent.js:282-286</span>
+    updates.enabled = <span class="f">1</span>;
+    updates.auto_disabled_at = <span class="k">null</span>;     <span class="c">// ← only scan-ctm-sessions ever reaches this</span>
+  }
+}</pre>
+  <p>So <code>scan-ctm-sessions</code> resurrects every boot; everyone else stays disabled until you
+  manually flip the toggle (<code>PUT /api/wall-e/skills/:id {enabled:1}</code>, which clears
+  <code>auto_disabled_at</code> at <code>api-walle.js:4084-4092</code>). That's the entire reason three
+  skills sat dead for two months. <b>There is no time-based, general recovery anywhere.</b></p>
+  <!-- ───────────────────────── 3 ───────────────────────── -->
+  <h2 id="goal">3 · What "self-heal" should actually mean</h2>
+  <p>The failures here split into two kinds, and a good design must treat them differently:</p>
+  <div class="grid cols-2">
+    <div class="panel"><div class="lead" style="color:var(--green)">Transient — should auto-recover</div>
+      <ul><li>OAuth access token expired (refreshes fine next run)</li>
+      <li>Network blip / Google API 5xx</li><li>Brain DB briefly locked</li>
+      <li>Rate-limit cooldown</li></ul>
+      <p style="font-size:13px;color:var(--muted)">→ A retry minutes-to-hours later just works. No human needed.</p>
+    </div>
+    <div class="panel"><div class="lead" style="color:var(--red)">Permanent — must escalate, not thrash</div>
+      <ul><li>macOS Calendar permission revoked</li><li><code>gws</code> CLI uninstalled / not on PATH</li>
+      <li>Refresh token revoked — re-auth required</li><li><code>swiftc</code> / toolchain missing</li></ul>
+      <p style="font-size:13px;color:var(--muted)">→ Retrying forever is pointless &amp; noisy. Tell the human, with the real error.</p>
+    </div>
+  </div>
+  <div class="callout good"><b>The textbook answer</b> (and your chosen direction): a
+  <b>half-open circuit breaker</b>. Probe cheaply after a cooldown; close on success; back off and
+  finally <b>escalate to a human</b> after a few failed probes. Transient breakage heals with zero
+  clicks; permanent breakage stops thrashing and surfaces a real, actionable error.</div>
+  <!-- ───────────────────────── 4 ───────────────────────── -->
+  <h2 id="design">4 · The design — a half-open breaker, expressed as data</h2>
+  <p>Wall-E's architecture has a hard rule (<code>docs/architecture.md</code> → Non-goals): loops
+  coordinate <b>via the brain DB and scheduler — never a manager-of-managers</b>. This design honors
+  that completely: <b>no new loop, no supervisor.</b> The half-open state is just two columns that the
+  existing pure decision function already reads on every tick.</p>
+  <div class="grid cols-3">
+    <div class="panel"><div class="lead">Add: 2 columns</div>
+      <p style="font-size:13.5px"><code>auto_disabled_attempts INTEGER DEFAULT 0</code><br><br>
+      <code>last_probe_at TEXT</code></p>
+      <p style="font-size:12.5px;color:var(--muted)"><code>auto_disabled_at</code> is preserved untouched
+      for display ("disabled 7d ago"); the new columns drive scheduling.</p>
+    </div>
+    <div class="panel"><div class="lead">Probe schedule</div>
+      <p style="font-size:13.5px"><span class="kpill">[ 1h, 6h, 24h ]</span> exponential, <b>+ jitter</b><br><br>
+      <code>MAX_PROBE_ATTEMPTS = 3</code></p>
+      <p style="font-size:12.5px;color:var(--muted)">Mirrors the <i>existing</i> pre-disable
+      <code>DEFAULT_BACKOFF_SCHEDULE_MS</code> table — same idea, applied on the post-disable side.</p>
+    </div>
+    <div class="panel"><div class="lead">Outcome</div>
+      <p style="font-size:13.5px"><span class="tag t-green">probe ok</span> → re-enable + reset<br><br>
+      <span class="tag t-amber">probe fail</span> → back off<br><br>
+      <span class="tag t-red">3× fail</span> → escalate &amp; stop</p>
+    </div>
+  </div>
+  <!-- ───────────────────────── 5 ───────────────────────── -->
+  <h2 id="machine">5 · State machine</h2>
+  <div class="panel">
+  <svg viewBox="0 0 820 300" role="img" aria-label="Half-open circuit breaker state machine">
+    <defs>
+      <marker id="arr" markerWidth="9" markerHeight="9" refX="7" refY="3" orient="auto">
+        <path d="M0,0 L7,3 L0,6 Z" fill="#5c6675"/>
+      </marker>
+      <marker id="arrG" markerWidth="9" markerHeight="9" refX="7" refY="3" orient="auto">
+        <path d="M0,0 L7,3 L0,6 Z" fill="#3fb950"/>
+      </marker>
+      <marker id="arrR" markerWidth="9" markerHeight="9" refX="7" refY="3" orient="auto">
+        <path d="M0,0 L7,3 L0,6 Z" fill="#f85149"/>
+      </marker>
+    </defs>
+    <!-- CLOSED -->
+    <rect x="40" y="120" width="150" height="58" rx="11" fill="#0e2415" stroke="#3fb950" stroke-width="1.5"/>
+    <text x="115" y="146" text-anchor="middle" fill="#3fb950" font-size="15" font-weight="700">CLOSED</text>
+    <text x="115" y="164" text-anchor="middle" fill="#8b97a7" font-size="11">enabled · running</text>
+    <!-- OPEN -->
+    <rect x="335" y="20" width="150" height="58" rx="11" fill="#2a1418" stroke="#f85149" stroke-width="1.5"/>
+    <text x="410" y="46" text-anchor="middle" fill="#f85149" font-size="15" font-weight="700">OPEN</text>
+    <text x="410" y="64" text-anchor="middle" fill="#8b97a7" font-size="11">auto_disabled · waiting</text>
+    <!-- HALF-OPEN -->
+    <rect x="335" y="120" width="150" height="58" rx="11" fill="#241d0c" stroke="#d29922" stroke-width="1.5"/>
+    <text x="410" y="146" text-anchor="middle" fill="#d29922" font-size="15" font-weight="700">HALF-OPEN</text>
+    <text x="410" y="164" text-anchor="middle" fill="#8b97a7" font-size="11">one silent probe</text>
+    <!-- ESCALATED -->
+    <rect x="630" y="120" width="150" height="58" rx="11" fill="#1a1230" stroke="#a371f7" stroke-width="1.5"/>
+    <text x="705" y="146" text-anchor="middle" fill="#a371f7" font-size="15" font-weight="700">ESCALATED</text>
+    <text x="705" y="164" text-anchor="middle" fill="#8b97a7" font-size="11">human alert · stop</text>
+    <!-- CLOSED -> OPEN -->
+    <path d="M180,128 C250,90 280,60 333,52" fill="none" stroke="#f85149" stroke-width="1.6" marker-end="url(#arrR)"/>
+    <text x="232" y="78" fill="#f85149" font-size="11">10 consec. fails</text>
+    <!-- OPEN -> HALF-OPEN -->
+    <path d="M410,80 L410,118" fill="none" stroke="#5c6675" stroke-width="1.6" marker-end="url(#arr)"/>
+    <text x="420" y="103" fill="#8b97a7" font-size="11">cooldown elapsed</text>
+    <!-- HALF-OPEN -> CLOSED (success) -->
+    <path d="M333,158 C260,196 240,180 190,162" fill="none" stroke="#3fb950" stroke-width="1.6" marker-end="url(#arrG)"/>
+    <text x="208" y="200" fill="#3fb950" font-size="11">probe ok → re-enable</text>
+    <!-- HALF-OPEN -> OPEN (fail, attempts left) -->
+    <path d="M428,118 C455,100 455,80 440,80" fill="none" stroke="#d29922" stroke-width="1.6" marker-end="url(#arr)"/>
+    <text x="470" y="100" fill="#d29922" font-size="11">probe fail → back off</text>
+    <!-- HALF-OPEN -> ESCALATED -->
+    <path d="M485,149 L628,149" fill="none" stroke="#a371f7" stroke-width="1.6" marker-end="url(#arr)"/>
+    <text x="500" y="140" fill="#a371f7" font-size="11">3rd probe fails</text>
+    <!-- ESCALATED -> CLOSED (manual) -->
+    <path d="M630,176 C500,250 250,250 115,180" fill="none" stroke="#5c6675" stroke-width="1.4" stroke-dasharray="5 4" marker-end="url(#arr)"/>
+    <text x="360" y="262" fill="#8b97a7" font-size="11">manual re-enable (toggle) — unchanged escape hatch</text>
+  </svg>
+  </div>
+  <div class="legend">
+    <span><span class="dot" style="background:#3fb950"></span>healthy / recovered</span>
+    <span><span class="dot" style="background:#d29922"></span>recovering (transient)</span>
+    <span><span class="dot" style="background:#f85149"></span>open / failing</span>
+    <span><span class="dot" style="background:#a371f7"></span>needs a human</span>
+  </div>
+  <!-- ───────────────────────── 6 ───────────────────────── -->
+  <h2 id="code">6 · Where it plugs in (two surgical edits)</h2>
+  <p>The decision logic is already a <b>pure function</b> that sees every skill — including disabled
+  ones — on every scheduler tick. Today its auto-disabled branch is a dead end. We make it bloom a probe.</p>
+  <h3>a · <code>skills/skill-dispatch-decision.js</code> — the disabled branch (currently lines 114-121)</h3>
+<pre><span class="c">// BEFORE — skips forever</span>
+<span class="k">if</span> (!skill.enabled) {
+  <span class="k">if</span> (skill.auto_disabled_at) {
+    <span class="k">return</span> { decision:<span class="s">'skip'</span>, reason:<span class="s">'auto_disabled'</span>, details:{ since:skill.auto_disabled_at } };
+  }
+  <span class="k">return</span> { decision:<span class="s">'skip'</span>, reason:<span class="s">'disabled'</span> };
+}
+<span class="c">// AFTER — half-open probe (only for auto_disabled, never user/deprecated-disabled)</span>
+<span class="k">if</span> (!skill.enabled) {
+  <span class="k">if</span> (skill.auto_disabled_at) {
+    <span class="k">const</span> attempts = skill.auto_disabled_attempts || <span class="f">0</span>;
+    <span class="k">if</span> (attempts &gt;= MAX_PROBE_ATTEMPTS)
+      <span class="k">return</span> { decision:<span class="s">'skip'</span>, reason:<span class="s">'auto_disabled_exhausted'</span>, details:{ attempts } };
+    <span class="k">const</span> anchor = <span class="k">new</span> Date(skill.last_probe_at || skill.auto_disabled_at).getTime();
+    <span class="k">const</span> nextProbeAt = anchor + probeScheduleFor(attempts);  <span class="c">// 1h,6h,24h +jitter</span>
+    <span class="k">if</span> (now &gt;= nextProbeAt)
+      <span class="k">return</span> { decision:<span class="s">'run'</span>, reason:<span class="s">'probe'</span>, details:{ attempt: attempts + <span class="f">1</span> } };
+    <span class="k">return</span> { decision:<span class="s">'skip'</span>, reason:<span class="s">'auto_disabled'</span>,
+             details:{ since:skill.auto_disabled_at, next_probe_at:<span class="k">new</span> Date(nextProbeAt).toISOString(), attempts } };
+  }
+  <span class="k">return</span> { decision:<span class="s">'skip'</span>, reason:<span class="s">'disabled'</span> };  <span class="c">// user/deprecated — untouched</span>
+}</pre>
+  <h3>b · <code>skills/skill-planner.js</code> — handle the probe outcome (next to the existing 733-748 block)</h3>
+<pre><span class="c">// after a `reason:'probe'` dispatch finishes, branch on the result:</span>
+<span class="k">if</span> (decision.reason === <span class="s">'probe'</span>) {
+  <span class="k">if</span> (result.ok) {                                  <span class="c">// CLOSE the breaker</span>
+    brain.updateSkill(skill.id, { enabled:<span class="f">1</span>, auto_disabled_at:<span class="k">null</span>,
+      auto_disabled_reason:<span class="k">null</span>, auto_disabled_attempts:<span class="f">0</span>, last_probe_at:<span class="k">null</span> });
+    clearServiceAlerts(skill.name);
+    recoveryLog.info(<span class="s">'skill-self-heal'</span>, <span class="s">'recovered'</span>, { skill: skill.name });
+  } <span class="k">else</span> {                                          <span class="c">// stay OPEN, back off</span>
+    <span class="k">const</span> attempts = (skill.auto_disabled_attempts || <span class="f">0</span>) + <span class="f">1</span>;
+    brain.updateSkill(skill.id, { auto_disabled_attempts: attempts, last_probe_at: nowIso });
+    <span class="k">if</span> (attempts &gt;= MAX_PROBE_ATTEMPTS)
+      addServiceAlert({ service:skill.name, type:<span class="s">'skill_needs_attention'</span>,
+        message:<span class="s">`"${skill.name}" failed ${attempts} recovery attempts: ${result.errorTail}`</span>,
+        action_url:<span class="s">'/setup.html'</span> });
+  }
+  <span class="k">continue</span>;
+}</pre>
+  <div class="callout"><b>That's the whole engine.</b> Everything else — the scheduler tick, the
+  per-skill decision recording (<code>recordSkillLastDecision</code>), the existing Service-Alert
+  plumbing — is reused as-is.</div>
+  <!-- ───────────────────────── 7 ───────────────────────── -->
+  <h2 id="diag">7 · Fixing blind failures (so probes are meaningful)</h2>
+  <p>A probe is only useful if a <i>failed</i> probe tells us <i>why</i>. Today the script runner
+  collapses everything to <code>exited code 1</code>. We capture the tail of the skill's stderr/stdout
+  into <code>skill_executions.error</code> and <code>skills.last_result</code>, so:</p>
+  <div class="grid cols-2">
+    <div class="panel"><div class="lead t-red" style="color:var(--red)">Today</div>
+      <pre style="margin:6px 0"><span class="c">error:</span> bundled skill "google-calendar" exited code 1</pre></div>
+    <div class="panel"><div class="lead" style="color:var(--green)">After</div>
+      <pre style="margin:6px 0"><span class="c">error:</span> Calendar access denied (status=2).
+Grant access in System Settings ›
+Privacy &amp; Security › Calendars.</pre></div>
+  </div>
+  <p style="font-size:13.5px;color:var(--muted)">This also lets the escalation alert quote the real cause,
+  and lets us confirm the suspicious ~10.15s flat runtime (a hidden timeout vs the documented 30s binary timeout).</p>
+  <!-- ───────────────────────── 8 ───────────────────────── -->
+  <h2 id="setup">8 · "Needs setup" is not "failure" (secondary)</h2>
+  <p>Some skills aren't <i>broken</i> — they're <i>unconfigured</i>. <code>gws-workspace</code> returns
+  "no accounts configured"; that currently exits non-zero and burns the failure budget until the breaker
+  trips. Wall-E already has a <code>requires</code> preflight (<code>skill-planner.js:757-779</code>) that
+  raises actionable alerts. We extend it: an unconfigured/missing-dependency outcome reports
+  <span class="tag t-blue">needs_setup</span> — it does <b>not</b> count as a failure and does <b>not</b>
+  auto-disable; it surfaces a one-click setup prompt instead of dying silently.</p>
+  <!-- ───────────────────────── 9 ───────────────────────── -->
+  <h2 id="retire">9 · Retiring <code>google-calendar</code></h2>
+  <p>Per your call — <i>"just use gws-workspace, I don't think you need google-calendar anymore."</i>
+  <code>gws-workspace</code>'s own SKILL.md says it <b>"replaces separate google-calendar, email-sync, and
+  gdrive skills"</b> with no macOS-only dependencies. Running both also double-writes <code>calendar</code>
+  memories under the same dedup key.</p>
+  <div class="step"><div class="n">1</div><div>One-time migration (#28): set <code>enabled=0</code>,
+    <code>auto_disabled_reason='deprecated'</code>, and crucially <code>auto_disabled_at=NULL</code>.</div></div>
+  <div class="step"><div class="n">2</div><div>Because the prober only probes rows with a non-null
+    <code>auto_disabled_at</code>, the deprecated skill is <b>deliberately never resurrected</b> — it reads as a clean "disabled", not a recovering one.</div></div>
+  <div class="step"><div class="n">3</div><div><b>Keep the bundled dir.</b> The implementation deliberately does
+    <i>not</i> delete <code>skills/_bundled/google-calendar/</code>: <code>listening/calendar.js</code> reuses
+    its compiled <code>cal-reader</code> binary, so retirement is a DB-row deprecation, not a file removal.</div></div>
+  <div class="callout warn"><b>Note:</b> <code>gws-workspace</code> itself is currently auto-disabled.
+  Once self-heal ships, it gets probed automatically. If its OAuth / <code>gws</code> CLI is genuinely
+  broken, it will escalate after 3 probes with the <i>real</i> error (per §7) so you can fix the setup once — then it heals itself.</div>
+  <!-- ───────────────────────── 10 ───────────────────────── -->
+  <h2 id="ui">10 · What you'll see in the UI</h2>
+  <p>No new screens — just richer status on the existing skill cards and health panel.</p>
+  <div class="mock">
+    <div class="row"><span class="dot" style="background:#d29922"></span>
+      <span class="nm">gws-workspace</span><span class="tag t-amber">recovering</span>
+      <span class="desc">probe 2 / 3 · next attempt in ~6h</span></div>
+    <div class="row"><span class="dot" style="background:#3fb950"></span>
+      <span class="nm">slack-sync</span><span class="tag t-green">self-healed</span>
+      <span class="desc">recovered automatically 3m ago</span></div>
+    <div class="row"><span class="dot" style="background:#a371f7"></span>
+      <span class="nm">mcp-scan</span><span class="tag t-purple">needs attention</span>
+      <span class="desc">3 recovery attempts failed · "MCP endpoint refused connection"</span></div>
+    <div class="row"><span class="dot" style="background:#5c6675"></span>
+      <span class="nm">google-calendar</span><span class="tag t-gray">deprecated</span>
+      <span class="desc">superseded by gws-workspace</span></div>
+  </div>
+  <ul>
+    <li><b>Card chip</b> (<code>renderSkillCard</code>, walle.js:5582) gains <code>recovering</code> /
+      <code>needs attention</code> / <code>deprecated</code> states.</li>
+    <li><b>Recovery toast</b> when a skill self-heals — positive signal, not just silence.</li>
+    <li><b>Health panel</b> (<code>renderServiceHealthPanel</code>, walle.js:6946) splits "recovering"
+      (transient, no action) from "needs attention" (escalated, click to fix).</li>
+  </ul>
+  <!-- ───────────────────────── 11 ───────────────────────── -->
+  <h2 id="safety">11 · Safety, flags &amp; tests</h2>
+  <div class="grid cols-2">
+    <div class="panel"><div class="lead">Why this can't make things worse</div>
+      <ul>
+        <li>Probes are <b>rate-limited</b> (1h→6h→24h) and <b>capped</b> (3) — no thrash.</li>
+        <li>A probe is one normal run; it touches nothing a manual re-enable wouldn't.</li>
+        <li>Permanent breakage <b>stops &amp; escalates</b> instead of looping.</li>
+        <li>User-disabled &amp; <code>deprecated</code> skills are never probed (no <code>auto_disabled_at</code>).</li>
+        <li>It only ever <i>adds</i> recovery; worst case it does what you'd do by hand, slower.</li>
+      </ul>
+    </div>
+    <div class="panel"><div class="lead">Flag &amp; rollback</div>
+      <p style="font-size:13.5px"><code>WALLE_SKILL_SELF_HEAL</code> — default <b>ON</b>.<br>
+      Probe schedule &amp; <code>MAX_PROBE_ATTEMPTS</code> configurable.<br>
+      <code>=0</code> restores today's open-forever behavior exactly.</p>
+      <div class="lead" style="margin-top:14px">Tests</div>
+      <ul style="font-size:13.5px">
+        <li>decision fn: cooldown-elapsed → <code>probe</code>; pending → <code>auto_disabled</code> w/ <code>next_probe_at</code>; capped → <code>exhausted</code>.</li>
+        <li>planner: probe ok re-enables &amp; clears; probe fail increments &amp; escalates at 3.</li>
+        <li>migration: <code>google-calendar</code> → deprecated, never probed.</li>
+      </ul>
+    </div>
+  </div>
+  <div class="callout"><b>Verification</b> — on a <code>/ctm-dev</code> instance only (never primary
+  3456/3457): seed an <code>auto_disabled</code> skill, advance the clock, confirm probe → recover, and
+  exhaustion → escalate. Then <code>cd wall-e &amp;&amp; npm test</code>. Server-side changes deploy on the
+  next primary restart, which you trigger.</div>
+  <!-- ───────────────────────── 12 ───────────────────────── -->
+  <h2 id="sources">12 · Research basis</h2>
+  <p style="font-size:13.5px;color:var(--muted)">The pattern is standard reliability engineering, adapted to Wall-E's data-driven, no-supervisor constraint: half-open probing, exponential backoff <i>with jitter</i>, a cheap health probe, and escalation-to-human after N attempts.</p>
+  <ul style="font-size:13.5px">
+    <li><a href="https://blog.meganova.ai/circuit-breakers-in-ai-agent-systems-reliability-at-scale/">Circuit Breakers in AI Agent Systems — Reliability at Scale</a> (half-open auto-recovery, cheap probe)</li>
+    <li><a href="https://planetscale.com/blog/how-we-made-planetscale-background-jobs-self-healing-with-sidekiq">PlanetScale — making background jobs self-healing</a> (data-driven re-queue, no supervisor)</li>
+    <li><a href="https://medium.com/@atulbanwar/smart-jenkins-management-auto-disabling-jobs-on-consecutive-failures-6dd80b15553b">Jenkins — auto-disabling jobs on consecutive failures</a> (the open side, and why it needs a close side)</li>
+    <li><a href="https://learn.microsoft.com/en-us/azure/well-architected/reliability/self-preservation">Azure Well-Architected — self-healing &amp; self-preservation</a> (escalate, don't thrash)</li>
+  </ul>
+  <footer>
+    Wall-E Skill Self-Heal · design review · 2026-06-24 · awaiting sign-off before any code is written.<br>
+    Next step on approval: implement §6 behind <code>WALLE_SKILL_SELF_HEAL</code>, validate on <code>/ctm-dev</code>, then <code>/tl-commit</code>.
+  </footer>
+</div>
+</body>
+</html>

package/template/wall-e/llm/client.js CHANGED Viewed

@@ -6,6 +6,15 @@ const { isPortkeyProviderConfig } = require('./portkey');
 const { withProviderMessageGuard } = require('./message-guard');
 const LOCAL_PROVIDER_TYPES = new Set(['ollama', 'mlx', 'lmstudio']);
+const DEFAULT_PROVIDER_HEALTH_FAILURE_TYPES = new Set([
+  'network',
+  'timeout',
+  'provider_unavailable',
+  'rate_limited',
+  'auth_error',
+  'quota_exceeded',
+  'model_unavailable',
+]);
 function normalizeProviderType(type) {
   const text = String(type || '').trim().toLowerCase().replace(/_/g, '-');
@@ -201,7 +210,7 @@ function _resolvePrimaryDefault() {
   // Anthropic auth_method=claude_cli routes through the user's `claude`
   // CLI (subscription billing) instead of the SDK against api.anthropic.com.
   if (providerType === 'anthropic' && authMethod === 'claude_cli') {
-    return { id: 'claude-cli', type: 'claude-cli', client: createClient('claude-cli', {}) };
+    return { id: 'claude-cli', type: 'claude-cli', client: createClient('claude-cli', {}), model: getDefaultModel() };
   }
   // Anthropic auth_method=oauth_proxy points the SDK at the local proxy
@@ -213,13 +222,14 @@ function _resolvePrimaryDefault() {
       id: `${providerType}-oauth`,
       type: providerType,
       client: createClient('anthropic', { apiKey: 'oauth-proxy-placeholder', baseUrl: `http://127.0.0.1:${port}` }),
+      model: getDefaultModel(),
     };
   }
   // OpenAI auth_method=codex_cli routes through the user's `codex` CLI
   // (ChatGPT subscription billing) instead of api.openai.com.
   if (providerType === 'openai' && authMethod === 'codex_cli') {
-    return { id: 'codex-cli', type: 'codex-cli', client: createClient('codex-cli', {}) };
+    return { id: 'codex-cli', type: 'codex-cli', client: createClient('codex-cli', {}), model: resolveCompatibleModel(getDefaultModel(), 'codex-cli') };
   }
   let client;
@@ -266,7 +276,7 @@ function _resolvePrimaryDefault() {
     const preferred = brain.getPreferredModelProviderForType?.(providerType);
     if (preferred?.id) id = preferred.id;
   } catch {}
-  return { id, type: providerType, client };
+  return { id, type: providerType, client, model: getDefaultModel() };
 }
 // Surface a fallback to the user/operator: the live default was unreachable and a
@@ -295,6 +305,17 @@ function _notifyDefaultFallback(from, to, error) {
     } catch {}
   }
   try { console.warn(`[llm] ${message}${detail ? ' — ' + detail : ''}`); } catch {}
+  if (error) {
+    try {
+      const providerErrorMod = require('./provider-error');
+      const providerError = providerErrorMod.classifyProviderError(error, {
+        provider: from?.type || from?.id || null,
+        model: from?.model || null,
+        providerId: from?.id || null,
+      });
+      providerErrorMod.recordProviderFailureAlert(providerError, require('../brain'), { recordHealth: false });
+    } catch {}
+  }
   try {
     const health = require('./health').default;
     health.emit('default-provider-fallback', { from: { id: from?.id, type: from?.type }, to: { id: to?.id, type: to?.type }, detail });
@@ -307,8 +328,20 @@ function _notifyDefaultExhausted(error) {
   // Secret-redacted detail (replaces the old raw error.message, which could echo creds).
   let detail = '';
   try { detail = require('./provider-error').fallbackErrorDetail(error); } catch {}
-  const message = `Every configured provider is unreachable. Check connectivity or Setup.${detail ? ' (' + detail + ')' : ''}`;
+  const localUnavailable = error?.suppressProviderHealthRecord === true
+    || error?.providerError?.type === 'provider_unavailable_local'
+    || error?.code === 'AI_PROVIDER_UNAVAILABLE_LOCAL';
+  const message = localUnavailable
+    ? `Every configured provider/model route is currently skipped by health policy. Wall-E will wait for recovery probes or Setup changes.${detail ? ' (' + detail + ')' : ''}`
+    : `Every configured provider is unreachable. Check connectivity or Setup.${detail ? ' (' + detail + ')' : ''}`;
   try { console.error(`[llm] ${message}`); } catch {}
+  if (error && !localUnavailable) {
+    try {
+      const providerErrorMod = require('./provider-error');
+      const providerError = providerErrorMod.classifyProviderError(error);
+      providerErrorMod.recordProviderFailureAlert(providerError, require('../brain'));
+    } catch {}
+  }
   try {
     const health = require('./health').default;
     health.emit('default-provider-unreachable', { error: detail || String(error?.message || error || '') });
@@ -320,6 +353,68 @@ function _clearDefaultRoutingNotice() {
   try { require('./health').default.clearRoutingNotice(); } catch {}
 }
+function _recordDefaultProviderHealthFailure({ provider, model, error, type, willRetry }) {
+  if (willRetry) return;
+  const providerType = provider?.type || provider?.id || '';
+  if (!providerType) return;
+  let providerError = null;
+  try {
+    const providerErrorMod = require('./provider-error');
+    providerError = providerErrorMod.classifyProviderError(error, {
+      provider: providerType,
+      model: model || provider?.model || '',
+      providerId: provider?.id || null,
+    });
+  } catch {}
+  const errorType = type || providerError?.type || 'provider_error';
+  if (!DEFAULT_PROVIDER_HEALTH_FAILURE_TYPES.has(errorType)) return;
+  try {
+    require('./provider-health-state').recordProviderHealthStatus({
+      provider: providerType,
+      provider_id: provider?.id || null,
+      runtime_type: providerType,
+      provider_type: providerType,
+      model: model || provider?.model || '',
+      ok: false,
+      source: 'default-client',
+      error_type: errorType,
+      error: providerError?.rawMessage || error?.message || String(error || ''),
+      checked_at: providerError?.createdAt || new Date().toISOString(),
+    }, { brain: require('../brain'), clearTransientAlerts: false });
+  } catch {}
+}
+function _recordDefaultProviderHealthOk(provider, model) {
+  try {
+    require('./provider-health-state').recordProviderHealthStatus({
+      provider: provider?.type || provider?.id || 'default',
+      provider_id: provider?.id || null,
+      runtime_type: provider?.type || '',
+      provider_type: provider?.type || '',
+      model: model || provider?.model || '',
+      ok: true,
+      source: 'default-client',
+    }, { brain: require('../brain') });
+  } catch {}
+}
+function _defaultProviderModelCircuitBlocks(id, cand, callOpts, brain) {
+  try {
+    const model = callOpts?.model || cand?.model || '';
+    const providers = [cand?.type, id].filter(Boolean);
+    const healthState = require('./provider-health-state');
+    const circuit = healthState.getFirstProviderModelCircuit(providers, model, { brain });
+    if (!circuit) return false;
+    if (circuit.probe_due) {
+      healthState.reserveProviderModelCircuitProbe(circuit.provider, circuit.model, { brain });
+      return false;
+    }
+    return true;
+  } catch {
+    return false;
+  }
+}
 function getDefaultClient() {
   if (_defaultClient) return _defaultClient;
@@ -351,6 +446,8 @@ function getDefaultClient() {
     if (routingSettings.mode === 'strict' || routingSettings.fallback_enabled === false) {
       _defaultClient = makeResilientClient([primary], {
         health,
+        isDown: (id, cand, callOpts) => _defaultProviderModelCircuitBlocks(id, cand, callOpts, brain),
+        onCandidateFailure: _recordDefaultProviderHealthFailure,
         onExhausted: ({ error }) => _notifyDefaultExhausted(error),
       });
       return _defaultClient;
@@ -371,15 +468,24 @@ function getDefaultClient() {
       // Nothing to fall back to — still add transient retries around the primary.
       _defaultClient = makeResilientClient([primary], {
         health,
+        isDown: (id, cand, callOpts) => _defaultProviderModelCircuitBlocks(id, cand, callOpts, brain),
+        onCandidateFailure: _recordDefaultProviderHealthFailure,
         onExhausted: ({ error }) => _notifyDefaultExhausted(error),
       });
       return _defaultClient;
     }
     _defaultClient = makeResilientClient(chain, {
       health,
-      isDown: (id) => {
-        try { return health?.getProviderHealth?.(id)?.status === 'down'; } catch { return false; }
+      isDown: (id, cand, callOpts) => {
+        try {
+          if (health?.getProviderHealth?.(id)?.status === 'down') return true;
+        } catch {}
+        return _defaultProviderModelCircuitBlocks(id, cand, callOpts, brain);
+      },
+      onCandidateOk: ({ provider, model }) => {
+        _recordDefaultProviderHealthOk(provider, model);
       },
+      onCandidateFailure: _recordDefaultProviderHealthFailure,
       onFallback: ({ from, to, error }) => _notifyDefaultFallback(from, to, error),
       onPrimaryOk: () => _clearDefaultRoutingNotice(),
       onExhausted: ({ error }) => _notifyDefaultExhausted(error),
@@ -513,4 +619,7 @@ module.exports = {
     providerRegistry.ensureBootstrapped();
     return providerRegistry.modelMatchesProvider(model, provider);
   },
+  _internals: {
+    _resolvePrimaryDefault,
+  },
 };