kaizen-agent 1.34.0 → 1.34.2

package/cli/sources.test.ts

@@ -7,7 +7,7 @@ import { tmpdir } from "node:os";
 import { readInbox, writeInbox, replaceIdea, parseItem } from "./state.ts";
 import {
   readSources, addSource, removeSource, ingest, checkUrl, resolvesPublic, exportUrl, fetchText, split, sift,
-  gatherSource, gatherAll, due, takeImages, sheetImages, docImages, imagesUrl, SIGN_IN, UNREADABLE, CAP, moveSource, gatherDirs, writeSources,
+  gatherSource, freshImages, gatherAll, due, takeImages, sheetImages, docImages, imagesUrl, SIGN_IN, UNREADABLE, CAP, moveSource, gatherDirs, writeSources,
   readIcebox, writeIcebox, promote, dismiss, markReplied, setCategory, socialUrl, socialOf, AGENT_ONLY, PLATFORMS,
 } from "./sources.ts";
 
@@ -764,3 +764,28 @@ test("sheet images placed over the grid land on their anchored row", () => {
     + `<script>posObj('0', 'embed_11', 1, 7, 14, 9);posObj('0', 'embed_22', 1, 7, 7, 316);</script>`;
   expect(sheetImages(html)).toEqual(new Map([[2, ["https://docs.google.com/sheets-images-rt/A=s2048", "https://docs.google.com/sheets-images-rt/B=s2048"]]]));
 });
+
+test("an item's images are read again from its source, since Google's links expire", async () => {
+  addSource(state, SHEET);
+  ingest(state, SHEET, [{ text: "Fix login", images: ["https://docs.google.com/sheets-images-rt/OLD=s2048"] }]);
+  const id = readIcebox(state)[0]!.id;
+  const html = `<table><tr><th id="0R0"><div>1</div></th><td>Request</td></tr><tr><th id="0R1"><div>2</div></th><td>Fix login</td><td><img src="https://docs.google.com/sheets-images-rt/NEW=s2048"></td></tr></table>`;
+  const page: any = async (u: string) => u.includes("htmlview")
+    ? new Response(html, { headers: { "content-type": "text/html" } })
+    : new Response("Request\nFix login\n", { headers: { "content-type": "text/plain" } });
+  expect(await freshImages(state, id, page, pub)).toEqual(["https://docs.google.com/sheets-images-rt/NEW=s2048"]);
+  // The source cannot be read: the kept links come back.
+  expect(await freshImages(state, id, (async () => new Response("", { status: 500 })) as any, pub)).toEqual(["https://docs.google.com/sheets-images-rt/OLD=s2048"]);
+  expect(await freshImages(state, "nope", page, pub)).toEqual([]);
+});
+
+test("a fresh image link on a host the item never used is not taken", async () => {
+  addSource(state, SHEET);
+  ingest(state, SHEET, [{ text: "Fix login", images: ["https://docs.google.com/sheets-images-rt/OLD=s2048"] }]);
+  const id = readIcebox(state)[0]!.id;
+  const html = `<table><tr><th id="0R1"><div>2</div></th><td>Fix login</td><td><img src="https://evil.example/x.png"></td></tr></table>`;
+  const page: any = async (u: string) => u.includes("htmlview")
+    ? new Response(html, { headers: { "content-type": "text/html" } })
+    : new Response("Request\nFix login\n", { headers: { "content-type": "text/plain" } });
+  expect(await freshImages(state, id, page, pub)).toEqual(["https://docs.google.com/sheets-images-rt/OLD=s2048"]);
+});

package/cli/sources.ts

@@ -182,6 +182,15 @@ export function takeImages(notes = ""): { notes: string; images: string[] } {
 
 // The single place outside text enters kaizen, and it enters the icebox, never Ideas.
 // Synchronous on purpose: nothing may run between reading the files and writing them.
+// A gathered item as the icebox keeps it, keyed the way dedup sees it.
+function shape(it: Item) {
+  const got = takeImages(it.notes);
+  const c = clean(it.text, got.notes);
+  if (!c) return null;
+  const notes = c.lines.join("\n").slice(0, 2000);
+  return { c, notes, images: pics([...(it.images ?? []), ...got.images]), key: rowKey(c.text, notes) };
+}
+
 export function ingest(state: string, url: string, items: Item[]): { added: number; held: number } {
   const sources = readSources(state);
   const src = sources.find((s) => s.url === url);
@@ -192,12 +201,9 @@ export function ingest(state: string, url: string, items: Item[]): { added: numb
   const seen: string[] = [];
   let held = 0, refreshed = false;
   for (const it of items) {
-    const got = takeImages(it.notes);
-    const images = pics([...(it.images ?? []), ...got.images]);
-    const c = clean(it.text, got.notes);
-    if (!c) continue;
-    const notes = c.lines.join("\n").slice(0, 2000);
-    const key = rowKey(c.text, notes);
+    const s = shape(it);
+    if (!s) continue;
+    const { c, notes, images, key } = s;
     if (known.has(key)) {
       seen.push(key);
       // Image links expire (social CDNs) or arrive late (items gathered before images
@@ -518,6 +524,39 @@ export function record(state: string, url: string, r: { added: number; held: num
   return { url, ...last };
 }
 
+// An item's images read again from its source now. Google signs sheet and doc image
+// links with a token that expires, so a link kept since the gather answers 403; this
+// reads the page again and returns the same row's links. Falls back to the kept ones.
+// A source's page is read once a minute however many items and images ask, and a
+// fresh link is only taken on a host the kept links already used.
+const pages = new Map<string, { at: number; p: Promise<Map<string, string[]> | null> }>();
+async function readPage(source: string, doFetch?: Fetch, lookup?: Lookup) {
+  const target = exportUrl(source), at = imagesUrl(source)!;
+  const [got, page] = await Promise.all([fetchText(target.url, doFetch, lookup), fetchText(at, doFetch, lookup, "html")]);
+  if (!got.ok || !page.ok) return null;
+  const found = target.kind === "sheet" ? sheetImages(page.body) : docImages(page.body);
+  const byKey = new Map<string, string[]>();
+  for (const x of sift(target.kind, got.body, found as Map<number | string, string[]>).items) {
+    const s = shape(x);
+    if (s?.images.length) byKey.set(s.key, s.images);
+  }
+  return byKey;
+}
+export async function freshImages(state: string, id: string, doFetch?: Fetch, lookup?: Lookup): Promise<string[]> {
+  const it = readIcebox(state).find((s) => s.id === id);
+  if (!it) return [];
+  const kept = it.images ?? [];
+  if (!imagesUrl(it.source)) return kept;
+  const now = Date.now();
+  for (const [k, v] of pages) if (now - v.at >= 60_000) pages.delete(k);
+  const hit = doFetch ? undefined : pages.get(it.source);   // a test's own fetch is never cached
+  const p = hit?.p ?? readPage(it.source, doFetch, lookup).catch(() => null);
+  if (!hit && !doFetch) pages.set(it.source, { at: now, p });
+  const hosts = new Set(kept.map((u) => new URL(u).hostname));
+  const fresh = ((await p)?.get(rowKey(it.text, it.notes)) ?? []).filter((u) => hosts.has(new URL(u).hostname));
+  return fresh.length ? fresh : kept;
+}
+
 // Never throws. A failure is recorded on the source and marks nothing seen.
 export async function gatherSource(state: string, url: string, doFetch?: Fetch, lookup?: Lookup): Promise<Result> {
   if (socialOf(url)) {

package/cli/web.ts

@@ -10,7 +10,7 @@ import {
   launchMode, readSessions, ownerOf, resumePrompt, runAgent, firstSession,
 } from "./state.ts";
 import { attachPty, writePty, resizePty, ptyState, ptyKeys, adoptLoginEnv } from "./pty.ts";
-import { readSources, addSource, removeSource, moveSource, setCategory, readIcebox, promote, dismiss, restore, markReplied, gatherSource, gatherAll, gatherDirs, due, socialUrl, socialOf } from "./sources.ts";
+import { readSources, addSource, removeSource, moveSource, setCategory, readIcebox, promote, dismiss, restore, markReplied, gatherSource, freshImages, gatherAll, gatherDirs, due, socialUrl, socialOf } from "./sources.ts";
 import * as chatgpt from "./chatgpt.ts";
 import { postLink, sendReply } from "./browser.ts";
 import { listNotes, saveNote, renameNote, deleteNote, boardIndex, linksTo } from "./notes.ts";
@@ -91,10 +91,10 @@ export async function web(repoDir: string, opts: Opts = {}) {
   // Before anything looks for an agent on PATH: the board starts agents itself, and
   // from a launcher it would otherwise have none of the user's shell setup.
   adoptLoginEnv();
-  // Not fixed for the life of the board: picking a project in the page moves here.
-  let state = locate();
-  if (state && dirname(state) !== home) remember(dirname(state));
-  let project = state ? label(state) : null;
+  // Where the board started. Each page picks its own project and names it in the
+  // x-kz-dir header, so a phone choosing another project never moves this tab.
+  const boardState = locate();
+  if (boardState && dirname(boardState) !== home) remember(dirname(boardState));
   // The "Projects folder" setting, resolved; empty when unset.
   const projectsRoot = async () => {
     const { listSettings } = await import("./settings.ts");
@@ -125,9 +125,9 @@ export async function web(repoDir: string, opts: Opts = {}) {
       .filter((d) => existsSync(join(d, ".kaizen")) ? (remember(d), false) : true).sort();
   };
 
-  const states = (all: boolean) => (all || !state
+  const states = (all: boolean, cur = boardState) => (all || !cur
     ? [...knownProjects().map((d) => join(d, ".kaizen")), join(home, ".kaizen")]
-    : [state]
+    : [cur]
   ).filter((d, i, arr) => arr.indexOf(d) === i && existsSync(d));
 
   // Hostnames the user put in KAIZEN_WEB_HOSTS (e.g. a `tailscale serve` name):
@@ -290,6 +290,9 @@ export async function web(repoDir: string, opts: Opts = {}) {
       if (hostname !== "127.0.0.1" && hostname !== "localhost" && !extraHosts.has(hostname)) return new Response("forbidden", { status: 403 });
       const url = new URL(req.url);
       const path = url.pathname;
+      const asked = req.headers.get("x-kz-dir");
+      const state = asked && states(true).includes(asked) ? asked : boardState;
+      const project = state ? label(state) : null;
 
       if (req.method === "GET" && path === "/") {
         const page = served(PAGE);
@@ -373,7 +376,7 @@ export async function web(repoDir: string, opts: Opts = {}) {
         const all = url.searchParams.get("all") === "1";
         // Before the project list: it registers folders that turn out to be set up.
         const unset = await fresh();
-        const dirs = states(all);
+        const dirs = states(all, state);
         // With a projects folder set, the picker offers only real projects inside it,
         // and always the one open now so it never hides where the board is.
         const root = await projectsRoot();
@@ -572,6 +575,16 @@ export async function web(repoDir: string, opts: Opts = {}) {
         return new Response(Bun.file(f), { headers: { "cache-control": "private, max-age=3600" } });
       }
 
+      // An icebox image, sent on to a link read fresh from its source: the one kept
+      // since the gather may have expired. Only an https link the item holds is followed.
+      if (req.method === "GET" && path === "/ice-image") {
+        const dir = url.searchParams.get("dir") ?? "";
+        if (!states(true).includes(dir)) return bad("unknown state dir", 404);
+        const u = (await freshImages(dir, url.searchParams.get("id") ?? ""))[Number(url.searchParams.get("i"))];
+        if (!u || !/^https:\/\//.test(u)) return bad("no such image", 404);
+        return new Response(null, { status: 302, headers: { location: u, "cache-control": "no-store", "referrer-policy": "no-referrer" } });
+      }
+
       if (req.method === "GET" && path === "/events") {
         let ctrl: ReadableStreamDefaultController;
         const stream = new ReadableStream({
@@ -612,14 +625,10 @@ export async function web(repoDir: string, opts: Opts = {}) {
         }
         if (!dir || !states(true).includes(dir)) return bad("unknown state dir", 404);
 
-        // The board moves to the chosen project: new ideas and runs default to it,
-        // and the process follows so anything resolved from cwd agrees.
+        // Only checks the project: the page that picked it sends it back in x-kz-dir.
         if (path === "/cd") {
-          state = dir;
-          project = label(dir);
-          try { process.chdir(dirname(dir)); } catch { /* the state dir is what matters */ }
           changed();
-          return json({ ok: true, project });
+          return json({ ok: true, project: label(dir) });
         }
 
         if (path === "/inbox") {
@@ -941,7 +950,7 @@ export async function web(repoDir: string, opts: Opts = {}) {
 
   const url = `http://127.0.0.1:${server.port}/`;
   console.log(`\n  kaizen web  ${url}`);
-  console.log(`  ${project ? `project  ${project}` : "no .kaizen here; showing every known project"}`);
+  console.log(`  ${boardState ? `project  ${label(boardState)}` : "no .kaizen here; showing every known project"}`);
   console.log(`  ctrl-c to stop\n`);
   if (opts.open !== false) await openApp(url);
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kaizen-agent",
-  "version": "1.34.0",
+  "version": "1.34.2",
   "description": "Staged workflow for AI agents — code, writing, research, operations: plan, human approval, build, independent review, bounded fix loop.",
   "type": "module",
   "bin": {

package/web/board/api.ts

@@ -7,11 +7,14 @@ import { notifyUser, toast } from "./toast.ts";
 // The server writes notifier() from state.ts over a marker in board.html: one rule for both UIs.
 declare const due: (key: string, say: string | null, done: boolean, now: number) => boolean;
 
+// This page's project, sent with each request: the server keeps none, so another
+// tab or a phone picking a project never moves this one.
+export const dirHeader = () => (store.dir ? { "x-kz-dir": store.dir } : {});
 // quiet: the server said something changed, nobody asked for a redraw.
 export async function load(quiet) {
   let r;
   try {
-    r = await fetch("/state" + (store.all ? "?all=1" : ""));
+    r = await fetch("/state" + (store.all ? "?all=1" : ""), { headers: dirHeader() });
   } catch {
     toast(GONE);
     return;
@@ -85,7 +88,7 @@ export async function post(path, body) {
   try {
     r = await fetch(path, {
       method: "POST",
-      headers: { "content-type": "application/json" },
+      headers: { "content-type": "application/json", ...dirHeader() },
       body: JSON.stringify(body),
     });
   } catch {

package/web/board/icebox.ts

@@ -279,7 +279,7 @@ export async function openSources() {
     return (
       `<div class="ice-row${iceCur === k ? " cur" : ""}${iceSel.has(k) ? " sel" : ""}" data-i="${i}" data-k="${esc(k)}"><input type="checkbox" id="ice-ck-${i}" data-ck="${i}"${iceSel.has(k) ? " checked" : ""} aria-label="Select: ${esc(it.text)}"><span class="ice-src" title="${esc(srcName(it))}">${fav(it.source, "link", list.find((x) => x.dir === it.dir && x.url === it.source)?.social)}</span><button class="ice-t" id="ice-o-${i}" data-o="${i}" aria-expanded="${open}" title="${esc(it.text)}">${esc(it.text)}</button><span class="ice-meta">${many ? `<span class="nm">${esc(it.project || "")}</span>` : ""}${it.replied ? `<span title="Replied ${esc(ago(it.replied))}">replied</span>` : ""}<span class="ice-age">${ago(it.at)}</span><span class="ice-ra"><button class="btn ghost sm" id="ice-pr-${i}" data-pr="${i}" title="Promote to Ideas as gathered (p)" aria-label="Promote to Ideas: ${esc(it.text)}">${ico("plus")}</button>${it.status === "dismissed" ? `<button class="btn ghost sm" id="ice-rs-${i}" data-rs="${i}" title="Restore to waiting" aria-label="Restore: ${esc(it.text)}">Restore</button>` : `<button class="btn ghost sm" id="ice-ds-${i}" data-ds="${i}" title="Dismiss (d)" aria-label="Dismiss: ${esc(it.text)}">${ico("x")}</button>`}</span></span></div>` +
       (open
-        ? `<div class="ice-open">${it.notes ? `<div class="ice-notes">${esc(it.notes)}</div>` : ""}${(it.images || []).filter((u) => /^https:\/\//.test(u)).length ? `<details class="ice-imgs"><summary>Show images (${it.images.filter((u) => /^https:\/\//.test(u)).length}) — loads them from the source's site</summary>${it.images.filter((u) => /^https:\/\//.test(u)).map((u) => `<a href="${esc(u)}" target="_blank" rel="noopener noreferrer"><img data-src="${esc(u)}" alt="" referrerpolicy="no-referrer"></a>`).join("")}</details>` : ""}<span class="hint" style="margin:0">From ${esc(srcName(it))} · ${ago(it.at)}${it.post ? ` · <a href="${esc(it.post)}" target="_blank" rel="noopener">open post</a>` : ""}</span>${replyBox(it)}${form(it)}</div>`
+        ? `<div class="ice-open">${it.notes ? `<div class="ice-notes">${esc(it.notes)}</div>` : ""}${(it.images || []).filter((u) => /^https:\/\//.test(u)).length ? `<details class="ice-imgs"><summary>Show images (${it.images.filter((u) => /^https:\/\//.test(u)).length}) — loads them from the source's site</summary>${it.images.filter((u) => /^https:\/\//.test(u)).map((u, n) => { const src = `/ice-image?dir=${encodeURIComponent(it.dir)}&id=${encodeURIComponent(it.id)}&i=${n}`; return `<a href="${esc(src)}" target="_blank" rel="noopener noreferrer"><img data-src="${esc(src)}" alt="" referrerpolicy="no-referrer"></a>`; }).join("")}</details>` : ""}<span class="hint" style="margin:0">From ${esc(srcName(it))} · ${ago(it.at)}${it.post ? ` · <a href="${esc(it.post)}" target="_blank" rel="noopener">open post</a>` : ""}</span>${replyBox(it)}${form(it)}</div>`
         : "")
     );
   };

package/web/board/live.ts

@@ -52,8 +52,10 @@ export function initLive() {
               body: JSON.stringify({ dir: d }),
             })
           ).ok
-        )
+        ) {
           store.all = false;
+          store.dir = d;
+        }
         else keep();
       } catch (e) {}
     try {

package/web/board/projects.ts

@@ -52,6 +52,7 @@ async function pick(v) {
     const j = await post("/cd", { dir: v });
     if (j.ok) {
       store.all = false;
+      store.dir = v;
       keep(v);
     }
   }

package/web/board/store.ts

@@ -4,6 +4,8 @@
 export const store = {
   S: { project: null, projects: [], cards: [], now: Date.now(), all: false },
   all: new URLSearchParams(location.search).get("all") !== "0",
+  // The project this page picked; null is wherever the board started.
+  dir: null,
   openKey: null,
   mode: null,
   modeKey: null,