create-evda 0.1.9 → 0.1.11

package/lib/gather.mjs

@@ -27,9 +27,19 @@
 // and placed nowhere, and a person who already has a page there is not asked about. So a second
 // gather asks only about what is new. Applying the proposal is lib/gather-apply.mjs.
 //
+// Byte-identical files are placed once and every other copy is named as a duplicate of that one; an
+// item nobody placed is left in the pile, where it is, and the summary says so (HQ-1368).
+//
 // A placed `.txt` note is named `.md` on the way in, its bytes unchanged: plain text is already
-// markdown, and every reader in EVDA (the occurrence index, doctor, search) reads `.md`. Nothing
-// else is renamed; an `.html` export keeps its name, because making it markdown would change it.
+// markdown, and every reader in EVDA (the occurrence index, doctor, search) reads `.md`. A note-shaped
+// `.html` file (Google Keep's Takeout, an Apple Notes export) is converted to markdown at apply and
+// named `.md`, its title a heading, its checklist `- [x]` items and Keep's labels `tags:`; the
+// proposal marks it `convert: "html-to-markdown"` so the owner sees the change before saying yes
+// (lib/html-notes.mjs). Any other `.html` (a saved web page) keeps its name and its bytes.
+//
+// Google Takeout's `.json` sidecars are paired with the photo or note each describes and never placed
+// themselves: a photo's date taken and caption, a Keep note's created and edited times, are folded into
+// that file's entry (lib/takeout.mjs), and the sidecar is named in the summary with the file it describes.
 //
 // A chat export (HQ-1385: Claude.ai's or ChatGPT's conversations.json) is not a file to place. It is
 // the backstop for what an agent did not file while the conversation happened: the owner's own
@@ -53,13 +63,16 @@ import { askOwnersAi } from "./organize.mjs";
 import { onlyKeys, refuse, dayFile } from "./tools.mjs";
 import { readChatExport, conversations, filedSections, backfillPlan, CHAT_FORMATS } from "./chat-export.mjs";
 import { EvdaError } from "./errors.mjs";
+import { noteShape, htmlNoteToMarkdown } from "./html-notes.mjs";
+import { pairSidecars } from "./takeout.mjs";
 import { frontmatterOf, heldBy, withoutSealed, carries, NOTED } from "./circles.mjs";
+import { outsideRecord } from "./record-walk.mjs";
 
 export const GATHER_VERSION = 1;
 // The person-layer kinds an item is typed with, and the folder type each lives under.
 export const GATHER_KINDS = ["note", "person", "project", "event", "log", "source", "piece", "blob", "capture"];
 export const FOLDER_TYPE_FOR = { note: "record", person: "person", project: "project", event: "event", log: "log", source: "source", piece: "piece", blob: "blob", capture: "capture" };
-export const FALSIFY_MODES = ["touch-source", "copy-contacts", "ask-everything", "answer-no-replace", "credentials-ordinary", "examine-nothing", "no-role"];
+export const FALSIFY_MODES = ["touch-source", "copy-contacts", "ask-everything", "answer-no-replace", "credentials-ordinary", "examine-nothing", "no-role", "keep-junk", "container-subjects", "place-duplicates", "html-as-is"];
 // The role each kind fills when the preset has no folder for it (Spec 07 section 2, HQ-1371):
 // what happened, dated, is journal; one folder per thing is collections; what is known is knowledge.
 export const ROLE_FOR_KIND = { note: "journal", capture: "journal", log: "journal", person: "collections", project: "collections", event: "collections", piece: "collections", blob: "collections", course: "collections", source: "knowledge", areas: "collections" };
@@ -102,12 +115,37 @@ const sha256 = (buf) => createHash("sha256").update(buf).digest("hex");
 
 const SKIP_NAMES = new Set([".DS_Store", "Thumbs.db", "desktop.ini", "node_modules", ".git", ".obsidian", ".trash", ".Trash", "__MACOSX"]);
 const APP_DIRS = { ".obsidian": "obsidian" };
+// App-state junk a real pile carries (HQ-1368, the 2026-09-27 trial placed every one of these): what
+// an app wrote for itself, not something the owner kept. Each is skipped, with why, and named in the
+// summary. `junkWhy(name, size, isDir)` is the reason, or null for a file the owner may want.
+export const JUNK_DIRS = new Set(["recently deleted"]);
+export const JUNK_FILES = new Set(["archive_browser.html"]);
+export function junkWhy(name, size, isDir = false) {
+  if (falsified("keep-junk")) return null;
+  const lower = name.toLowerCase();
+  if (isDir) return JUNK_DIRS.has(lower) ? "what the app had already deleted" : null;
+  if (name.startsWith("~$")) return "an Office lock file, written while a document was open";
+  if (JUNK_FILES.has(lower)) return "Google Takeout's index of the export, not the owner's";
+  if (size === 0) return "an empty file (zero bytes)";
+  return null;
+}
 const ARCHIVE_EXT = new Set([".zip", ".tar", ".gz", ".tgz", ".7z", ".rar"]);
 const BINARY_EXT = new Set([".png", ".jpg", ".jpeg", ".gif", ".heic", ".webp", ".pdf", ".mov", ".mp4", ".m4a", ".mp3", ".wav", ".xlsx", ".xls", ".docx", ".doc", ".pptx", ".key", ".pages", ".numbers", ".sqlite", ".db"]);
 const TEXT_MAX = 2 * 1024 * 1024;
 const NOTE_SIZED = 20000; // characters: longer text is read for therapy words only in its name and heading
 // Folder names that hold things without saying what they are about, so they are not a subject.
 const CONTAINERS = new Set(["documents", "desktop", "downloads", "notes", "daily", "inbox", "captures", "vault", "obsidian vault", "misc", "stuff", "untitled folder", "new folder", "export", "icloud drive"]);
+// The folders an export or an app names for itself (HQ-1368, the 2026-09-27 trial read Keep's `Keep/`
+// as a subject and cited "My Contacts" and "Photos from 2025" in the preset's reason): they say which
+// app the files came out of, never what they are about, so none is ever a subject, at any depth.
+export const EXPORT_CONTAINERS = new Set(["takeout", "google takeout", "keep", "google keep", "drive", "my drive", "google drive", "shared with me", "google photos", "my contacts", "contacts", "apple notes", "icloud", "onedrive", "dropbox", "evernote", "notion", "exports", "backup", "backups"]);
+const EXPORT_PATTERNS = [/^photos from \d{4}$/i, /^takeout[-_]\d/i];
+export function isContainer(name) {
+  const lower = name.toLowerCase();
+  if (CONTAINERS.has(lower)) return true;
+  if (falsified("container-subjects")) return false;
+  return EXPORT_CONTAINERS.has(lower) || EXPORT_PATTERNS.some((re) => re.test(name));
+}
 const COURSE = /\b[A-Z]{2,4}\s?\d{3}[A-Z]?\b/;
 const COURSE_WORD = /\b(syllabus|lecture|homework|problem set|midterm)\b/i;
 const THERAPY = /\b(therapy|therapist|counsell?or|counselling|counseling|psychiatrist|psychologist|CBT|EMDR)\b/i;
@@ -182,7 +220,7 @@ const slugId = (s) => s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-
 // added to its frontmatter, a finished status) no longer hashes like its source; this is how a
 // second gather still knows it.
 export const GATHERED = ".evda/gathered.jsonl";
-const WALK_SKIP = new Set([".git", "node_modules", ".evda", ".obsidian", ".trash", ".DS_Store"]);
+const WALK_SKIP = new Set([".git", ".evda", ".obsidian", ".trash", ".DS_Store"]); // plus outsideRecord
 
 // Every file in the record by the hash of its bytes, and the source hashes it has gathered.
 // Returns { byHash: Map<sha256, rel>, gathered: Map<sha256, rel> }.
@@ -192,7 +230,7 @@ export function presentIndex(dir) {
   if (falsified("reapply-resorts")) return { byHash, gathered };
   const go = (d) => {
     for (const name of readdirSync(d).sort()) {
-      if (WALK_SKIP.has(name)) continue;
+      if (WALK_SKIP.has(name) || outsideRecord(name)) continue;
       const abs = join(d, name);
       const st = lstatSync(abs);
       if (st.isDirectory()) go(abs);
@@ -240,9 +278,11 @@ function walk(root) {
       const abs = join(dir, name);
       const rel = posix(relative(root, abs));
       if (APP_DIRS[name]) apps.add(APP_DIRS[name]);
-      if (SKIP_NAMES.has(name)) { skipped.push({ rel, why: APP_DIRS[name] ? `${APP_DIRS[name]}'s own settings, not the owner's notes` : "app or system state" }); continue; }
+      if (SKIP_NAMES.has(name) || outsideRecord(name)) { skipped.push({ rel, why: APP_DIRS[name] ? `${APP_DIRS[name]}'s own settings, not the owner's notes` : "app or system state" }); continue; }
       const s = lstatSync(abs);
       if (s.isSymbolicLink()) { skipped.push({ rel, why: "a symlink; not followed" }); continue; }
+      const junk = s.isDirectory() || s.isFile() ? junkWhy(name, s.size, s.isDirectory()) : null;
+      if (junk) { skipped.push({ rel: s.isDirectory() ? `${rel}/` : rel, why: junk }); continue; }
       if (s.isDirectory()) { go(abs); continue; }
       if (!s.isFile()) { skipped.push({ rel, why: "not a regular file" }); continue; }
       if (name.startsWith(".")) { skipped.push({ rel, why: "a hidden file" }); continue; }
@@ -259,7 +299,7 @@ function subjectOf(rel, projectRoot) {
   const parts = rel.split("/").slice(0, -1);
   for (let i = parts.length - 1; i >= 0; i--) {
     const p = parts[i];
-    if (CONTAINERS.has(p.toLowerCase())) continue;
+    if (isContainer(p)) continue;
     if (/^(people|contacts|meetings|projects|drafts|writing|essays|clippings|reading|articles|bookmarks|logs|calendar|photos|attachments)$/i.test(p)) continue;
     return p;
   }
@@ -349,7 +389,13 @@ export function scan(sources, { now = new Date() } = {}) {
     const readmeDirs = new Set(w.files.filter((f) => /^readme\.(md|txt)$/i.test(basename(f.rel)) && f.rel.includes("/")).map((f) => dirname(f.rel)).filter((d) => w.files.filter((f) => dirname(f.rel) === d).length >= 2));
     out.sources.push({ id: `s${si}`, path: root, files: w.files.length, skipped: w.skipped.length, apps: w.apps });
     for (const s of w.skipped) out.skipped.push({ source: `s${si}`, ...s });
+    // Google Takeout's sidecars (HQ-1368, lib/takeout.mjs): each is paired with the photo or note it
+    // describes, its date and caption folded into that file's entry, and is never an entry of its own.
+    const side = pairSidecars(w.files, (abs) => readFileSync(abs));
+    const sidecarAt = new Map(side.sidecars.map((x) => [x.rel, x]));
+    for (const x of side.sidecars) out.skipped.push({ source: `s${si}`, ...x });
     for (const f of w.files) {
+      if (sidecarAt.has(f.rel)) continue;
       const buf = readFileSync(f.abs);
       const ext = extname(f.rel).toLowerCase();
       const chat = ext === ".json" ? readChatExport(buf) : null;
@@ -364,12 +410,22 @@ export function scan(sources, { now = new Date() } = {}) {
       // section is cut before any file is read for its heading, people or words. Its bytes and hash
       // are unchanged, so apply copies it with its flag.
       const seal = sealedOf(ext, buf);
-      const facts = typeItem({ rel: f.rel, buf: seal.buf, ext, projectRoot });
+      // A note-shaped HTML file is read as the markdown it will become (HQ-1368), so it is typed,
+      // searched for people and checked for a credential by its words, not its tags.
+      const shape = (ext === ".html" || ext === ".htm") && !falsified("html-as-is") ? noteShape(buf.toString("utf8")) : null;
+      const conv = shape ? htmlNoteToMarkdown(buf.toString("utf8")) : null;
+      if (conv?.empty) { out.skipped.push({ source: `s${si}`, rel: f.rel, why: "an empty note: no title and no body" }); continue; }
+      if (conv) seal.buf = Buffer.from(conv.markdown, "utf8");
+      const facts = typeItem({ rel: f.rel, buf: seal.buf, ext: conv ? ".md" : ext, projectRoot });
+      if (conv) facts.html = shape;
       if (seal.sealed) facts.sealed = true;
       if (seal.noted) facts.noted = true;
       const item = { id: `s${si}:${f.rel}`, source: `s${si}`, path: f.rel, size: f.size, sha256: sha256(buf), ...facts };
       if (projectRoot) item.project = projectRoot;
+      if (side.folded.has(f.rel)) item.takeout = side.folded.get(f.rel);
       item.subject = subjectOf(f.rel, projectRoot);
+      // A Keep note sits in Takeout/Keep, which says nothing of what it is about; its first label does.
+      if (!item.subject && !projectRoot && conv?.labels.length) item.subject = conv.labels[0];
       if (!item.refused && !item.credential && !item.sealed && !item.noted && item.kind !== "blob" && item.mode !== "reference") item.text = seal.buf.toString("utf8");
       out.items.push(item);
     }
@@ -417,9 +473,11 @@ function findPeople(items) {
   for (const p of byName.values()) {
     const first = p.name.split(" ")[0];
     const re = new RegExp(`(^|[^\\p{L}])(${escape(p.name)}${firsts.get(first) === 1 && first.length >= 3 ? `|${escape(first)}` : ""})($|[^\\p{L}])`, "u");
+    // Identical copies of one file are one mention, so a duplicate does not count a person twice.
+    const seen = new Set();
     for (const i of items) {
-      if (i.text === undefined || p.pages.includes(i.id)) continue;
-      if (re.test(i.text) || re.test(i.path)) p.files.add(i.id);
+      if (i.text === undefined || p.pages.includes(i.id) || seen.has(i.sha256)) continue;
+      if (re.test(i.text) || re.test(i.path)) { p.files.add(i.id); seen.add(i.sha256); }
     }
   }
   return [...byName.values()].map((p) => ({ name: p.name, ref: p.cards[0] ?? null, page: p.pages[0] ?? null, files: [...p.files].sort() })).sort((a, b) => a.name.localeCompare(b.name));
@@ -484,8 +542,12 @@ function homes(preset) {
   };
 }
 
+// A name a copy is given by the system that made it: "x (1)", "x copy", "x copy 2", "Copy of x".
+const COPY_MARKER = /(?: \(\d+\)| copy(?: \d+)?|^copy of .+)$/i;
+
 // A placed text item's name: `.txt` becomes `.md`, bytes unchanged. Every other name is kept.
-const asMarkdown = (i, name) => (i.kind !== "blob" && /\.txt$/i.test(name) ? name.replace(/\.txt$/i, ".md") : name);
+// A note-shaped `.html` becomes `.md` too, its bytes converted at apply (lib/html-notes.mjs).
+const asMarkdown = (i, name) => (i.kind === "blob" ? name : /\.txt$/i.test(name) ? name.replace(/\.txt$/i, ".md") : i.html && /\.html?$/i.test(name) ? name.replace(/\.html?$/i, ".md") : name);
 
 // The placement of every item, given the facts and the answers. Pure.
 export function place(p) {
@@ -526,7 +588,7 @@ export function place(p) {
       // Only markdown carries `sealed: true`, so a credential in any other format is never copied.
       pl = { status: "refused", to: null, why: `${i.credential} and is not a note that can be sealed; left where it is and quoted nowhere. Keep the secret in a password manager.` };
     } else if (i.present) pl = { status: "present", to: null, at: i.present, why: `already in the record at ${i.present}; left where it is` };
-    else if (i.archive) pl = { status: "unplaced", to: null, why: "an archive file; unpack it and gather the folder" };
+    else if (i.archive) pl = { status: "unplaced", to: null, why: "an archive file, left in the pile; unpack it and gather the folder" };
     else if (i.cards) pl = { status: "reference", to: null, why: "held by Contacts; a person page keeps a contacts: reference, never the card" };
     else if (i.kind === "project") {
       // Status is a field, never a move: a finished project stays in its collection and says so.
@@ -541,18 +603,44 @@ export function place(p) {
       const course = h.courses && i.subject && COURSE.test(i.subject);
       if (course) pl = { status: "placed", to: `${h.courses}/${i.subject}/${file}` };
       else if (i.subject) pl = { status: "placed", to: `${folderFor("areas", "record")}/${i.subject}/${file}` };
-      else pl = { status: "unplaced", to: null, why: `no subject: it sits in no folder that says what it is about${h.captures ? `; ${h.captures}/ holds it until someone says` : ""}` };
+      else pl = { status: "unplaced", to: null, why: "no subject: it sits in no folder that says what it is about; left in the pile, where it is, until someone says where it goes" };
     } else if (i.mode === "reference") pl = { status: "reference", to: `${folderFor(i.kind, FOLDER_TYPE_FOR[i.kind])}/${basename(i.path, extname(i.path))}.md`, refs: i.refs, why: "a new entry holding the reference; the file stays where it is" };
     else pl = { status: "placed", to: `${folderFor(i.kind, FOLDER_TYPE_FOR[i.kind])}/${file}` };
-    if (pl.to) pl.to = claim(pl.to, i.id);
-    if (pl.to) pl.role = roleOf(pl.to);
     if (i.refs && pl.status === "placed") pl.refs = i.refs;
     if (i.kind === "person" && !i.cards) {
       const who = p.people.find((x) => x.page === i.id);
       if (who?.ref) pl.refs = [who.ref];
     }
+    if (i.html && pl.to && pl.status === "placed") pl.convert = "html-to-markdown";
     placements.push({ item: i.id, ...pl });
   }
+  // Byte-identical files are placed once (HQ-1368): the copy placed is one that has somewhere to go,
+  // then one whose name carries no copy marker ("ENERGY STAR (1).pdf", "Plan copy.txt"), then the
+  // shortest path, then the first walked; every other copy stays in the pile, named as a duplicate of
+  // that one. A project's own files are kept whole, so an identical file in two projects is in both.
+  if (!falsified("place-duplicates")) {
+    const items = new Map(p.items.map((i) => [i.id, i]));
+    const groups = new Map();
+    placements.forEach((pl, n) => {
+      const i = items.get(pl.item);
+      if (!["placed", "reference", "unplaced"].includes(pl.status) || i.kind === "project" || i.cards || i.credential) return;
+      groups.set(i.sha256, [...(groups.get(i.sha256) ?? []), n]);
+    });
+    const marked = (path) => COPY_MARKER.test(basename(path, extname(path)));
+    for (const g of groups.values()) {
+      if (g.length < 2) continue;
+      const rank = (n) => [placements[n].status === "unplaced" ? 1 : 0, marked(items.get(placements[n].item).path) ? 1 : 0, placements[n].item.length, n];
+      const order = (a, b) => { const x = rank(a), y = rank(b); const k = x.findIndex((v, j) => v !== y[j]); return k === -1 ? 0 : x[k] - y[k]; };
+      const keep = [...g].sort(order)[0];
+      const kept = placements[keep];
+      for (const n of g) if (n !== keep) placements[n] = { item: placements[n].item, status: "duplicate", to: null, of: kept.item, why: `the same bytes as ${kept.item}, which is ${kept.status === "unplaced" ? "also left in the pile" : "placed"}; this copy is left in the pile` };
+    }
+  }
+  // Names are claimed after the duplicates are out, so the copy kept never yields its name to one left behind.
+  for (const pl of placements) {
+    if (pl.to) pl.to = claim(pl.to, pl.item);
+    if (pl.to) pl.role = roleOf(pl.to);
+  }
   // Person pages the owner said yes to, made new, holding a Contacts reference when a card names them.
   for (const who of p.people) {
     if (ans[`person:${slugId(who.name)}`] !== "yes" || who.page) continue;
@@ -622,7 +710,7 @@ export function questions(p) {
 // ---- the proposal ----
 
 function counts(p) {
-  const c = { items: p.items.length, placed: 0, reference: 0, create: 0, unplaced: 0, refused: 0, present: 0, skipped: p.skipped.length };
+  const c = { items: p.items.length, placed: 0, reference: 0, create: 0, unplaced: 0, refused: 0, present: 0, duplicate: 0, skipped: p.skipped.length, sidecars: p.skipped.filter((x) => x.sidecar).length };
   for (const pl of p.placements) c[pl.status] += 1;
   c.chats = (p.chats ?? []).length;
   c.chatMessages = (p.chats ?? []).reduce((n, x) => n + x.messages, 0);
@@ -782,11 +870,12 @@ export function checkedLines(p) {
   const chats = p.chats ?? [];
   return [
     ...(chats.length ? [`CHECKED ${c.chatMessages} message(s) of the owner's in ${chats.reduce((n, x) => n + x.conversations, 0)} conversation(s) from ${chats.length} chat export(s)${p.into ? `  ${chats.reduce((n, x) => n + (x.add ?? 0), 0)} not yet in the record` : ""}`] : []),
-    `CHECKED ${c.items} items in ${p.sources.length} source(s)  ${c.placed} placed, ${c.reference} by reference, ${c.create} to create, ${c.unplaced} left for a person, ${c.refused} refused, ${c.skipped} skipped${p.into ? `, ${c.present} already in the record` : ""}`,
+    `CHECKED ${c.items} items in ${p.sources.length} source(s)  ${c.placed} placed, ${c.reference} by reference, ${c.create} to create, ${c.unplaced} left for a person, ${c.refused} refused, ${c.skipped} skipped${p.into ? `, ${c.present} already in the record` : ""}${c.duplicate ? `, ${c.duplicate} duplicate(s) left in the pile` : ""}${c.sidecars ? ` (${c.sidecars} of them Takeout sidecars, ${p.items.filter((i) => i.takeout).length} folded into their files)` : ""}`,
     c.questions ? `CHECKED ${c.questions} question(s)  each changes where at least one item goes; ${c.open} open` : "SKIPPED questions: nothing in this pile needs the owner to decide a placement",
   ];
 }
 
+const SKIPPED_SHOWN = 20;
 export function summaryText(p, { detail = false } = {}) {
   const lines = [`A proposal for ${p.into ? `growing the EVDA at ${p.into} from` : "turning"} ${p.sources.map((s) => s.path).join(", ")}${p.into ? "" : " into an EVDA"}. Nothing was moved, copied or changed.`, ""];
   lines.push(`Preset: ${p.preset.title} (${p.preset.id}), because ${p.preset.why.join("; ")}.`, "", "Folders:");
@@ -794,12 +883,24 @@ export function summaryText(p, { detail = false } = {}) {
   const byStatus = (s) => p.placements.filter((x) => x.status === s);
   if (byStatus("reference").length) { lines.push("", "Kept as references, never copied:"); for (const x of byStatus("reference")) lines.push(`- ${x.item}${x.to ? ` -> ${x.to}` : ""}: ${x.why}`); }
   if (byStatus("create").length) { lines.push("", "New pages:"); for (const x of byStatus("create")) lines.push(`- ${x.to}${x.refs.length ? ` (${x.refs.join(", ")})` : ""}`); }
-  if (byStatus("unplaced").length) { lines.push("", "Left for a person:"); for (const x of byStatus("unplaced")) lines.push(`- ${x.item}: ${x.why}`); }
+  if (byStatus("unplaced").length) { lines.push("", "Left in the pile, where they are, for a person to place:"); for (const x of byStatus("unplaced")) lines.push(`- ${x.item}: ${x.why}`); }
+  if (byStatus("duplicate").length) { lines.push("", "Duplicates, left in the pile; one copy of each is placed:"); for (const x of byStatus("duplicate")) lines.push(`- ${x.item}: the same bytes as ${x.of}`); }
   if (byStatus("refused").length) { lines.push("", "Refused:"); for (const x of byStatus("refused")) lines.push(`- ${x.item}: ${x.why}`); }
+  if (p.skipped.length) {
+    // Named, so the owner sees what was left out and why (HQ-1368). A long list is cut unless --all.
+    const shown = detail ? p.skipped : p.skipped.slice(0, SKIPPED_SHOWN);
+    lines.push("", "Skipped, never placed:");
+    for (const x of shown) lines.push(`- ${x.rel}: ${x.why}`);
+    if (shown.length < p.skipped.length) lines.push(`- and ${p.skipped.length - shown.length} more (show --all lists them)`);
+  }
   if (byStatus("present").length) lines.push("", `Already in the record, left where they are: ${byStatus("present").length} item(s).`);
   for (const c of p.chats ?? []) lines.push("", `A ${CHAT_FORMATS[c.format]} export (${c.path}): ${c.messages} message(s) of the owner's in ${c.conversations} conversation(s)${c.add !== undefined ? `, ${c.add} of them not in the record yet, to be added under each conversation's date and time` : ", each added to its day unless the record already holds it"}. Only their messages; the model's replies are never written.`);
   if (p.circles.length) { lines.push("", "Circles:"); for (const c of p.circles) lines.push(`- ${c.name} over ${c.covers}/, ${c.status}`); }
-  if (detail) { lines.push("", "Every placement:"); for (const x of byStatus("placed")) lines.push(`- ${x.item} -> ${x.to}`); }
+  const converted = byStatus("placed").filter((x) => x.convert === "html-to-markdown");
+  if (converted.length) { lines.push("", `Converted from HTML to markdown on the way in, so they read as notes: ${converted.length} note(s); the pile keeps its .html files.`); for (const x of converted.slice(0, 20)) lines.push(`- ${x.item} -> ${x.to}`); if (converted.length > 20) lines.push(`- and ${converted.length - 20} more`); }
+  const folded = p.items.filter((i) => i.takeout);
+  if (folded.length) { lines.push("", `Google Takeout's details folded in, so each sidecar is no entry of its own: ${folded.length} file(s); a photo's date taken becomes its file's date, a note's times its frontmatter. No place and no tagged person is read.`); for (const i of folded.slice(0, 20)) lines.push(`- ${i.id}${i.takeout.taken ? `, taken ${i.takeout.taken}` : ""}${i.takeout.created ? `, created ${i.takeout.created}` : ""}${i.takeout.caption ? ", with a caption" : ""} (from ${i.takeout.sidecar})`); if (folded.length > 20) lines.push(`- and ${folded.length - 20} more`); }
+  if (detail) { lines.push("", "Every placement:"); for (const x of byStatus("placed")) lines.push(`- ${x.item} -> ${x.to}${x.convert ? " (converted from HTML)" : ""}`); }
   const open = p.questions.filter((q) => q.answer === undefined);
   lines.push("", open.length ? `${open.length} question(s) for the owner:` : "No open questions.");
   for (const q of open) lines.push(`- [${q.id}] ${q.text} (${q.options.join(" / ")}${q.default ? `, default ${q.default}` : ""})`);

package/lib/html-notes.mjs

@@ -0,0 +1,124 @@
+// Note-shaped HTML to markdown (HQ-1368). Google Keep's Takeout export and Apple Notes' HTML export
+// write each note as a small HTML file, and every reader in EVDA (search, the occurrence index,
+// doctor) reads markdown, so a note left as HTML is invisible to all of them. A gather converts a
+// note-shaped HTML file into a `.md` note at apply, and says so in the proposal before anyone says yes.
+//
+// Only note-shaped HTML is converted. A saved web page (scripts, stylesheet links, navigation, forms,
+// a "saved from url" comment) is not a note the owner wrote: it keeps its `.html` name and its bytes.
+//
+//   noteShape(html)      "keep", "apple-notes", "html-note", or null when the file is not a note
+//   htmlNoteToMarkdown(html)  { markdown, title, labels, empty }: the note as markdown, frontmatter
+//                        `tags:` holding Keep's labels; `empty` when it has neither a title nor a body
+//
+// EVDA_GATHER_FALSIFY=html-as-is turns the conversion off (lib/gather.mjs), so the tests can show
+// their checks fail when it is gone.
+
+const NAMED = { amp: "&", lt: "<", gt: ">", quot: "\"", apos: "'", nbsp: " ", ndash: "-", mdash: "-", hellip: "...", lsquo: "'", rsquo: "'", ldquo: "\"", rdquo: "\"", bull: "*", copy: "(c)", reg: "(R)", trade: "(TM)" };
+export function decodeEntities(s) {
+  return s.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (m, e) => {
+    if (e[0] === "#") {
+      const n = e[1] === "x" || e[1] === "X" ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10);
+      return Number.isFinite(n) && n > 0 && n <= 0x10ffff ? String.fromCodePoint(n) : m;
+    }
+    return NAMED[e.toLowerCase()] ?? m;
+  }).replace(/ /g, " ");
+}
+
+const KEEP = /<div\s+class="note\b[^"]*"/i;
+const KEEP_PART = /class="(?:title|content|heading|chips|list(?:item)?)"/i;
+// What a saved web page carries and a note never does.
+const NOT_A_NOTE = /<!--\s*saved from url|<(?:nav|form|iframe|input|button|select|textarea|header|footer|aside|noscript|frameset|object|embed)\b|<link\b[^>]*rel=["']?stylesheet|<script\b|<meta\b[^>]*property=["']?og:/i;
+// The tags a note's body is made of. Anything else and it is a page, not a note.
+const NOTE_TAGS = new Set(["html", "head", "meta", "title", "style", "body", "div", "p", "br", "span", "font", "b", "strong", "i", "em", "u", "s", "strike", "del", "h1", "h2", "h3", "h4", "h5", "h6", "ul", "ol", "li", "a", "img", "tt", "code", "pre", "blockquote", "table", "thead", "tbody", "tr", "td", "th", "hr", "sup", "sub", "center", "small", "big"]);
+const NOTE_SIZED = 20000;
+
+export function noteShape(html) {
+  const raw = String(html ?? "");
+  if (!/<(?:html|body|div|p|br|h1)\b/i.test(raw)) return null;
+  if (KEEP.test(raw) && KEEP_PART.test(raw)) return "keep";
+  if (NOT_A_NOTE.test(raw)) return null;
+  const withoutStyle = raw.replace(/<style\b[\s\S]*?<\/style\s*>/gi, "").replace(/<!--[\s\S]*?-->/g, "").replace(/<!doctype[^>]*>/gi, "");
+  for (const m of withoutStyle.matchAll(/<\/?([a-z][a-z0-9]*)\b/gi)) if (!NOTE_TAGS.has(m[1].toLowerCase())) return null;
+  const text = decodeEntities(withoutStyle.replace(/<[^>]+>/g, " ")).replace(/\s+/g, " ").trim();
+  if (text.length > NOTE_SIZED) return null;
+  if (/<meta\b[^>]*(?:name=["']?generator["']?[^>]*content=["'][^"']*notes|apple-notes)/i.test(raw) || /<div>[\s\S]*<\/div>/i.test(withoutStyle) && !/<p\b/i.test(withoutStyle)) return "apple-notes";
+  return "html-note";
+}
+
+// The element whose opening tag matches `open`, with everything up to its balanced closing tag.
+function element(html, open) {
+  const m = open.exec(html);
+  if (!m) return null;
+  const tag = /^<([a-z0-9]+)/i.exec(m[0])[1].toLowerCase();
+  const re = new RegExp(`<(/?)${tag}\\b[^>]*>`, "gi");
+  re.lastIndex = m.index + m[0].length;
+  let depth = 1;
+  for (let t; (t = re.exec(html));) {
+    depth += t[1] ? -1 : 1;
+    if (depth === 0) return { inner: html.slice(m.index + m[0].length, t.index), start: m.index, end: re.lastIndex };
+  }
+  return { inner: html.slice(m.index + m[0].length), start: m.index, end: html.length };
+}
+
+// Inline tags to markdown. Entities are decoded once, at the end of a block pass, so a decoded "&lt;b&gt;"
+// is never read again as a tag.
+const tags = (s) => (s
+  .replace(/<br\s*\/?>/gi, "\n")
+  .replace(/<(b|strong)\b[^>]*>([\s\S]*?)<\/\1\s*>/gi, (_, __, t) => (t.replace(/<[^>]+>/g, "").trim() ? `**${t.trim()}**` : t))
+  .replace(/<(i|em)\b[^>]*>([\s\S]*?)<\/\1\s*>/gi, (_, __, t) => (t.replace(/<[^>]+>/g, "").trim() ? `*${t.trim()}*` : t))
+  .replace(/<a\b[^>]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a\s*>/gi, (_, href, t) => { const label = t.replace(/<[^>]+>/g, "").trim(); return label && label !== href ? `[${label}](${href})` : href; })
+  .replace(/<img\b[^>]*src=["']([^"']+)["'][^>]*>/gi, (_, src) => (src.startsWith("data:") ? "" : `![](${src})`))
+  .replace(/<[^>]+>/g, ""));
+const inline = (s) => decodeEntities(tags(s));
+
+// A body of HTML to markdown lines: blocks become lines, list items bullets, Keep's checked items `- [x]`.
+function blocks(html) {
+  let s = html
+    .replace(/<(script|style)\b[\s\S]*?<\/\1\s*>/gi, "")
+    .replace(/<!--[\s\S]*?-->/g, "")
+    .replace(/<head\b[\s\S]*?<\/head\s*>/gi, "");
+  s = s.replace(/<li\b([^>]*)>([\s\S]*?)<\/li\s*>/gi, (_, attrs, inner) => {
+    const keep = /class="[^"]*listitem/i.test(attrs);
+    const checked = keep && /class="[^"]*\bchecked\b/i.test(attrs);
+    const text = keep ? (/<span\s+class="text"[^>]*>([\s\S]*?)<\/span>/i.exec(inner)?.[1] ?? inner.replace(/<span\s+class="bullet"[^>]*>[\s\S]*?<\/span>/gi, "")) : inner;
+    const t = tags(text).replace(/\s*\n\s*/g, " ").trim();
+    return `\n${keep ? (checked ? "- [x] " : "- [ ] ") : "- "}${t}\n`;
+  });
+  s = s.replace(/<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1\s*>/gi, (_, n, t) => `\n${"#".repeat(Number(n))} ${tags(t).replace(/\s+/g, " ").trim()}\n\n`);
+  s = s.replace(/<hr\b[^>]*>/gi, "\n---\n");
+  s = s.replace(/<\/(div|p|tr|blockquote|pre|ul|ol|table)\s*>/gi, "\n").replace(/<(div|p|tr|blockquote|pre|ul|ol|table)\b[^>]*>/gi, "\n").replace(/<\/t[dh]\s*>/gi, " ");
+  const lines = inline(s).split("\n").map((l) => l.replace(/[ \t]+/g, " ").trim());
+  // A list is tight: no blank line between two items of it.
+  return lines.join("\n").replace(/\n{3,}/g, "\n\n").replace(/^(- .*)\n\n(?=- )/gm, "$1\n").replace(/^(- .*)\n\n(?=- )/gm, "$1\n").trim();
+}
+
+const yamlString = (s) => JSON.stringify(s);
+
+export function htmlNoteToMarkdown(html) {
+  const raw = String(html ?? "");
+  const shape = noteShape(raw);
+  let title = "";
+  let body = "";
+  let labels = [];
+  if (shape === "keep") {
+    const t = element(raw, /<div\s+class="title"[^>]*>/i);
+    title = t ? inline(t.inner).replace(/\s+/g, " ").trim() : "";
+    const c = element(raw, /<div\s+class="content"[^>]*>/i);
+    body = c ? blocks(c.inner) : "";
+    labels = [...raw.matchAll(/<span\s+class="label-name"[^>]*>([\s\S]*?)<\/span>/gi)].map((m) => inline(m[1]).trim()).filter(Boolean);
+  } else {
+    const b = element(raw, /<body\b[^>]*>/i);
+    body = blocks(b ? b.inner : raw);
+    const docTitle = /<title\b[^>]*>([\s\S]*?)<\/title\s*>/i.exec(raw);
+    const first = body.split("\n")[0] ?? "";
+    const h = /^#{1,6} (.+)$/.exec(first);
+    const bold = /^\*\*(.+)\*\*$/.exec(first);
+    if (h || bold) { title = (h ?? bold)[1].trim(); body = body.split("\n").slice(1).join("\n").trim(); }
+    else if (docTitle) title = inline(docTitle[1]).replace(/\s+/g, " ").trim();
+    if (title && body.split("\n")[0].trim() === title) body = body.split("\n").slice(1).join("\n").trim();
+  }
+  const empty = !title && !body;
+  const front = labels.length ? `---\ntags: [${labels.map(yamlString).join(", ")}]\n---\n` : "";
+  const markdown = empty ? "" : `${front}${title ? `# ${title}\n` : ""}${title && body ? "\n" : ""}${body ? `${body}\n` : ""}`;
+  return { markdown, title: title || null, labels, empty, shape };
+}

package/lib/keep-hook.mjs

@@ -29,14 +29,23 @@
 // anything. A block stores the event index it blocked at, so the next block needs a new stretch.
 // Any error of its own exits 0 without output: a broken hook must never trap a session.
 //
+// NAMED ENTRIES (HQ-2799). When the day check lets the session stop, a second check runs: a tool
+// result in the transcript named existing entries (lib/touches.mjs's line, parsed by day-file.mjs's
+// namedIn), and nothing in the session wrote to any of them (a write tool or an evda_* call naming the
+// entry's path, or the entry's file modified since it was first named). Then it blocks once per
+// session, naming them and saying that adding nothing is fine. Not when the MCP server already asked
+// in a tool result of this session. Open tickets a result named (HQ-2798's line) count the same way,
+// answered by a board write naming the ticket's number. The transcript is the state; the state file only remembers the
+// ask.
+//
 // EVDA_RECORD_FALSIFY=no-block makes decide() never block, for the tests that show the block is
-// what the pass depends on.
+// what the pass depends on; no-entries-ask turns off the named-entries check alone.
 
 import { existsSync, mkdirSync, readFileSync, realpathSync, statSync, writeFileSync } from "node:fs";
 import { dirname, join } from "node:path";
 import { tmpdir } from "node:os";
 import { fileURLToPath } from "node:url";
-import { dayRole, findRecord, filingOf, localDate } from "./day-file.mjs";
+import { dayRole, findRecord, filingOf, localDate, namedIn, ticketsNamedIn, askedIn, entriesAsk } from "./day-file.mjs";
 
 export { dayRole, findRecord };
 
@@ -85,6 +94,15 @@ export function events(transcriptText) {
     const said = userText(d);
     if (said.trim()) out.push({ kind: "owner", words: said.trim().split(/\s+/).length, at });
     const c = d.message?.content;
+    if (d.type === "user" && Array.isArray(c)) {
+      for (const x of c) {
+        if (x?.type !== "tool_result") continue;
+        const text = typeof x.content === "string" ? x.content : Array.isArray(x.content) ? x.content.filter((y) => y?.type === "text").map((y) => y.text ?? "").join("\n") : "";
+        const named = namedIn(text);
+        const tickets = ticketsNamedIn(text);
+        if (named.length || tickets.length || askedIn(text)) out.push({ kind: "result", named, tickets, asked: askedIn(text), at });
+      }
+    }
     if (d.type === "assistant" && Array.isArray(c)) {
       for (const x of c) if (x?.type === "tool_use") out.push({ kind: "tool", name: String(x.name ?? ""), input: JSON.stringify(x.input ?? {}), at });
     }
@@ -109,7 +127,9 @@ export function classify(e, facts) {
 
 // The decision, pure. `lastBlock` is the event index this hook last blocked at (0 for never).
 // `fileMtime` is the day file's modification time in ms, or null when it does not exist.
-export function decide(evs, facts, { lastBlock = 0, fileMtime = null } = {}) {
+// `nonWriter` (HQ-2817) is the EVDA_DAY_WRITER value on a machine that does not write the day: the
+// block then says evda_write, because a direct write here is the iCloud race and the guard refuses it.
+export function decide(evs, facts, { lastBlock = 0, fileMtime = null, nonWriter = "" } = {}) {
   let lastWrite = 0;
   evs.forEach((e, i) => { if (classify(e, facts) === "write") lastWrite = i + 1; });
   const start = Math.max(lastWrite, lastBlock);
@@ -127,19 +147,62 @@ export function decide(evs, facts, { lastBlock = 0, fileMtime = null } = {}) {
   const why = reads && words >= OWNER_WORDS ? "read the record and the owner talked" : reads ? "read the record" : "the owner talked";
   const reason = [
     `evda: this conversation ${why}, and nothing from it is in today's record (${tools} tool call(s), ${writes} write(s), none to ${facts.fileRel}).`,
-    `Before you stop, file what the owner said that is worth keeping: their words verbatim, filler and typos intact, appended to ${facts.fileRel} under a "## HH:MM" heading or through evda_write.`,
+    nonWriter
+      ? `Before you stop, file what the owner said that is worth keeping: their words verbatim, filler and typos intact, through evda_write. This machine does not write ${facts.fileRel} (EVDA_DAY_WRITER names the one that does, ${nonWriter}), so never write that file directly here.`
+      : `Before you stop, file what the owner said that is worth keeping: their words verbatim, filler and typos intact, appended to ${facts.fileRel} under a "## HH:MM" heading or through evda_write.`,
     "Then tell them the path in the past tense. If a person, project or place with an entry came up, add a dated section to that entry too (never to a noted one); one with no entry, offer to start it and wait for their yes.",
     "If nothing they said is worth keeping, stop. This asks once per stretch.",
   ].join(" ");
   return { block: true, reason, ...facts_ };
 }
 
+const ANY_WRITE = /(^|__)evda_(write|note|append|append_row|append_item|record_person_note|replace|create)$/;
+
+// The named-entries decision, pure. `mtimeOf(rel)` is an entry's modification time in ms, or null.
+export function decideEntries(evs, facts, { asked = false, mtimeOf = () => null } = {}) {
+  if (asked || falsified("no-entries-ask")) return { block: false };
+  const named = [];
+  const tickets = [];
+  let since = NaN;
+  for (const e of evs) {
+    if (e.kind !== "result") continue;
+    if (e.asked) return { block: false };
+    for (const n of e.named) if (!named.some((x) => x.path === n.path)) named.push(n);
+    for (const k of e.tickets ?? []) if (!tickets.includes(k)) tickets.push(k);
+    if (e.named.length && !Number.isFinite(since)) since = e.at;
+  }
+  if (!named.length && !tickets.length) return { block: false };
+  const wrote = (rel) => evs.some((e) => {
+    if (e.kind !== "tool" || !names(e.input, rel, join(facts.root, rel))) return false;
+    return WRITE_TOOLS.has(e.name) || ANY_WRITE.test(e.name) || (e.name === "Bash" && BASH_WRITE.test(e.input));
+  }) || (Number.isFinite(since) && (mtimeOf(rel) ?? -Infinity) >= since);
+  // A ticket is answered by any board write naming its number: a harlequin tool other than a read,
+  // or the board CLI's update, done, handoff or schedule.
+  const updated = (id) => {
+    const n = id.slice(3);
+    const hit = new RegExp(`(HQ-${n}\\b|"number":"?${n}\\b|--number[ =]${n}\\b)`);
+    return evs.some((e) => e.kind === "tool" && hit.test(e.input) && ((/harlequin/i.test(e.name) && !/_(show|list|status)$/.test(e.name)) || (e.name === "Bash" && /\bboard\b.*\b(update|done|handoff|schedule)\b/.test(e.input))));
+  };
+  if (named.some((n) => wrote(n.path)) || tickets.some(updated)) return { block: false };
+  return { block: true, reason: `evda: ${entriesAsk(named.map((n) => n.path), "a dated section at the end, by Edit or evda_append; a person's page with evda_record_person_note", tickets)}` };
+}
+
 function stateFile(sid) {
   const dir = process.env.EVDA_KEEP_STATE_DIR || join(tmpdir(), "evda-keep-record");
   mkdirSync(dir, { recursive: true });
   return join(dir, `${String(sid).replace(/[^A-Za-z0-9_-]/g, "_")}.json`);
 }
 
+// When the prompt hook (lib/prompt-hook.mjs) last filed or queued a message of this session, in ms,
+// or null. Its state file is the one place both hooks can see without the record changing.
+function promptFiledAt(sid) {
+  const dir = process.env.EVDA_PROMPT_STATE_DIR || join(tmpdir(), "evda-file-prompt");
+  try {
+    const t = Date.parse(JSON.parse(readFileSync(join(dir, `${String(sid ?? "unknown").replace(/[^A-Za-z0-9_-]/g, "_")}.json`), "utf8")).filedAt);
+    return Number.isFinite(t) ? t : null;
+  } catch { return null; }
+}
+
 // One Stop event in, the hook's stdout out ("" for let it stop). Never throws.
 export function run(payload, { now = new Date() } = {}) {
   try {
@@ -154,14 +217,23 @@ export function run(payload, { now = new Date() } = {}) {
     if (!facts || facts.filing === "asked") return "";
     const evs = events(readFileSync(tpath, "utf8"));
     const sf = stateFile(payload.session_id ?? "unknown");
-    let lastBlock = 0;
-    try { lastBlock = Number(JSON.parse(readFileSync(sf, "utf8")).at) || 0; } catch { /* first stop of the session */ }
-    if (lastBlock > evs.length) lastBlock = 0; // a different transcript under the same id
-    const fileMtime = existsSync(facts.fileAbs) ? statSync(facts.fileAbs).mtimeMs : null;
-    const d = decide(evs, facts, { lastBlock, fileMtime });
-    if (!d.block) return "";
-    writeFileSync(sf, JSON.stringify({ at: d.at, when: now.toISOString() }) + "\n");
-    return JSON.stringify({ decision: "block", reason: d.reason });
+    let state = {};
+    try { state = JSON.parse(readFileSync(sf, "utf8")) ?? {}; } catch { /* first stop of the session */ }
+    let lastBlock = Number(state.at) || 0;
+    if (lastBlock > evs.length) { lastBlock = 0; state = {}; } // a different transcript under the same id
+    let fileMtime = existsSync(facts.fileAbs) ? statSync(facts.fileAbs).mtimeMs : null;
+    // HQ-2817: the prompt hook's last hand-off counts as a write to the day. On a machine that does
+    // not write the day the file here does not change when a message is filed through the writer.
+    const filedAt = promptFiledAt(payload.session_id);
+    if (filedAt !== null && (fileMtime === null || filedAt > fileMtime)) fileMtime = filedAt;
+    const nonWriter = (process.env.EVDA_DAY_WRITER ?? "").trim();
+    const d = decide(evs, facts, { lastBlock, fileMtime, nonWriter });
+    const mtimeOf = (rel) => { try { return statSync(join(root, rel)).mtimeMs; } catch { return null; } };
+    const e = d.block ? null : decideEntries(evs, facts, { asked: Boolean(state.entries), mtimeOf });
+    if (!d.block && !e.block) return "";
+    const next = d.block ? { ...state, at: d.at, when: now.toISOString() } : { ...state, at: lastBlock, entries: now.toISOString() };
+    writeFileSync(sf, JSON.stringify(next) + "\n");
+    return JSON.stringify({ decision: "block", reason: d.block ? d.reason : e.reason });
   } catch {
     return "";
   }