isorite 0.10.0 → 0.12.0

package/README.md

@@ -9,8 +9,7 @@ npm install -g isorite
 isorite guide               # the guide: a knowledge base about isorite, with things to do, in ~/isorite-guide
 ```
 
-On your own notes, a folder of Markdown files (`isorite new <notes>` starts one). They're drawn as you write them:
-maths, code coloured by its language, and Mermaid diagrams.
+On your own notes, a folder of Markdown files (`isorite new <notes>` starts one). They're drawn as you write them: maths, code coloured by its language, and Mermaid diagrams.
 
 ```sh
 isorite serve <notes>       # the browser, at http://localhost:4767/; set up Iso's provider in Iso's settings
@@ -19,9 +18,7 @@ isorite model               # optional: search by meaning (fetches a 53 MB model
 isorite memory              # optional: the agent's memory, a base of its own lent to every session
 ```
 
-Setting it up, at work too (a company's gateway, a client certificate, a proxy): `docs/setup.md` in this package.
-When something fails, `isorite doctor` says what to change; `isorite doctor --report` gives it as one block, its
-secrets hidden, to paste to someone helping. For an AI helping you: `isorite help setup`.
+Setting it up, at work too (a company's gateway, a client certificate, a proxy): `docs/setup.md` in this package. When something fails, `isorite doctor` says what to change; `isorite doctor --report` gives it as one block, its secrets hidden, to paste to someone helping. For an AI helping you: `isorite help setup`.
 
 The guide in a browser alone: https://lachlanjs.github.io/isorite/
 

package/dist/core.js

@@ -440,6 +440,38 @@ const pictureType = (path) => {
 	const dot = path.lastIndexOf(".");
 	return dot < 0 || path.lastIndexOf("/") > dot ? null : TYPES[path.slice(dot + 1).toLowerCase()] ?? null;
 };
+const OTHER = {
+	html: "text/html",
+	htm: "text/html",
+	csv: "text/csv",
+	tsv: "text/tab-separated-values",
+	json: "application/json",
+	jsonl: "application/jsonl",
+	ndjson: "application/x-ndjson",
+	txt: "text/plain",
+	tex: "text/plain",
+	pdf: "application/pdf",
+	glb: "model/gltf-binary",
+	gltf: "model/gltf+json",
+	stl: "model/stl",
+	obj: "model/obj",
+	zip: "application/zip",
+	parquet: "application/vnd.apache.parquet",
+	arrow: "application/vnd.apache.arrow.file",
+	npy: "application/octet-stream",
+	xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+	docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+	epub: "application/epub+zip",
+	odt: "application/vnd.oasis.opendocument.text"
+};
+/** Any file's media type in the notes (M23), by its extension: a picture's, one listed, or octet-stream. Null for a
+*  note (Markdown is read as a note, never as a file) and for a path with no extension. */
+const fileType = (path) => {
+	const dot = path.lastIndexOf(".");
+	if (dot < 0 || path.lastIndexOf("/") > dot) return null;
+	const ext = path.slice(dot + 1).toLowerCase();
+	return ext === "md" ? null : pictureType(path) ?? OTHER[ext] ?? "application/octet-stream";
+};
 /** Width and height in pixels, or null. */
 function pictureSize(bytes, type) {
 	const v = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength), n = bytes.length;
@@ -584,10 +616,19 @@ function outwardLinks(body) {
 /** The base's own instructions, for Iso and any other agent, at the top of the notes (M15). Not a note. */
 const AGENTS_FILE = "AGENTS.md";
 const OKF_VERSION = "0.2";
-const ARTIFACT = /\.html?$/i;
-const IMAGE = /\.(png|jpe?g|gif|svg|webp|avif)$/i;
-/** What kind of citable file a path is, or null. */
-const citeKind = (path) => ARTIFACT.test(path) ? "artifact" : IMAGE.test(path) ? "image" : null;
+const KINDS$2 = [
+	["artifact", /\.html?$/i],
+	["image", /\.(png|jpe?g|gif|svg|webp|avif)$/i],
+	["data", /\.(csv|tsv|json|jsonl|ndjson|parquet|arrow|feather|npy|npz|xlsx|xls|h5|hdf5|nc)$/i],
+	["document", /\.(pdf|docx?|odt|rtf|epub|pptx?|odp|txt|tex)$/i]
+];
+/** What kind of file a path is: null for a note, or a path with no extension (a note or a folder named without one);
+*  anything not listed is misc rather than dropped. */
+const citeKind = (path) => {
+	const name = basename(path);
+	if (/\.md$/i.test(name) || !/\.[^.]+$/.test(name.replace(/^\./, ""))) return null;
+	return KINDS$2.find(([, re]) => re.test(name))?.[0] ?? "misc";
+};
 const isEntry = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
 var Concept = class {
 	id;
@@ -1160,6 +1201,76 @@ function spliceText(before, changes, body) {
 	return bom + head + newBody;
 }
 //#endregion
+//#region packages/core/src/unwrap.ts
+const ITEM = /^\s*([-*+]|\d+[.)])\s+/;
+const APART = /^\s*(#{1,6}\s|\||<|```|~~~|\$\$|>|---\s*$|\*\*\*\s*$|___\s*$|\[[^\]]+\]:\s)/;
+const QUOTED = /^\s*>\s?\S/;
+function unwrap(text) {
+	const lines = text.split("\n"), out = [];
+	let i = 0, fence = "", maths = false;
+	/** What the last line written may be continued by: prose, a quoted line, or nothing. */
+	let open = "";
+	if (lines[0] === "---") {
+		const end = lines.indexOf("---", 1);
+		if (end > 0) {
+			out.push(...lines.slice(0, end + 1));
+			i = end + 1;
+		}
+	}
+	for (; i < lines.length; i++) {
+		const l = lines[i], t = l.trim();
+		if (fence) {
+			out.push(l);
+			if (t.startsWith(fence)) fence = "";
+			continue;
+		}
+		if (maths) {
+			out.push(l);
+			if (t.endsWith("$$")) maths = false;
+			continue;
+		}
+		if (t.startsWith("```") || t.startsWith("~~~")) {
+			fence = t.slice(0, 3);
+			out.push(l);
+			open = "";
+			continue;
+		}
+		if (t.startsWith("$$") && !t.slice(2).includes("$$")) {
+			maths = true;
+			out.push(l);
+			open = "";
+			continue;
+		}
+		if (!t) {
+			out.push(l);
+			open = "";
+			continue;
+		}
+		const hard = /( {2}|\\)$/.test(out.at(-1) ?? "");
+		if (APART.test(l)) {
+			if (open === "quote" && QUOTED.test(l) && !/^\s*>\s*([-*+]|\d+[.)]|#)/.test(l) && !hard) {
+				out[out.length - 1] += " " + l.replace(/^\s*>\s?/, "").trim();
+				continue;
+			}
+			out.push(l);
+			open = QUOTED.test(l) ? "quote" : "";
+			continue;
+		}
+		if (ITEM.test(l)) {
+			out.push(l);
+			open = "prose";
+			continue;
+		}
+		if (open === "prose" && !hard) {
+			out[out.length - 1] += " " + t;
+			continue;
+		}
+		out.push(l);
+		open = open === "" && /^( {4}|\t)/.test(l) ? "" : "prose";
+	}
+	return out.join("\n");
+}
+//#endregion
 //#region packages/core/src/search.ts
 const TOKEN = /[a-z0-9]+(?:['’][a-z]+)?/g;
 const STOP = new Set("a an and are as at be by for from has have in is it its of on or that the this to was were will with not but if then than so do does did can could should would into about".split(" "));
@@ -1606,6 +1717,8 @@ const KINDS$1 = [
 ];
 const SNAPSHOT_EVERY = 40;
 const SESSION_GAP_MS = 18e5;
+/** Your edits in a row are one moment, one layer of a note's history (M24, T), until 10 minutes pass with none. */
+const LAYER_IDLE_MS = 6e5;
 const hashText = (text) => sha256(text).slice(0, 16);
 /** A stamp for a new entry. `last` is the newest entry anywhere in the study base, which decides the session. */
 function stamp(who, device, last, now = Date.now()) {
@@ -1958,16 +2071,21 @@ function insertAt(note, line) {
 		before: prev + 1 < all.length ? anchorOf(all, prev + 1) : null
 	};
 }
-/** The journal as the timeline shows it. Your own edits in a row within a session are one moment; a proposal is
-*  one moment whose `what` says how it ended. Revisions are not moments. */
+/** The journal as the timeline shows it. Your own edits in a row within a session are one moment, until ten minutes
+*  pass with none (LAYER_IDLE_MS); a proposal is one moment whose `what` says how it ended. Revisions are not
+*  moments. */
 function moments(entries) {
 	const out = [], offered = /* @__PURE__ */ new Map(), asked = /* @__PURE__ */ new Map();
+	/** Lines a change put in and took out: a note's first entry, all of its text. */
+	const size = (c) => c.why === "start" || !c.hunks.length && c.before === null ? [(c.text ?? "").split("\n").length, 0] : [c.hunks.reduce((n, h) => n + h.ins.length, 0), c.hunks.reduce((n, h) => n + h.del.length, 0)];
 	for (const e of entries) {
 		const base = {
 			ids: [e.id],
 			at: e.at,
 			session: e.session,
-			who: e.who
+			who: e.who,
+			added: 0,
+			removed: 0
 		};
 		if (e.kind === "proposal") {
 			const m = {
@@ -1986,22 +2104,29 @@ function moments(entries) {
 			}
 		} else if (e.kind === "change") {
 			const m = e.proposal ? offered.get(e.proposal) : void 0;
+			const [added, removed] = size(e);
 			if (e.why === "keep" && m) {
 				m.what = "kept";
 				m.ids.push(e.id);
 				m.at = e.at;
+				m.added += added;
+				m.removed += removed;
 				continue;
 			}
 			const last = out[out.length - 1];
-			if (e.why === "edit" && last && last.what === "edited" && last.who === e.who && last.session === e.session) {
+			if (e.why === "edit" && last && last.what === "edited" && last.who === e.who && last.session === e.session && Date.parse(e.at) - Date.parse(last.at) <= 6e5) {
 				last.ids.push(e.id);
 				last.at = e.at;
+				last.added += added;
+				last.removed += removed;
 				continue;
 			}
 			const what = e.why === "start" ? e.who === "external" ? "found" : "wrote" : e.why === "edit" ? "edited" : e.why === "restore" ? "restored" : e.why === "keep" ? "kept" : e.why === "move" ? "moved" : "changed elsewhere";
 			out.push({
 				...base,
 				what,
+				added,
+				removed,
 				...e.asked ? { asked: e.asked } : {}
 			});
 		}
@@ -2096,8 +2221,8 @@ function checkId(id) {
 	if (!SAFE.test(id) || id.startsWith("/") || id.split("/").some((part) => part === ".." || part === "." || !part)) throw new Error(`not a note's id: ${JSON.stringify(id)}`);
 	return id;
 }
-/** Where a note's journal is. Drafts and Understanding notes are journalled under their `@draft/` and
-*  `@understanding/` ids. */
+/** Where a note's journal is. Drafts are journalled under their `@draft/` ids (and Understanding notes were, before
+*  M24, under `@understanding/`). */
 const journalPath = (noteId) => `journal/${checkId(noteId)}.jsonl`;
 /** This device's record of usage: what was cleared from Iso's context, and what each turn cost (record.ts). */
 const recordPath = (device) => `record/${checkId(device)}.jsonl`;
@@ -2159,10 +2284,54 @@ function mapModel(base) {
 		]);
 	}
 	const holdsFiles = /* @__PURE__ */ new Set();
-	for (const f of base.files) holdsFiles.add(f.includes("/") ? f.slice(0, f.lastIndexOf("/")) : "");
+	const parentOf = (p) => p.includes("/") ? p.slice(0, p.lastIndexOf("/")) : "";
+	for (const f of base.files) holdsFiles.add(parentOf(f));
+	const pictureFolder = (d) => {
+		const dd = base.directories.get(d);
+		const here = [...base.files].filter((f) => parentOf(f) === d);
+		return !!d && (!dd || !dd.concepts.length && !dd.children.length) && here.length > 0 && here.every((f) => citeKind(f) === "image");
+	};
+	const refOf = (f) => citeKind(f) === "image" && pictureFolder(parentOf(f)) ? parentOf(f) + "/" : f;
+	const fileEdges = /* @__PURE__ */ new Map();
+	for (const c of base.concepts.values()) for (const x of c.cites) {
+		if (x.broken || !base.files.has(x.target)) continue;
+		const ref = refOf(x.target);
+		if (!fileEdges.has(c.id)) fileEdges.set(c.id, /* @__PURE__ */ new Set());
+		fileEdges.get(c.id).add(ref);
+	}
+	const rankOf = /* @__PURE__ */ new Map();
+	for (const [from, refs] of fileEdges) for (const ref of refs) {
+		edges.push([
+			from,
+			ref,
+			STRENGTH.uses
+		]);
+		rankOf.set(ref, Math.max(rankOf.get(ref) ?? 0, order.get(from)?.depth ?? 0));
+	}
+	const filesIn = /* @__PURE__ */ new Map();
+	for (const f of [...base.files].sort()) {
+		const of = citeKind(f), ref = of ? refOf(f) : null;
+		if (!of || !ref) continue;
+		const at = ref.endsWith("/") ? parentOf(ref.slice(0, -1)) : parentOf(f);
+		const list = filesIn.get(at) ?? [];
+		if (!list.some((n) => n.ref === ref)) list.push({
+			kind: "concept",
+			id: "f:" + ref,
+			ref,
+			weight: 1,
+			rank: rankOf.get(ref) ?? 0,
+			of
+		});
+		filesIn.set(at, list);
+	}
 	const dir = (id) => {
 		const d = base.directories.get(id);
-		const children = [...[...d.children].sort().map(dir).filter((c) => c !== null), ...[...d.concepts].sort().map(note)];
+		if (id && pictureFolder(id)) return null;
+		const children = [
+			...[...d.children].sort().map(dir).filter((c) => c !== null),
+			...[...d.concepts].sort().map(note),
+			...filesIn.get(id) ?? []
+		];
 		return id && !children.length && (holdsFiles.has(id) || d.children.length) ? null : {
 			kind: "dir",
 			id: "d:" + id,
@@ -2175,6 +2344,8 @@ function mapModel(base) {
 		edges
 	};
 }
+/** Whether a map item is a file (M23), not a note: its ref has an extension, or is a folder of pictures. */
+const isFileRef = (ref) => ref.endsWith("/") || citeKind(ref) !== null;
 /** What lights when a note is chosen on the Atlas (refs, as the map model names notes). Where the base rates its
 *  links, only "requires" is followed back; where it rates none, every drawn link is. */
 function reach(model, note) {
@@ -2290,4 +2461,4 @@ function ago(at, now = Date.now()) {
 /** A moment is named by its last entry: the one the note is shown at. */
 const momentKey = (m) => m.ids.at(-1);
 //#endregion
-export { ABOUT, AGENTS_FILE, Bundle, CONFIG_FILE, Concept, DEVICE_RE, DONE, DiffError, Directory, FrontmatterError, GITATTRIBUTES, ID_RE, KINDS, LearnerError, MAX_EVENT_BYTES, OKF_VERSION, OWN_DIR, OWN_FORMAT, RATINGS, SESSION_GAP_MS, SETTINGS_FILE, SNAPSHOT_EVERY, STRENGTH, STUDY_DIR, STUDY_FORMAT, SearchIndex, StudyError, agentName, ago, anchorOf, applyHunks, authors, blocks, change, citeKind, cleanEvent, cmp, cmpTuple, codeTarget, contentHash, describe, diffLines, doneOf, dumpFrontmatter, editFrontmatter, embedFor, embedRefs, findAnchor, findPassage, floatRepr, frontmatterText, hashText, headings, idsOf, impliedLinks, insertAt, iso, journalLine, journalPath, kindOf, legacyId, linkRefs, linkTexts, locate, mapModel, mergeRecords, momentKey, moments, newId, noteIds, outwardLinks, pagerank, parseSettings, parseStudyConfig, parseYaml, passageOf, pending, pictureSize, pictureType, pyRepr, rating, reach, readJournal, readRecord, reconcile, recordPath, replay, round3, section, sessionMoments, settingsText, sha256, slugify, snippet, spliceText, splitFrontmatter, splitSource, stamp, strengthEdges, strip, stronglyConnected, text, textAt, toTime, tokenize, touched, wordAuthors, written };
+export { ABOUT, AGENTS_FILE, Bundle, CONFIG_FILE, Concept, DEVICE_RE, DONE, DiffError, Directory, FrontmatterError, GITATTRIBUTES, ID_RE, KINDS, LAYER_IDLE_MS, LearnerError, MAX_EVENT_BYTES, OKF_VERSION, OWN_DIR, OWN_FORMAT, RATINGS, SESSION_GAP_MS, SETTINGS_FILE, SNAPSHOT_EVERY, STRENGTH, STUDY_DIR, STUDY_FORMAT, SearchIndex, StudyError, agentName, ago, anchorOf, applyHunks, authors, blocks, change, citeKind, cleanEvent, cmp, cmpTuple, codeTarget, contentHash, describe, diffLines, doneOf, dumpFrontmatter, editFrontmatter, embedFor, embedRefs, fileType, findAnchor, findPassage, floatRepr, frontmatterText, hashText, headings, idsOf, impliedLinks, insertAt, isFileRef, iso, journalLine, journalPath, kindOf, legacyId, linkRefs, linkTexts, locate, mapModel, mergeRecords, momentKey, moments, newId, noteIds, outwardLinks, pagerank, parseSettings, parseStudyConfig, parseYaml, passageOf, pending, pictureSize, pictureType, pyRepr, rating, reach, readJournal, readRecord, reconcile, recordPath, replay, round3, section, sessionMoments, settingsText, sha256, slugify, snippet, spliceText, splitFrontmatter, splitSource, stamp, strengthEdges, strip, stronglyConnected, text, textAt, toTime, tokenize, touched, unwrap, wordAuthors, written };