isorite 0.3.0 → 0.5.0

package/README.md

@@ -6,10 +6,10 @@ Needs Node 24 or later. For the terminal, OpenCode 2: `npm i -g @opencode/cli`.
 
 ```sh
 npm install -g isorite
-isorite --tutorial          # the tutorial: the terminal's tour, and the browser's beside it, on an example base
+isorite guide               # the guide: a knowledge base about isorite, with things to do, in ~/isorite-guide
 ```
 
-On your own notes, a folder of Markdown files (empty is fine):
+On your own notes, a folder of Markdown files (`isorite new <notes>` starts one):
 
 ```sh
 isorite serve <notes>       # the browser, at http://localhost:4767/; set Iso's provider key behind the logo
@@ -21,6 +21,6 @@ Setting it up, at work too (a company's gateway, a client certificate, a proxy):
 When something fails, `isorite doctor` says what to change; `isorite doctor --report` gives it as one block, its
 secrets hidden, to paste to someone helping. For an AI helping you: `isorite help setup`.
 
-The tutorial in a browser alone: https://lachlanjs.github.io/isorite/
+The guide in a browser alone: https://lachlanjs.github.io/isorite/
 
 Source, and what is on disk: https://github.com/lachlanjs/isorite

package/dist/core.js

@@ -511,6 +511,8 @@ function outwardLinks(body) {
 }
 //#endregion
 //#region packages/core/src/bundle.ts
+/** The base's own instructions, for Iso and any other agent, at the top of the notes (M15). Not a note. */
+const AGENTS_FILE = "AGENTS.md";
 const OKF_VERSION = "0.2";
 const ARTIFACT = /\.html?$/i;
 const IMAGE = /\.(png|jpe?g|gif|svg|webp|avif)$/i;
@@ -668,7 +670,7 @@ var Bundle = class Bundle {
 				this.ensureDir(rel);
 				continue;
 			}
-			if (!rel.endsWith(".md")) {
+			if (!rel.endsWith(".md") || rel === "AGENTS.md") {
 				this.files.add(rel);
 				continue;
 			}
@@ -2214,4 +2216,4 @@ function ago(at, now = Date.now()) {
 /** A moment is named by its last entry: the one the note is shown at. */
 const momentKey = (m) => m.ids.at(-1);
 //#endregion
-export { ABOUT, Bundle, CONFIG_FILE, Concept, DEVICE_RE, DONE, DiffError, Directory, FrontmatterError, GITATTRIBUTES, ID_RE, KINDS, LearnerError, MAX_EVENT_BYTES, OKF_VERSION, OWN_DIR, OWN_FORMAT, RATINGS, SESSION_GAP_MS, SETTINGS_FILE, SNAPSHOT_EVERY, STRENGTH, STUDY_DIR, STUDY_FORMAT, SearchIndex, StudyError, agentName, ago, anchorOf, applyHunks, authors, blocks, change, citeKind, cleanEvent, cmp, cmpTuple, codeTarget, contentHash, describe, diffLines, doneOf, dumpFrontmatter, editFrontmatter, embedFor, embedRefs, findAnchor, findPassage, floatRepr, frontmatterText, hashText, headings, idsOf, impliedLinks, insertAt, iso, journalLine, journalPath, kindOf, legacyId, linkRefs, linkTexts, locate, mapModel, mergeRecords, momentKey, moments, newId, noteIds, outwardLinks, pagerank, parseSettings, parseStudyConfig, parseYaml, passageOf, pending, pyRepr, rating, reach, readJournal, readRecord, reconcile, recordPath, replay, round3, section, sessionMoments, settingsText, sha256, slugify, snippet, spliceText, splitFrontmatter, splitSource, stamp, strengthEdges, strip, stronglyConnected, text, textAt, toTime, tokenize, touched, wordAuthors, written };
+export { ABOUT, AGENTS_FILE, Bundle, CONFIG_FILE, Concept, DEVICE_RE, DONE, DiffError, Directory, FrontmatterError, GITATTRIBUTES, ID_RE, KINDS, LearnerError, MAX_EVENT_BYTES, OKF_VERSION, OWN_DIR, OWN_FORMAT, RATINGS, SESSION_GAP_MS, SETTINGS_FILE, SNAPSHOT_EVERY, STRENGTH, STUDY_DIR, STUDY_FORMAT, SearchIndex, StudyError, agentName, ago, anchorOf, applyHunks, authors, blocks, change, citeKind, cleanEvent, cmp, cmpTuple, codeTarget, contentHash, describe, diffLines, doneOf, dumpFrontmatter, editFrontmatter, embedFor, embedRefs, findAnchor, findPassage, floatRepr, frontmatterText, hashText, headings, idsOf, impliedLinks, insertAt, iso, journalLine, journalPath, kindOf, legacyId, linkRefs, linkTexts, locate, mapModel, mergeRecords, momentKey, moments, newId, noteIds, outwardLinks, pagerank, parseSettings, parseStudyConfig, parseYaml, passageOf, pending, pyRepr, rating, reach, readJournal, readRecord, reconcile, recordPath, replay, round3, section, sessionMoments, settingsText, sha256, slugify, snippet, spliceText, splitFrontmatter, splitSource, stamp, strengthEdges, strip, stronglyConnected, text, textAt, toTime, tokenize, touched, wordAuthors, written };

package/dist/embed-worker.js

@@ -3212,14 +3212,15 @@ var init_ort_wasm_bundle_min = __esmMin((() => {
 //#endregion
 //#region packages/server/src/embed-worker.ts
 const MAX_TOKENS = 512;
-const { dir, dim } = workerData;
+const { dir, dim, pooling } = workerData;
 const ort = await Promise.resolve().then(() => (init_ort_wasm_bundle_min(), ort_wasm_bundle_min_exports));
 const { Tokenizer } = await import("@huggingface/tokenizers");
 ort.env.wasm.numThreads = 1;
 const WASM = "ort-wasm-simd-threaded.wasm";
 const beside = join(dirname(fileURLToPath(import.meta.url)), WASM);
 ort.env.wasm.wasmBinary = readFileSync(existsSync(beside) ? beside : join(dirname(createRequire(import.meta.url).resolve("onnxruntime-web")), WASM));
-const tok = new Tokenizer(JSON.parse(readFileSync(join(dir, "tokenizer.json"), "utf8")), JSON.parse(readFileSync(join(dir, "tokenizer_config.json"), "utf8")));
+const config = join(dir, "tokenizer_config.json");
+const tok = new Tokenizer(JSON.parse(readFileSync(join(dir, "tokenizer.json"), "utf8")), existsSync(config) ? JSON.parse(readFileSync(config, "utf8")) : {});
 const externalData = readdirSync(dir).filter((f) => f.endsWith(".onnx_data")).map((path) => ({
 	path,
 	data: readFileSync(join(dir, path))
@@ -3238,8 +3239,12 @@ async function embed(text) {
 		attention_mask: big(ids.map(() => 1)),
 		...typed ? { token_type_ids: big(ids.map(() => 0)) } : {}
 	});
-	const state = out.last_hidden_state ?? Object.values(out)[0];
-	return Float32Array.from(state.data.subarray(0, dim));
+	const state = out.last_hidden_state ?? Object.values(out)[0], width = state.dims.at(-1), data = state.data;
+	if (width !== dim) throw new Error(`its vectors have ${width} numbers, not ${dim}`);
+	if (pooling === "cls" || state.dims.length < 3) return Float32Array.from(data.subarray(0, dim));
+	const n = data.length / dim, mean = new Float32Array(dim);
+	for (let t = 0; t < n; t++) for (let k = 0; k < dim; k++) mean[k] += data[t * dim + k] / n;
+	return mean;
 }
 parentPort.on("message", (m) => {
 	embed(m.text).then((vector) => parentPort.postMessage({