@mushinjs/plugin-knowledge 0.0.0-stage → 0.2.0-next.20261113103537

package/dist/search.js

@@ -0,0 +1,218 @@
+// src/search.ts
+/**
+* Hybrid search over the chunks of one knowledge base: BM25 over words (CJK text as pairs of
+* characters) and cosine similarity over embeddings, fused by reciprocal rank.
+*/
+const CJK = /[㐀-鿿豈-﫿]/;
+const WORD = /[\p{L}\p{N}]+/gu;
+/**
+* Words of a text for keyword search: Latin words and numbers as they are, runs of CJK characters
+* as overlapping pairs (`能发广告` → `能发 发广 广告`), a lone CJK character as itself. Pairs work
+* without a dictionary; SQLite's trigram tokenizer cannot match two-character words at all.
+*/
+function tokenize(text) {
+	const tokens = [];
+	for (const [word] of text.toLowerCase().normalize("NFKC").matchAll(WORD)) {
+		let latin = "";
+		let cjk = "";
+		const flush = () => {
+			if (latin) tokens.push(latin);
+			if (cjk.length === 1) tokens.push(cjk);
+			for (let index = 0; index < cjk.length - 1; index++) tokens.push(cjk.slice(index, index + 2));
+			latin = "";
+			cjk = "";
+		};
+		for (const char of word) if (CJK.test(char)) {
+			if (latin) flush();
+			cjk += char;
+		} else {
+			if (cjk) flush();
+			latin += char;
+		}
+		flush();
+	}
+	return tokens;
+}
+/** A vector scaled to length 1, so a dot product is the cosine similarity. */
+function normalize(vector) {
+	const result = Float32Array.from(vector);
+	let sum = 0;
+	for (const value of result) sum += value * value;
+	const norm = Math.sqrt(sum);
+	if (norm > 0) for (let index = 0; index < result.length; index++) result[index] /= norm;
+	return result;
+}
+function dot(a, b) {
+	let sum = 0;
+	const length = Math.min(a.length, b.length);
+	for (let index = 0; index < length; index++) sum += a[index] * b[index];
+	return sum;
+}
+const K1 = 1.2;
+const B = .75;
+/** Reciprocal rank fusion constant; 60 is the usual choice. */
+const RRF_K = 60;
+/** How many of each list take part in fusion. */
+const CANDIDATES = 50;
+var SearchIndex = class {
+	entries = /* @__PURE__ */ new Map();
+	df = /* @__PURE__ */ new Map();
+	totalLength = 0;
+	constructor(chunks = []) {
+		for (const chunk of chunks) this.add(chunk);
+	}
+	get size() {
+		return this.entries.size;
+	}
+	/** Chunks that have a vector. */
+	get embedded() {
+		let count = 0;
+		for (const entry of this.entries.values()) if (entry.chunk.vector) count++;
+		return count;
+	}
+	chunks() {
+		return [...this.entries.values()].map((entry) => entry.chunk);
+	}
+	get(id) {
+		return this.entries.get(id)?.chunk;
+	}
+	add(chunk) {
+		this.remove(chunk.id);
+		const terms = /* @__PURE__ */ new Map();
+		const tokens = tokenize(`${chunk.title}\n${chunk.text}`);
+		for (const token of tokens) terms.set(token, (terms.get(token) ?? 0) + 1);
+		for (const token of terms.keys()) this.df.set(token, (this.df.get(token) ?? 0) + 1);
+		this.entries.set(chunk.id, {
+			chunk,
+			terms,
+			length: tokens.length
+		});
+		this.totalLength += tokens.length;
+	}
+	remove(id) {
+		const entry = this.entries.get(id);
+		if (!entry) return;
+		for (const token of entry.terms.keys()) {
+			const count = (this.df.get(token) ?? 1) - 1;
+			if (count > 0) this.df.set(token, count);
+			else this.df.delete(token);
+		}
+		this.totalLength -= entry.length;
+		this.entries.delete(id);
+	}
+	idf(token) {
+		const n = this.entries.size;
+		const df = this.df.get(token) ?? 0;
+		return Math.log(1 + (n - df + .5) / (df + .5));
+	}
+	/** Searches by words, and by meaning when a normalized question vector is given. */
+	search(text, vector, options = {}) {
+		const topK = options.topK ?? 5;
+		const minScore = options.minScore ?? 0;
+		const query = [...new Set(tokenize(text))];
+		const average = this.entries.size ? this.totalLength / this.entries.size : 0;
+		const keyword = [];
+		const vectorList = [];
+		for (const entry of this.entries.values()) {
+			let bm25 = 0;
+			let matched = 0;
+			for (const token of query) {
+				const tf = entry.terms.get(token);
+				if (!tf) continue;
+				matched++;
+				const norm = .25 + B * (average ? entry.length / average : 1);
+				bm25 += this.idf(token) * (tf * 2.2) / (tf + K1 * norm);
+			}
+			if (bm25 > 0) keyword.push({
+				entry,
+				bm25,
+				coverage: query.length ? matched / query.length : 0
+			});
+			if (vector && entry.chunk.vector && entry.chunk.vector.length === vector.length) vectorList.push({
+				entry,
+				similarity: dot(vector, entry.chunk.vector)
+			});
+		}
+		keyword.sort((a, b) => b.bm25 - a.bm25);
+		vectorList.sort((a, b) => b.similarity - a.similarity);
+		const hits = /* @__PURE__ */ new Map();
+		const hitOf = (entry) => {
+			let hit = hits.get(entry.chunk.id);
+			if (!hit) {
+				const { id, doc, title, text } = entry.chunk;
+				hit = {
+					id,
+					doc,
+					title,
+					text,
+					score: 0,
+					coverage: 0,
+					bm25: 0,
+					ranks: {},
+					fused: 0
+				};
+				hits.set(id, hit);
+			}
+			return hit;
+		};
+		keyword.slice(0, CANDIDATES).forEach((item, index) => {
+			const hit = hitOf(item.entry);
+			hit.bm25 = item.bm25;
+			hit.coverage = item.coverage;
+			hit.ranks.keyword = index + 1;
+			hit.fused += 1 / (RRF_K + index + 1);
+		});
+		vectorList.slice(0, CANDIDATES).forEach((item, index) => {
+			const hit = hitOf(item.entry);
+			hit.similarity = item.similarity;
+			hit.ranks.vector = index + 1;
+			hit.fused += 1 / (RRF_K + index + 1);
+		});
+		for (const hit of hits.values()) hit.score = Math.max(hit.similarity ?? 0, hit.coverage);
+		return [...hits.values()].filter((hit) => hit.score >= minScore).sort((a, b) => b.fused - a.fused || b.score - a.score).slice(0, topK);
+	}
+	/**
+	* Pairs of similar chunks, for the similarity map: each chunk's closest `neighbors` above
+	* `minSimilarity`. Uses vectors when chunks have them, shared words otherwise.
+	*/
+	links(options = {}) {
+		const neighbors = options.neighbors ?? 3;
+		const minSimilarity = options.minSimilarity ?? .6;
+		const entries = [...this.entries.values()].slice(0, options.limit ?? 500);
+		const byVector = entries.length > 0 && entries.every((entry) => entry.chunk.vector);
+		const similarity = (a, b) => {
+			if (byVector) return dot(a.chunk.vector, b.chunk.vector);
+			let shared = 0;
+			for (const token of a.terms.keys()) if (b.terms.has(token)) shared++;
+			const union = a.terms.size + b.terms.size - shared;
+			return union ? shared / union : 0;
+		};
+		const edges = /* @__PURE__ */ new Map();
+		for (const a of entries) {
+			const closest = entries.filter((b) => b !== a).map((b) => ({
+				b,
+				value: similarity(a, b)
+			})).filter((item) => item.value >= minSimilarity).sort((x, y) => y.value - x.value).slice(0, neighbors);
+			for (const { b, value } of closest) {
+				const [source, target] = a.chunk.id < b.chunk.id ? [a.chunk.id, b.chunk.id] : [b.chunk.id, a.chunk.id];
+				edges.set(`${source}|${target}`, {
+					source,
+					target,
+					similarity: Math.round(value * 1e3) / 1e3
+				});
+			}
+		}
+		return {
+			nodes: entries.map((entry) => entry.chunk),
+			edges: [...edges.values()],
+			byVector
+		};
+	}
+};
+/** Hits as numbered passages for a model: `[1] title` and the text. */
+function passages(hits) {
+	return hits.map((hit, index) => `[${index + 1}] ${hit.title || hit.docTitle || ""}\n${hit.text}`.trim()).join("\n\n");
+}
+export { SearchIndex, dot, normalize, passages, tokenize };
+
+//# sourceMappingURL=search.js.map

package/dist/search.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search.js","names":[],"sources":["../src/search.ts"],"sourcesContent":["/**\n * Hybrid search over the chunks of one knowledge base: BM25 over words (CJK text as pairs of\n * characters) and cosine similarity over embeddings, fused by reciprocal rank.\n */\n\nconst CJK = /[㐀-鿿豈-﫿]/\nconst WORD = /[\\p{L}\\p{N}]+/gu\n\n/**\n * Words of a text for keyword search: Latin words and numbers as they are, runs of CJK characters\n * as overlapping pairs (`能发广告` → `能发 发广 广告`), a lone CJK character as itself. Pairs work\n * without a dictionary; SQLite's trigram tokenizer cannot match two-character words at all.\n */\nexport function tokenize(text: string): string[] {\n  const tokens: string[] = []\n  for (const [word] of text.toLowerCase().normalize('NFKC').matchAll(WORD)) {\n    let latin = ''\n    let cjk = ''\n    const flush = () => {\n      if (latin) tokens.push(latin)\n      if (cjk.length === 1) tokens.push(cjk)\n      for (let index = 0; index < cjk.length - 1; index++) tokens.push(cjk.slice(index, index + 2))\n      latin = ''\n      cjk = ''\n    }\n    for (const char of word) {\n      if (CJK.test(char)) {\n        if (latin) flush()\n        cjk += char\n      } else {\n        if (cjk) flush()\n        latin += char\n      }\n    }\n    flush()\n  }\n  return tokens\n}\n\n/** A vector scaled to length 1, so a dot product is the cosine similarity. */\nexport function normalize(vector: ArrayLike<number>): Float32Array {\n  const result = Float32Array.from(vector)\n  let sum = 0\n  for (const value of result) sum += value * value\n  const norm = Math.sqrt(sum)\n  if (norm > 0) for (let index = 0; index < result.length; index++) result[index] /= norm\n  return result\n}\n\nexport function dot(a: Float32Array, b: Float32Array) {\n  let sum = 0\n  const length = Math.min(a.length, b.length)\n  for (let index = 0; index < length; index++) sum += a[index] * b[index]\n  return sum\n}\n\nexport interface IndexedChunk {\n  id: string\n  doc: string\n  title: string\n  text: string\n  /** Length 1; missing until the chunk is embedded. */\n  vector?: Float32Array\n}\n\nexport interface SearchOptions {\n  topK?: number\n  /** Hits whose score (the better of similarity and keyword coverage) is lower are dropped. */\n  minScore?: number\n}\n\nexport interface SearchHit {\n  id: string\n  doc: string\n  title: string\n  text: string\n  /** What `minScore` compares: the better of `similarity` and `coverage`, 0 to 1. */\n  score: number\n  /** Cosine similarity to the question, when both are embedded. */\n  similarity?: number\n  /** Share of the question's words found in the chunk, 0 to 1. */\n  coverage: number\n  /** BM25 score; 0 when no word matched. */\n  bm25: number\n  /** Place in each list (from 1), and the fused order that decides the result. */\n  ranks: { keyword?: number; vector?: number }\n  fused: number\n}\n\nconst K1 = 1.2\nconst B = 0.75\n/** Reciprocal rank fusion constant; 60 is the usual choice. */\nconst RRF_K = 60\n/** How many of each list take part in fusion. */\nconst CANDIDATES = 50\n\ninterface Entry {\n  chunk: IndexedChunk\n  terms: Map<string, number>\n  length: number\n}\n\nexport class SearchIndex {\n  private entries = new Map<string, Entry>()\n  private df = new Map<string, number>()\n  private totalLength = 0\n\n  constructor(chunks: IndexedChunk[] = []) {\n    for (const chunk of chunks) this.add(chunk)\n  }\n\n  get size() {\n    return this.entries.size\n  }\n\n  /** Chunks that have a vector. */\n  get embedded() {\n    let count = 0\n    for (const entry of this.entries.values()) if (entry.chunk.vector) count++\n    return count\n  }\n\n  chunks() {\n    return [...this.entries.values()].map((entry) => entry.chunk)\n  }\n\n  get(id: string) {\n    return this.entries.get(id)?.chunk\n  }\n\n  add(chunk: IndexedChunk) {\n    this.remove(chunk.id)\n    const terms = new Map<string, number>()\n    const tokens = tokenize(`${chunk.title}\\n${chunk.text}`)\n    for (const token of tokens) terms.set(token, (terms.get(token) ?? 0) + 1)\n    for (const token of terms.keys()) this.df.set(token, (this.df.get(token) ?? 0) + 1)\n    this.entries.set(chunk.id, { chunk, terms, length: tokens.length })\n    this.totalLength += tokens.length\n  }\n\n  remove(id: string) {\n    const entry = this.entries.get(id)\n    if (!entry) return\n    for (const token of entry.terms.keys()) {\n      const count = (this.df.get(token) ?? 1) - 1\n      if (count > 0) this.df.set(token, count)\n      else this.df.delete(token)\n    }\n    this.totalLength -= entry.length\n    this.entries.delete(id)\n  }\n\n  private idf(token: string) {\n    const n = this.entries.size\n    const df = this.df.get(token) ?? 0\n    return Math.log(1 + (n - df + 0.5) / (df + 0.5))\n  }\n\n  /** Searches by words, and by meaning when a normalized question vector is given. */\n  search(text: string, vector: Float32Array | undefined, options: SearchOptions = {}): SearchHit[] {\n    const topK = options.topK ?? 5\n    const minScore = options.minScore ?? 0\n    const query = [...new Set(tokenize(text))]\n    const average = this.entries.size ? this.totalLength / this.entries.size : 0\n\n    const keyword: { entry: Entry; bm25: number; coverage: number }[] = []\n    const vectorList: { entry: Entry; similarity: number }[] = []\n    for (const entry of this.entries.values()) {\n      let bm25 = 0\n      let matched = 0\n      for (const token of query) {\n        const tf = entry.terms.get(token)\n        if (!tf) continue\n        matched++\n        const norm = 1 - B + B * (average ? entry.length / average : 1)\n        bm25 += this.idf(token) * (tf * (K1 + 1)) / (tf + K1 * norm)\n      }\n      if (bm25 > 0) keyword.push({ entry, bm25, coverage: query.length ? matched / query.length : 0 })\n      if (vector && entry.chunk.vector && entry.chunk.vector.length === vector.length) {\n        vectorList.push({ entry, similarity: dot(vector, entry.chunk.vector) })\n      }\n    }\n    keyword.sort((a, b) => b.bm25 - a.bm25)\n    vectorList.sort((a, b) => b.similarity - a.similarity)\n\n    const hits = new Map<string, SearchHit>()\n    const hitOf = (entry: Entry) => {\n      let hit = hits.get(entry.chunk.id)\n      if (!hit) {\n        const { id, doc, title, text } = entry.chunk\n        hit = { id, doc, title, text, score: 0, coverage: 0, bm25: 0, ranks: {}, fused: 0 }\n        hits.set(id, hit)\n      }\n      return hit\n    }\n    keyword.slice(0, CANDIDATES).forEach((item, index) => {\n      const hit = hitOf(item.entry)\n      hit.bm25 = item.bm25\n      hit.coverage = item.coverage\n      hit.ranks.keyword = index + 1\n      hit.fused += 1 / (RRF_K + index + 1)\n    })\n    vectorList.slice(0, CANDIDATES).forEach((item, index) => {\n      const hit = hitOf(item.entry)\n      hit.similarity = item.similarity\n      hit.ranks.vector = index + 1\n      hit.fused += 1 / (RRF_K + index + 1)\n    })\n    for (const hit of hits.values()) hit.score = Math.max(hit.similarity ?? 0, hit.coverage)\n    return [...hits.values()]\n      .filter((hit) => hit.score >= minScore)\n      .sort((a, b) => b.fused - a.fused || b.score - a.score)\n      .slice(0, topK)\n  }\n\n  /**\n   * Pairs of similar chunks, for the similarity map: each chunk's closest `neighbors` above\n   * `minSimilarity`. Uses vectors when chunks have them, shared words otherwise.\n   */\n  links(options: { neighbors?: number; minSimilarity?: number; limit?: number } = {}) {\n    const neighbors = options.neighbors ?? 3\n    const minSimilarity = options.minSimilarity ?? 0.6\n    const entries = [...this.entries.values()].slice(0, options.limit ?? 500)\n    const byVector = entries.length > 0 && entries.every((entry) => entry.chunk.vector)\n    const similarity = (a: Entry, b: Entry) => {\n      if (byVector) return dot(a.chunk.vector!, b.chunk.vector!)\n      let shared = 0\n      for (const token of a.terms.keys()) if (b.terms.has(token)) shared++\n      const union = a.terms.size + b.terms.size - shared\n      return union ? shared / union : 0\n    }\n    const edges = new Map<string, { source: string; target: string; similarity: number }>()\n    for (const a of entries) {\n      const closest = entries\n        .filter((b) => b !== a)\n        .map((b) => ({ b, value: similarity(a, b) }))\n        .filter((item) => item.value >= minSimilarity)\n        .sort((x, y) => y.value - x.value)\n        .slice(0, neighbors)\n      for (const { b, value } of closest) {\n        const [source, target] = a.chunk.id < b.chunk.id ? [a.chunk.id, b.chunk.id] : [b.chunk.id, a.chunk.id]\n        edges.set(`${source}|${target}`, { source, target, similarity: Math.round(value * 1000) / 1000 })\n      }\n    }\n    return { nodes: entries.map((entry) => entry.chunk), edges: [...edges.values()], byVector }\n  }\n}\n\n/** Hits as numbered passages for a model: `[1] title` and the text. */\nexport function passages(hits: { title: string; text: string; docTitle?: string }[]) {\n  return hits.map((hit, index) => `[${index + 1}] ${hit.title || hit.docTitle || ''}\\n${hit.text}`.trim()).join('\\n\\n')\n}\n"],"mappings":";;;;;AAKA,MAAM,MAAM;AACZ,MAAM,OAAO;;;;;;AAOb,SAAgB,SAAS,MAAwB;CAC/C,MAAM,SAAmB,CAAC;CAC1B,KAAK,MAAM,CAAC,SAAS,KAAK,YAAY,CAAC,CAAC,UAAU,MAAM,CAAC,CAAC,SAAS,IAAI,GAAG;EACxE,IAAI,QAAQ;EACZ,IAAI,MAAM;EACV,MAAM,cAAc;GAClB,IAAI,OAAO,OAAO,KAAK,KAAK;GAC5B,IAAI,IAAI,WAAW,GAAG,OAAO,KAAK,GAAG;GACrC,KAAK,IAAI,QAAQ,GAAG,QAAQ,IAAI,SAAS,GAAG,SAAS,OAAO,KAAK,IAAI,MAAM,OAAO,QAAQ,CAAC,CAAC;GAC5F,QAAQ;GACR,MAAM;EACR;EACA,KAAK,MAAM,QAAQ,MACjB,IAAI,IAAI,KAAK,IAAI,GAAG;GAClB,IAAI,OAAO,MAAM;GACjB,OAAO;EACT,OAAO;GACL,IAAI,KAAK,MAAM;GACf,SAAS;EACX;EAEF,MAAM;CACR;CACA,OAAO;AACT;;AAGA,SAAgB,UAAU,QAAyC;CACjE,MAAM,SAAS,aAAa,KAAK,MAAM;CACvC,IAAI,MAAM;CACV,KAAK,MAAM,SAAS,QAAQ,OAAO,QAAQ;CAC3C,MAAM,OAAO,KAAK,KAAK,GAAG;CAC1B,IAAI,OAAO,GAAG,KAAK,IAAI,QAAQ,GAAG,QAAQ,OAAO,QAAQ,SAAS,OAAO,UAAU;CACnF,OAAO;AACT;AAEA,SAAgB,IAAI,GAAiB,GAAiB;CACpD,IAAI,MAAM;CACV,MAAM,SAAS,KAAK,IAAI,EAAE,QAAQ,EAAE,MAAM;CAC1C,KAAK,IAAI,QAAQ,GAAG,QAAQ,QAAQ,SAAS,OAAO,EAAE,SAAS,EAAE;CACjE,OAAO;AACT;AAmCA,MAAM,KAAK;AACX,MAAM,IAAI;;AAEV,MAAM,QAAQ;;AAEd,MAAM,aAAa;AAQnB,IAAa,cAAb,MAAyB;CACvB,0BAAkB,IAAI,IAAmB;CACzC,qBAAa,IAAI,IAAoB;CACrC,cAAsB;CAEtB,YAAY,SAAyB,CAAC,GAAG;EACvC,KAAK,MAAM,SAAS,QAAQ,KAAK,IAAI,KAAK;CAC5C;CAEA,IAAI,OAAO;EACT,OAAO,KAAK,QAAQ;CACtB;;CAGA,IAAI,WAAW;EACb,IAAI,QAAQ;EACZ,KAAK,MAAM,SAAS,KAAK,QAAQ,OAAO,GAAG,IAAI,MAAM,MAAM,QAAQ;EACnE,OAAO;CACT;CAEA,SAAS;EACP,OAAO,CAAC,GAAG,KAAK,QAAQ,OAAO,CAAC,CAAC,CAAC,KAAK,UAAU,MAAM,KAAK;CAC9D;CAEA,IAAI,IAAY;EACd,OAAO,KAAK,QAAQ,IAAI,EAAE,CAAC,EAAE;CAC/B;CAEA,IAAI,OAAqB;EACvB,KAAK,OAAO,MAAM,EAAE;EACpB,MAAM,wBAAQ,IAAI,IAAoB;EACtC,MAAM,SAAS,SAAS,GAAG,MAAM,MAAM,IAAI,MAAM,MAAM;EACvD,KAAK,MAAM,SAAS,QAAQ,MAAM,IAAI,QAAQ,MAAM,IAAI,KAAK,KAAK,KAAK,CAAC;EACxE,KAAK,MAAM,SAAS,MAAM,KAAK,GAAG,KAAK,GAAG,IAAI,QAAQ,KAAK,GAAG,IAAI,KAAK,KAAK,KAAK,CAAC;EAClF,KAAK,QAAQ,IAAI,MAAM,IAAI;GAAE;GAAO;GAAO,QAAQ,OAAO;EAAO,CAAC;EAClE,KAAK,eAAe,OAAO;CAC7B;CAEA,OAAO,IAAY;EACjB,MAAM,QAAQ,KAAK,QAAQ,IAAI,EAAE;EACjC,IAAI,CAAC,OAAO;EACZ,KAAK,MAAM,SAAS,MAAM,MAAM,KAAK,GAAG;GACtC,MAAM,SAAS,KAAK,GAAG,IAAI,KAAK,KAAK,KAAK;GAC1C,IAAI,QAAQ,GAAG,KAAK,GAAG,IAAI,OAAO,KAAK;QAClC,KAAK,GAAG,OAAO,KAAK;EAC3B;EACA,KAAK,eAAe,MAAM;EAC1B,KAAK,QAAQ,OAAO,EAAE;CACxB;CAEA,IAAY,OAAe;EACzB,MAAM,IAAI,KAAK,QAAQ;EACvB,MAAM,KAAK,KAAK,GAAG,IAAI,KAAK,KAAK;EACjC,OAAO,KAAK,IAAI,KAAK,IAAI,KAAK,OAAQ,KAAK,GAAI;CACjD;;CAGA,OAAO,MAAc,QAAkC,UAAyB,CAAC,GAAgB;EAC/F,MAAM,OAAO,QAAQ,QAAQ;EAC7B,MAAM,WAAW,QAAQ,YAAY;EACrC,MAAM,QAAQ,CAAC,GAAG,IAAI,IAAI,SAAS,IAAI,CAAC,CAAC;EACzC,MAAM,UAAU,KAAK,QAAQ,OAAO,KAAK,cAAc,KAAK,QAAQ,OAAO;EAE3E,MAAM,UAA8D,CAAC;EACrE,MAAM,aAAqD,CAAC;EAC5D,KAAK,MAAM,SAAS,KAAK,QAAQ,OAAO,GAAG;GACzC,IAAI,OAAO;GACX,IAAI,UAAU;GACd,KAAK,MAAM,SAAS,OAAO;IACzB,MAAM,KAAK,MAAM,MAAM,IAAI,KAAK;IAChC,IAAI,CAAC,IAAI;IACT;IACA,MAAM,OAAO,MAAQ,KAAK,UAAU,MAAM,SAAS,UAAU;IAC7D,QAAQ,KAAK,IAAI,KAAK,KAAK,KAAM,QAAY,KAAK,KAAK;GACzD;GACA,IAAI,OAAO,GAAG,QAAQ,KAAK;IAAE;IAAO;IAAM,UAAU,MAAM,SAAS,UAAU,MAAM,SAAS;GAAE,CAAC;GAC/F,IAAI,UAAU,MAAM,MAAM,UAAU,MAAM,MAAM,OAAO,WAAW,OAAO,QACvE,WAAW,KAAK;IAAE;IAAO,YAAY,IAAI,QAAQ,MAAM,MAAM,MAAM;GAAE,CAAC;EAE1E;EACA,QAAQ,MAAM,GAAG,MAAM,EAAE,OAAO,EAAE,IAAI;EACtC,WAAW,MAAM,GAAG,MAAM,EAAE,aAAa,EAAE,UAAU;EAErD,MAAM,uBAAO,IAAI,IAAuB;EACxC,MAAM,SAAS,UAAiB;GAC9B,IAAI,MAAM,KAAK,IAAI,MAAM,MAAM,EAAE;GACjC,IAAI,CAAC,KAAK;IACR,MAAM,EAAE,IAAI,KAAK,OAAO,SAAS,MAAM;IACvC,MAAM;KAAE;KAAI;KAAK;KAAO;KAAM,OAAO;KAAG,UAAU;KAAG,MAAM;KAAG,OAAO,CAAC;KAAG,OAAO;IAAE;IAClF,KAAK,IAAI,IAAI,GAAG;GAClB;GACA,OAAO;EACT;EACA,QAAQ,MAAM,GAAG,UAAU,CAAC,CAAC,SAAS,MAAM,UAAU;GACpD,MAAM,MAAM,MAAM,KAAK,KAAK;GAC5B,IAAI,OAAO,KAAK;GAChB,IAAI,WAAW,KAAK;GACpB,IAAI,MAAM,UAAU,QAAQ;GAC5B,IAAI,SAAS,KAAK,QAAQ,QAAQ;EACpC,CAAC;EACD,WAAW,MAAM,GAAG,UAAU,CAAC,CAAC,SAAS,MAAM,UAAU;GACvD,MAAM,MAAM,MAAM,KAAK,KAAK;GAC5B,IAAI,aAAa,KAAK;GACtB,IAAI,MAAM,SAAS,QAAQ;GAC3B,IAAI,SAAS,KAAK,QAAQ,QAAQ;EACpC,CAAC;EACD,KAAK,MAAM,OAAO,KAAK,OAAO,GAAG,IAAI,QAAQ,KAAK,IAAI,IAAI,cAAc,GAAG,IAAI,QAAQ;EACvF,OAAO,CAAC,GAAG,KAAK,OAAO,CAAC,CAAC,CACtB,QAAQ,QAAQ,IAAI,SAAS,QAAQ,CAAC,CACtC,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,CACtD,MAAM,GAAG,IAAI;CAClB;;;;;CAMA,MAAM,UAA0E,CAAC,GAAG;EAClF,MAAM,YAAY,QAAQ,aAAa;EACvC,MAAM,gBAAgB,QAAQ,iBAAiB;EAC/C,MAAM,UAAU,CAAC,GAAG,KAAK,QAAQ,OAAO,CAAC,CAAC,CAAC,MAAM,GAAG,QAAQ,SAAS,GAAG;EACxE,MAAM,WAAW,QAAQ,SAAS,KAAK,QAAQ,OAAO,UAAU,MAAM,MAAM,MAAM;EAClF,MAAM,cAAc,GAAU,MAAa;GACzC,IAAI,UAAU,OAAO,IAAI,EAAE,MAAM,QAAS,EAAE,MAAM,MAAO;GACzD,IAAI,SAAS;GACb,KAAK,MAAM,SAAS,EAAE,MAAM,KAAK,GAAG,IAAI,EAAE,MAAM,IAAI,KAAK,GAAG;GAC5D,MAAM,QAAQ,EAAE,MAAM,OAAO,EAAE,MAAM,OAAO;GAC5C,OAAO,QAAQ,SAAS,QAAQ;EAClC;EACA,MAAM,wBAAQ,IAAI,IAAoE;EACtF,KAAK,MAAM,KAAK,SAAS;GACvB,MAAM,UAAU,QACb,QAAQ,MAAM,MAAM,CAAC,CAAC,CACtB,KAAK,OAAO;IAAE;IAAG,OAAO,WAAW,GAAG,CAAC;GAAE,EAAE,CAAC,CAC5C,QAAQ,SAAS,KAAK,SAAS,aAAa,CAAC,CAC7C,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,CACjC,MAAM,GAAG,SAAS;GACrB,KAAK,MAAM,EAAE,GAAG,WAAW,SAAS;IAClC,MAAM,CAAC,QAAQ,UAAU,EAAE,MAAM,KAAK,EAAE,MAAM,KAAK,CAAC,EAAE,MAAM,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,IAAI,EAAE,MAAM,EAAE;IACrG,MAAM,IAAI,GAAG,OAAO,GAAG,UAAU;KAAE;KAAQ;KAAQ,YAAY,KAAK,MAAM,QAAQ,GAAI,IAAI;IAAK,CAAC;GAClG;EACF;EACA,OAAO;GAAE,OAAO,QAAQ,KAAK,UAAU,MAAM,KAAK;GAAG,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC;GAAG;EAAS;CAC5F;AACF;;AAGA,SAAgB,SAAS,MAA4D;CACnF,OAAO,KAAK,KAAK,KAAK,UAAU,IAAI,QAAQ,EAAE,IAAI,IAAI,SAAS,IAAI,YAAY,GAAG,IAAI,IAAI,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,MAAM;AACtH"}

package/dist/text.d.ts

@@ -0,0 +1,32 @@
+/** Document formats a knowledge base reads. PDF and DOCX need optional packages. */
+export declare const FORMATS: readonly ["md", "txt", "html", "pdf", "docx"];
+export type DocFormat = typeof FORMATS[number];
+/** The format of a file by its name, or nothing for one the knowledge base cannot read. */
+export declare function formatOf(name: string): DocFormat | undefined;
+/** Readable text of an HTML page, with headings kept as Markdown headings so chunks know their section. */
+export declare function htmlToText(html: string): {
+  title?: string;
+  text: string;
+};
+/** The text of a document. */
+export declare function extractText(format: DocFormat, data: Uint8Array): Promise<{
+  title?: string;
+  text: string;
+}>;
+export interface ChunkOptions {
+  /** Most characters in a chunk. */
+  size: number;
+  /** Characters a chunk repeats from the end of the one before it, within a section. */
+  overlap: number;
+}
+export interface TextChunk {
+  /** The headings the chunk sits under, such as `群规 > 广告`. */
+  title: string;
+  text: string;
+}
+/**
+ * Splits a document into chunks: by Markdown headings first (each chunk keeps its heading path
+ * as its title), then by blank lines, then long paragraphs by sentence. Paragraphs are joined
+ * while they fit in `size`; the next chunk of a section starts with the last `overlap` characters.
+ */
+export declare function chunkText(text: string, options: ChunkOptions, docTitle?: string): TextChunk[];

package/dist/text.js

@@ -0,0 +1,183 @@
+import { MushinError } from "@mushinjs/core";
+// src/text.ts
+/** Document formats a knowledge base reads. PDF and DOCX need optional packages. */
+const FORMATS = [
+	"md",
+	"txt",
+	"html",
+	"pdf",
+	"docx"
+];
+/** The format of a file by its name, or nothing for one the knowledge base cannot read. */
+function formatOf(name) {
+	switch (/\.([a-z0-9]+)$/i.exec(name)?.[1]?.toLowerCase()) {
+		case "md":
+		case "markdown":
+		case "mdx": return "md";
+		case "txt":
+		case "text":
+		case "log":
+		case "csv":
+		case "json":
+		case "yaml":
+		case "yml": return "txt";
+		case "html":
+		case "htm": return "html";
+		case "pdf": return "pdf";
+		case "docx": return "docx";
+	}
+}
+const ENTITIES = {
+	amp: "&",
+	lt: "<",
+	gt: ">",
+	quot: "\"",
+	apos: "'",
+	nbsp: " "
+};
+function decodeEntities(text) {
+	return text.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (whole, name) => {
+		if (name[0] === "#") {
+			const code = name[1].toLowerCase() === "x" ? Number.parseInt(name.slice(2), 16) : Number(name.slice(1));
+			return Number.isFinite(code) ? String.fromCodePoint(code) : whole;
+		}
+		return ENTITIES[name.toLowerCase()] ?? whole;
+	});
+}
+/** Readable text of an HTML page, with headings kept as Markdown headings so chunks know their section. */
+function htmlToText(html) {
+	const title = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(html)?.[1];
+	let body = /<body[^>]*>([\s\S]*)<\/body>/i.exec(html)?.[1] ?? html;
+	body = body.replace(/<!--[\s\S]*?-->/g, "").replace(/<(script|style|noscript|svg|template|nav|footer|header|aside|form)\b[\s\S]*?<\/\1>/gi, "").replace(/<h([1-6])[^>]*>([\s\S]*?)<\/h\1>/gi, (_, level, inner) => `\n\n${"#".repeat(Number(level))} ${inner.replace(/<[^>]+>/g, "").trim()}\n\n`).replace(/<li[^>]*>/gi, "\n- ").replace(/<br\s*\/?>/gi, "\n").replace(/<\/(p|div|section|article|ul|ol|table|tr|blockquote|pre|h[1-6])>/gi, "\n\n").replace(/<\/t[dh]>/gi, "	").replace(/<[^>]+>/g, "");
+	const text = decodeEntities(body).split("\n").map((line) => line.replace(/[ \t ]+/g, " ").trim()).join("\n").replace(/\n{3,}/g, "\n\n").trim();
+	return {
+		...title ? { title: decodeEntities(title).trim() } : {},
+		text
+	};
+}
+/** Imports a package the knowledge base only needs for some formats. */
+async function optional(name, format) {
+	try {
+		return await import(name);
+	} catch {
+		throw new MushinError("unavailable", `Reading ${format} files needs the "${name}" package. Install it in the Mushin.js folder with: pnpm add -w ${name}`);
+	}
+}
+/** Text of a PDF's pages; scanned PDFs (images only) have none. */
+async function pdfText(data) {
+	const pdf = await (await optional("pdfjs-dist/legacy/build/pdf.mjs", "PDF")).getDocument({
+		data,
+		useSystemFonts: true,
+		isEvalSupported: false
+	}).promise;
+	const pages = [];
+	for (let number = 1; number <= pdf.numPages; number++) {
+		const content = await (await pdf.getPage(number)).getTextContent();
+		let line = "";
+		const lines = [];
+		for (const item of content.items) {
+			line += item.str ?? "";
+			if (item.hasEOL) {
+				lines.push(line);
+				line = "";
+			}
+		}
+		if (line) lines.push(line);
+		pages.push(lines.join("\n"));
+	}
+	await pdf.destroy?.();
+	const text = pages.join("\n\n").trim();
+	if (!text) throw new MushinError("invalid_input", "This PDF has no text to read; scanned PDFs need OCR, which is not supported");
+	return text;
+}
+async function docxText(data) {
+	const mammoth = await optional("mammoth", "DOCX");
+	return htmlToText((await (mammoth.default ?? mammoth).convertToHtml({ buffer: Buffer.from(data) })).value).text;
+}
+/** The text of a document. */
+async function extractText(format, data) {
+	switch (format) {
+		case "md":
+		case "txt": return { text: new TextDecoder().decode(data).replace(/^/, "").replace(/\r\n?/g, "\n") };
+		case "html": return htmlToText(new TextDecoder().decode(data));
+		case "pdf": return { text: await pdfText(data) };
+		case "docx": return { text: await docxText(data) };
+	}
+}
+/** Pieces of at most `size` characters, cut after sentence ends where possible. */
+function splitLong(text, size) {
+	if (text.length <= size) return [text];
+	const sentences = text.match(/[^。！？!?；;\n.]+[。！？!?；;\n.]*|[。！？!?；;\n.]+/g) ?? [text];
+	const pieces = [];
+	let current = "";
+	for (const sentence of sentences) {
+		if (current && current.length + sentence.length > size) {
+			pieces.push(current);
+			current = "";
+		}
+		if (sentence.length > size) {
+			for (let index = 0; index < sentence.length; index += size) pieces.push(sentence.slice(index, index + size));
+			continue;
+		}
+		current += sentence;
+	}
+	if (current) pieces.push(current);
+	return pieces.map((piece) => piece.trim()).filter(Boolean);
+}
+/**
+* Splits a document into chunks: by Markdown headings first (each chunk keeps its heading path
+* as its title), then by blank lines, then long paragraphs by sentence. Paragraphs are joined
+* while they fit in `size`; the next chunk of a section starts with the last `overlap` characters.
+*/
+function chunkText(text, options, docTitle = "") {
+	const size = Math.max(50, options.size);
+	const overlap = Math.min(Math.max(0, options.overlap), Math.floor(size / 2));
+	const sections = [];
+	const path = [];
+	let current = {
+		path: [],
+		lines: []
+	};
+	let fence = false;
+	for (const line of text.replace(/\r\n?/g, "\n").split("\n")) {
+		if (/^\s*(```|~~~)/.test(line)) fence = !fence;
+		const heading = !fence && /^(#{1,6})\s+(.+?)\s*#*\s*$/.exec(line);
+		if (heading) {
+			sections.push(current);
+			const level = heading[1].length;
+			path.length = level - 1;
+			path[level - 1] = heading[2].trim();
+			current = {
+				path: path.filter(Boolean),
+				lines: []
+			};
+		} else current.lines.push(line);
+	}
+	sections.push(current);
+	const chunks = [];
+	for (const section of sections) {
+		const title = section.path.length ? section.path.join(" > ") : docTitle;
+		const pieces = section.lines.join("\n").split(/\n\s*\n/).map((paragraph) => paragraph.trim()).filter(Boolean).flatMap((paragraph) => splitLong(paragraph, size - overlap));
+		let buffer = "";
+		let tail = "";
+		const flush = () => {
+			if (!buffer.trim()) return;
+			chunks.push({
+				title,
+				text: buffer.trim()
+			});
+			tail = overlap ? buffer.trim().slice(-overlap) : "";
+			buffer = "";
+		};
+		for (const piece of pieces) {
+			if (buffer && buffer.length + piece.length + 2 > size) flush();
+			if (!buffer && tail) buffer = `${tail}…`;
+			buffer += (buffer && !buffer.endsWith("…") ? "\n\n" : "") + piece;
+		}
+		flush();
+	}
+	return chunks;
+}
+export { FORMATS, chunkText, extractText, formatOf, htmlToText };
+
+//# sourceMappingURL=text.js.map

package/dist/text.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"text.js","names":[],"sources":["../src/text.ts"],"sourcesContent":["import { MushinError } from '@mushinjs/core'\n\n/** Document formats a knowledge base reads. PDF and DOCX need optional packages. */\nexport const FORMATS = ['md', 'txt', 'html', 'pdf', 'docx'] as const\nexport type DocFormat = typeof FORMATS[number]\n\n/** The format of a file by its name, or nothing for one the knowledge base cannot read. */\nexport function formatOf(name: string): DocFormat | undefined {\n  const ext = /\\.([a-z0-9]+)$/i.exec(name)?.[1]?.toLowerCase()\n  switch (ext) {\n    case 'md': case 'markdown': case 'mdx': return 'md'\n    case 'txt': case 'text': case 'log': case 'csv': case 'json': case 'yaml': case 'yml': return 'txt'\n    case 'html': case 'htm': return 'html'\n    case 'pdf': return 'pdf'\n    case 'docx': return 'docx'\n  }\n}\n\nconst ENTITIES: Record<string, string> = { amp: '&', lt: '<', gt: '>', quot: '\"', apos: '\\'', nbsp: ' ' }\n\nfunction decodeEntities(text: string) {\n  return text.replace(/&(#x[0-9a-f]+|#\\d+|[a-z]+);/gi, (whole, name: string) => {\n    if (name[0] === '#') {\n      const code = name[1].toLowerCase() === 'x' ? Number.parseInt(name.slice(2), 16) : Number(name.slice(1))\n      return Number.isFinite(code) ? String.fromCodePoint(code) : whole\n    }\n    return ENTITIES[name.toLowerCase()] ?? whole\n  })\n}\n\n/** Readable text of an HTML page, with headings kept as Markdown headings so chunks know their section. */\nexport function htmlToText(html: string): { title?: string; text: string } {\n  const title = /<title[^>]*>([\\s\\S]*?)<\\/title>/i.exec(html)?.[1]\n  let body = /<body[^>]*>([\\s\\S]*)<\\/body>/i.exec(html)?.[1] ?? html\n  body = body\n    .replace(/<!--[\\s\\S]*?-->/g, '')\n    .replace(/<(script|style|noscript|svg|template|nav|footer|header|aside|form)\\b[\\s\\S]*?<\\/\\1>/gi, '')\n    .replace(/<h([1-6])[^>]*>([\\s\\S]*?)<\\/h\\1>/gi, (_, level: string, inner: string) => `\\n\\n${'#'.repeat(Number(level))} ${inner.replace(/<[^>]+>/g, '').trim()}\\n\\n`)\n    .replace(/<li[^>]*>/gi, '\\n- ')\n    .replace(/<br\\s*\\/?>/gi, '\\n')\n    .replace(/<\\/(p|div|section|article|ul|ol|table|tr|blockquote|pre|h[1-6])>/gi, '\\n\\n')\n    .replace(/<\\/t[dh]>/gi, '\\t')\n    .replace(/<[^>]+>/g, '')\n  const text = decodeEntities(body)\n    .split('\\n')\n    .map((line) => line.replace(/[ \\t ]+/g, ' ').trim())\n    .join('\\n')\n    .replace(/\\n{3,}/g, '\\n\\n')\n    .trim()\n  return { ...(title ? { title: decodeEntities(title).trim() } : {}), text }\n}\n\n/** Imports a package the knowledge base only needs for some formats. */\nasync function optional<T>(name: string, format: string): Promise<T> {\n  try {\n    return await import(name) as T\n  } catch {\n    throw new MushinError('unavailable', `Reading ${format} files needs the \"${name}\" package. Install it in the Mushin.js folder with: pnpm add -w ${name}`)\n  }\n}\n\n/** Text of a PDF's pages; scanned PDFs (images only) have none. */\nasync function pdfText(data: Uint8Array) {\n  const pdfjs = await optional<any>('pdfjs-dist/legacy/build/pdf.mjs', 'PDF')\n  const pdf = await pdfjs.getDocument({ data, useSystemFonts: true, isEvalSupported: false }).promise\n  const pages: string[] = []\n  for (let number = 1; number <= pdf.numPages; number++) {\n    const page = await pdf.getPage(number)\n    const content = await page.getTextContent()\n    let line = ''\n    const lines: string[] = []\n    for (const item of content.items as { str?: string; hasEOL?: boolean }[]) {\n      line += item.str ?? ''\n      if (item.hasEOL) {\n        lines.push(line)\n        line = ''\n      }\n    }\n    if (line) lines.push(line)\n    pages.push(lines.join('\\n'))\n  }\n  await pdf.destroy?.()\n  const text = pages.join('\\n\\n').trim()\n  if (!text) throw new MushinError('invalid_input', 'This PDF has no text to read; scanned PDFs need OCR, which is not supported')\n  return text\n}\n\nasync function docxText(data: Uint8Array) {\n  const mammoth = await optional<any>('mammoth', 'DOCX')\n  const lib = mammoth.default ?? mammoth\n  // Converting to HTML keeps headings, which chunks use as their titles.\n  const result = await lib.convertToHtml({ buffer: Buffer.from(data) })\n  return htmlToText(result.value).text\n}\n\n/** The text of a document. */\nexport async function extractText(format: DocFormat, data: Uint8Array): Promise<{ title?: string; text: string }> {\n  switch (format) {\n    case 'md':\n    case 'txt':\n      return { text: new TextDecoder().decode(data).replace(/^/, '').replace(/\\r\\n?/g, '\\n') }\n    case 'html':\n      return htmlToText(new TextDecoder().decode(data))\n    case 'pdf':\n      return { text: await pdfText(data) }\n    case 'docx':\n      return { text: await docxText(data) }\n  }\n}\n\nexport interface ChunkOptions {\n  /** Most characters in a chunk. */\n  size: number\n  /** Characters a chunk repeats from the end of the one before it, within a section. */\n  overlap: number\n}\n\nexport interface TextChunk {\n  /** The headings the chunk sits under, such as `群规 > 广告`. */\n  title: string\n  text: string\n}\n\n/** Pieces of at most `size` characters, cut after sentence ends where possible. */\nfunction splitLong(text: string, size: number): string[] {\n  if (text.length <= size) return [text]\n  const sentences = text.match(/[^。！？!?；;\\n.]+[。！？!?；;\\n.]*|[。！？!?；;\\n.]+/g) ?? [text]\n  const pieces: string[] = []\n  let current = ''\n  for (const sentence of sentences) {\n    if (current && current.length + sentence.length > size) {\n      pieces.push(current)\n      current = ''\n    }\n    if (sentence.length > size) {\n      for (let index = 0; index < sentence.length; index += size) pieces.push(sentence.slice(index, index + size))\n      continue\n    }\n    current += sentence\n  }\n  if (current) pieces.push(current)\n  return pieces.map((piece) => piece.trim()).filter(Boolean)\n}\n\n/**\n * Splits a document into chunks: by Markdown headings first (each chunk keeps its heading path\n * as its title), then by blank lines, then long paragraphs by sentence. Paragraphs are joined\n * while they fit in `size`; the next chunk of a section starts with the last `overlap` characters.\n */\nexport function chunkText(text: string, options: ChunkOptions, docTitle = ''): TextChunk[] {\n  const size = Math.max(50, options.size)\n  const overlap = Math.min(Math.max(0, options.overlap), Math.floor(size / 2))\n  const sections: { path: string[]; lines: string[] }[] = []\n  const path: string[] = []\n  let current = { path: [] as string[], lines: [] as string[] }\n  let fence = false\n  for (const line of text.replace(/\\r\\n?/g, '\\n').split('\\n')) {\n    if (/^\\s*(```|~~~)/.test(line)) fence = !fence\n    const heading = !fence && /^(#{1,6})\\s+(.+?)\\s*#*\\s*$/.exec(line)\n    if (heading) {\n      sections.push(current)\n      const level = heading[1].length\n      path.length = level - 1\n      path[level - 1] = heading[2].trim()\n      current = { path: path.filter(Boolean), lines: [] }\n    } else {\n      current.lines.push(line)\n    }\n  }\n  sections.push(current)\n\n  const chunks: TextChunk[] = []\n  for (const section of sections) {\n    const title = section.path.length ? section.path.join(' > ') : docTitle\n    const paragraphs = section.lines.join('\\n').split(/\\n\\s*\\n/).map((paragraph) => paragraph.trim()).filter(Boolean)\n    const pieces = paragraphs.flatMap((paragraph) => splitLong(paragraph, size - overlap))\n    let buffer = ''\n    let tail = ''\n    const flush = () => {\n      if (!buffer.trim()) return\n      chunks.push({ title, text: buffer.trim() })\n      tail = overlap ? buffer.trim().slice(-overlap) : ''\n      buffer = ''\n    }\n    for (const piece of pieces) {\n      if (buffer && buffer.length + piece.length + 2 > size) flush()\n      if (!buffer && tail) buffer = `${tail}…`\n      buffer += (buffer && !buffer.endsWith('…') ? '\\n\\n' : '') + piece\n    }\n    flush()\n  }\n  return chunks\n}\n"],"mappings":";;;AAGA,MAAa,UAAU;CAAC;CAAM;CAAO;CAAQ;CAAO;AAAM;;AAI1D,SAAgB,SAAS,MAAqC;CAE5D,QADY,kBAAkB,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,YAAY,GAC3D;EACE,KAAK;EAAM,KAAK;EAAY,KAAK,OAAO,OAAO;EAC/C,KAAK;EAAO,KAAK;EAAQ,KAAK;EAAO,KAAK;EAAO,KAAK;EAAQ,KAAK;EAAQ,KAAK,OAAO,OAAO;EAC9F,KAAK;EAAQ,KAAK,OAAO,OAAO;EAChC,KAAK,OAAO,OAAO;EACnB,KAAK,QAAQ,OAAO;CACtB;AACF;AAEA,MAAM,WAAmC;CAAE,KAAK;CAAK,IAAI;CAAK,IAAI;CAAK,MAAM;CAAK,MAAM;CAAM,MAAM;AAAI;AAExG,SAAS,eAAe,MAAc;CACpC,OAAO,KAAK,QAAQ,kCAAkC,OAAO,SAAiB;EAC5E,IAAI,KAAK,OAAO,KAAK;GACnB,MAAM,OAAO,KAAK,EAAE,CAAC,YAAY,MAAM,MAAM,OAAO,SAAS,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,OAAO,KAAK,MAAM,CAAC,CAAC;GACtG,OAAO,OAAO,SAAS,IAAI,IAAI,OAAO,cAAc,IAAI,IAAI;EAC9D;EACA,OAAO,SAAS,KAAK,YAAY,MAAM;CACzC,CAAC;AACH;;AAGA,SAAgB,WAAW,MAAgD;CACzE,MAAM,QAAQ,mCAAmC,KAAK,IAAI,CAAC,GAAG;CAC9D,IAAI,OAAO,gCAAgC,KAAK,IAAI,CAAC,GAAG,MAAM;CAC9D,OAAO,KACJ,QAAQ,oBAAoB,EAAE,CAAC,CAC/B,QAAQ,wFAAwF,EAAE,CAAC,CACnG,QAAQ,uCAAuC,GAAG,OAAe,UAAkB,OAAO,IAAI,OAAO,OAAO,KAAK,CAAC,EAAE,GAAG,MAAM,QAAQ,YAAY,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,CAClK,QAAQ,eAAe,MAAM,CAAC,CAC9B,QAAQ,gBAAgB,IAAI,CAAC,CAC7B,QAAQ,sEAAsE,MAAM,CAAC,CACrF,QAAQ,eAAe,GAAI,CAAC,CAC5B,QAAQ,YAAY,EAAE;CACzB,MAAM,OAAO,eAAe,IAAI,CAAC,CAC9B,MAAM,IAAI,CAAC,CACX,KAAK,SAAS,KAAK,QAAQ,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CACnD,KAAK,IAAI,CAAC,CACV,QAAQ,WAAW,MAAM,CAAC,CAC1B,KAAK;CACR,OAAO;EAAE,GAAI,QAAQ,EAAE,OAAO,eAAe,KAAK,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC;EAAI;CAAK;AAC3E;;AAGA,eAAe,SAAY,MAAc,QAA4B;CACnE,IAAI;EACF,OAAO,MAAM,OAAO;CACtB,QAAQ;EACN,MAAM,IAAI,YAAY,eAAe,WAAW,OAAO,oBAAoB,KAAK,kEAAkE,MAAM;CAC1J;AACF;;AAGA,eAAe,QAAQ,MAAkB;CAEvC,MAAM,MAAM,OAAM,MADE,SAAc,mCAAmC,KAAK,EAAA,CAClD,YAAY;EAAE;EAAM,gBAAgB;EAAM,iBAAiB;CAAM,CAAC,CAAC,CAAC;CAC5F,MAAM,QAAkB,CAAC;CACzB,KAAK,IAAI,SAAS,GAAG,UAAU,IAAI,UAAU,UAAU;EAErD,MAAM,UAAU,OAAM,MADH,IAAI,QAAQ,MAAM,EAAA,CACV,eAAe;EAC1C,IAAI,OAAO;EACX,MAAM,QAAkB,CAAC;EACzB,KAAK,MAAM,QAAQ,QAAQ,OAA+C;GACxE,QAAQ,KAAK,OAAO;GACpB,IAAI,KAAK,QAAQ;IACf,MAAM,KAAK,IAAI;IACf,OAAO;GACT;EACF;EACA,IAAI,MAAM,MAAM,KAAK,IAAI;EACzB,MAAM,KAAK,MAAM,KAAK,IAAI,CAAC;CAC7B;CACA,MAAM,IAAI,UAAU;CACpB,MAAM,OAAO,MAAM,KAAK,MAAM,CAAC,CAAC,KAAK;CACrC,IAAI,CAAC,MAAM,MAAM,IAAI,YAAY,iBAAiB,6EAA6E;CAC/H,OAAO;AACT;AAEA,eAAe,SAAS,MAAkB;CACxC,MAAM,UAAU,MAAM,SAAc,WAAW,MAAM;CAIrD,OAAO,YAAW,OAHN,QAAQ,WAAW,QAAA,CAEN,cAAc,EAAE,QAAQ,OAAO,KAAK,IAAI,EAAE,CAAC,EAAA,CAC3C,KAAK,CAAC,CAAC;AAClC;;AAGA,eAAsB,YAAY,QAAmB,MAA6D;CAChH,QAAQ,QAAR;EACE,KAAK;EACL,KAAK,OACH,OAAO,EAAE,MAAM,IAAI,YAAY,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,QAAQ,MAAM,EAAE,CAAC,CAAC,QAAQ,UAAU,IAAI,EAAE;EAC1F,KAAK,QACH,OAAO,WAAW,IAAI,YAAY,CAAC,CAAC,OAAO,IAAI,CAAC;EAClD,KAAK,OACH,OAAO,EAAE,MAAM,MAAM,QAAQ,IAAI,EAAE;EACrC,KAAK,QACH,OAAO,EAAE,MAAM,MAAM,SAAS,IAAI,EAAE;CACxC;AACF;;AAgBA,SAAS,UAAU,MAAc,MAAwB;CACvD,IAAI,KAAK,UAAU,MAAM,OAAO,CAAC,IAAI;CACrC,MAAM,YAAY,KAAK,MAAM,4CAA4C,KAAK,CAAC,IAAI;CACnF,MAAM,SAAmB,CAAC;CAC1B,IAAI,UAAU;CACd,KAAK,MAAM,YAAY,WAAW;EAChC,IAAI,WAAW,QAAQ,SAAS,SAAS,SAAS,MAAM;GACtD,OAAO,KAAK,OAAO;GACnB,UAAU;EACZ;EACA,IAAI,SAAS,SAAS,MAAM;GAC1B,KAAK,IAAI,QAAQ,GAAG,QAAQ,SAAS,QAAQ,SAAS,MAAM,OAAO,KAAK,SAAS,MAAM,OAAO,QAAQ,IAAI,CAAC;GAC3G;EACF;EACA,WAAW;CACb;CACA,IAAI,SAAS,OAAO,KAAK,OAAO;CAChC,OAAO,OAAO,KAAK,UAAU,MAAM,KAAK,CAAC,CAAC,CAAC,OAAO,OAAO;AAC3D;;;;;;AAOA,SAAgB,UAAU,MAAc,SAAuB,WAAW,IAAiB;CACzF,MAAM,OAAO,KAAK,IAAI,IAAI,QAAQ,IAAI;CACtC,MAAM,UAAU,KAAK,IAAI,KAAK,IAAI,GAAG,QAAQ,OAAO,GAAG,KAAK,MAAM,OAAO,CAAC,CAAC;CAC3E,MAAM,WAAkD,CAAC;CACzD,MAAM,OAAiB,CAAC;CACxB,IAAI,UAAU;EAAE,MAAM,CAAC;EAAe,OAAO,CAAC;CAAc;CAC5D,IAAI,QAAQ;CACZ,KAAK,MAAM,QAAQ,KAAK,QAAQ,UAAU,IAAI,CAAC,CAAC,MAAM,IAAI,GAAG;EAC3D,IAAI,gBAAgB,KAAK,IAAI,GAAG,QAAQ,CAAC;EACzC,MAAM,UAAU,CAAC,SAAS,6BAA6B,KAAK,IAAI;EAChE,IAAI,SAAS;GACX,SAAS,KAAK,OAAO;GACrB,MAAM,QAAQ,QAAQ,EAAE,CAAC;GACzB,KAAK,SAAS,QAAQ;GACtB,KAAK,QAAQ,KAAK,QAAQ,EAAE,CAAC,KAAK;GAClC,UAAU;IAAE,MAAM,KAAK,OAAO,OAAO;IAAG,OAAO,CAAC;GAAE;EACpD,OACE,QAAQ,MAAM,KAAK,IAAI;CAE3B;CACA,SAAS,KAAK,OAAO;CAErB,MAAM,SAAsB,CAAC;CAC7B,KAAK,MAAM,WAAW,UAAU;EAC9B,MAAM,QAAQ,QAAQ,KAAK,SAAS,QAAQ,KAAK,KAAK,KAAK,IAAI;EAE/D,MAAM,SADa,QAAQ,MAAM,KAAK,IAAI,CAAC,CAAC,MAAM,SAAS,CAAC,CAAC,KAAK,cAAc,UAAU,KAAK,CAAC,CAAC,CAAC,OAAO,OACjF,CAAC,CAAC,SAAS,cAAc,UAAU,WAAW,OAAO,OAAO,CAAC;EACrF,IAAI,SAAS;EACb,IAAI,OAAO;EACX,MAAM,cAAc;GAClB,IAAI,CAAC,OAAO,KAAK,GAAG;GACpB,OAAO,KAAK;IAAE;IAAO,MAAM,OAAO,KAAK;GAAE,CAAC;GAC1C,OAAO,UAAU,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,IAAI;GACjD,SAAS;EACX;EACA,KAAK,MAAM,SAAS,QAAQ;GAC1B,IAAI,UAAU,OAAO,SAAS,MAAM,SAAS,IAAI,MAAM,MAAM;GAC7D,IAAI,CAAC,UAAU,MAAM,SAAS,GAAG,KAAK;GACtC,WAAW,UAAU,CAAC,OAAO,SAAS,GAAG,IAAI,SAAS,MAAM;EAC9D;EACA,MAAM;CACR;CACA,OAAO;AACT"}

package/package.json

@@ -1,6 +1,73 @@
 {
   "name": "@mushinjs/plugin-knowledge",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "version": "0.2.0-next.20261113103537",
+  "description": "Knowledge bases: documents searched by keywords and meaning, from flows, the Agent and chat",
+  "type": "module",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/mushinjs/mushin.git",
+    "directory": "packages/plugin-knowledge"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "keywords": [
+    "mushin-plugin"
+  ],
+  "mushin": {
+    "kind": "app",
+    "category": "ai",
+    "displayName": {
+      "en": "Knowledge bases",
+      "zh-CN": "知识库"
+    },
+    "description": {
+      "en": "Upload documents and let bots and the Agent answer from them; admins add knowledge from chat, and bases can learn from chats for review",
+      "zh-CN": "上传资料，让机器人和智能体据此回答；管理员可以在聊天里补充知识，也可以从对话里学习，审核后加入"
+    },
+    "provides": [
+      "knowledge"
+    ],
+    "requires": {
+      "mushin": ">=0.1.0",
+      "plugins": {
+        "@mushinjs/plugin-bots": "*",
+        "@mushinjs/plugin-database": "*",
+        "@mushinjs/plugin-providers": "*",
+        "@mushinjs/plugin-storage": "*"
+      }
+    },
+    "optional": {
+      "plugins": {
+        "@mushinjs/plugin-flows": "*",
+        "@mushinjs/plugin-agent": "*"
+      }
+    },
+    "nodes": [
+      "kb.*"
+    ]
+  },
+  "dependencies": {
+    "@mushinjs/core": "0.2.0-next.20261113103537",
+    "@mushinjs/plugin-bots": "0.2.0-next.20261113103537",
+    "@mushinjs/plugin-database": "0.2.0-next.20261113103537",
+    "@mushinjs/plugin-flows": "0.2.0-next.20261113103537",
+    "@mushinjs/plugin-providers": "0.2.0-next.20261113103537",
+    "@mushinjs/plugin-storage": "0.2.0-next.20261113103537",
+    "minato": "4.0.1"
+  },
+  "devDependencies": {
+    "@mushinjs/plugin-agent": "0.2.0-next.20261113103537"
+  }
 }

package/README.md

@@ -1,3 +0,0 @@
-# Temporary Holding Version
-
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.