@mushinjs/plugin-knowledge 0.0.0-stage → 0.2.0-next.20261113103537

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Setobox
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/dist/index.d.ts

@@ -0,0 +1,381 @@
+import { IndexedChunk, SearchHit, SearchIndex, SearchOptions, dot, normalize, passages, tokenize } from "./search.js";
+import { Candidate, ChatLineLike, LEARN_PROMPT, candidateText, parseCandidates, transcript } from "./learn.js";
+import { ChunkOptions, DocFormat, FORMATS, TextChunk, chunkText, extractText, formatOf, htmlToText } from "./text.js";
+import { Context, Service, z } from "@mushinjs/core";
+import { BotRole } from "@mushinjs/plugin-bots";
+declare module '@mushinjs/core' {
+  interface Context {
+    knowledge: Knowledge;
+  }
+  interface Events {
+    /** Bases, documents, chunks or candidates of a base changed. */
+    'knowledge/changed'(change: {
+      base: string;
+    }): void;
+    /** A document or a reindex is being processed. */
+    'knowledge/progress'(progress: KnowledgeProgress): void;
+  }
+}
+declare module 'minato' {
+  interface Tables {
+    knowledge_base: KnowledgeBase;
+    knowledge_doc: KnowledgeDoc;
+    knowledge_chunk: ChunkRow;
+    knowledge_candidate: CandidateRow;
+  }
+}
+export interface LearnSettings {
+  /** Pick out candidate knowledge from the chats of the base's bots. */
+  enabled: boolean;
+  /** `group:<id>` or `private:<id>`; empty learns from every chat of the bots. */
+  channels: string[];
+  /** Learn once this many new messages arrived in a chat. */
+  every: number;
+  /** Add candidates straight to the base instead of waiting for review. */
+  autoAccept: boolean;
+  /** Most learning requests per day for this base. */
+  dailyLimit: number;
+  /** Provider and model for learning; empty uses the bot's, then the default. */
+  provider: string;
+  model: string;
+}
+export interface KnowledgeBase {
+  id: string;
+  name: string;
+  description: string;
+  /** Embedding provider and model; empty means keyword search only. */
+  provider: string;
+  model: string;
+  /** Length of the vectors; 0 until the base is embedded with its current model. */
+  dimension: number;
+  chunkSize: number;
+  chunkOverlap: number;
+  /** Defaults for searches that do not say. */
+  topK: number;
+  minScore: number;
+  /** Bots whose admins may use `/记住`, `/收录` and `/忘记`, and whose chats it learns from. */
+  bots: string[];
+  commands: {
+    enabled: boolean;
+    role: BotRole;
+  };
+  learn: LearnSettings;
+  createdAt: number;
+  updatedAt: number;
+}
+/**
+ * `file`: an uploaded file. `url`: a web page, fetched again when reparsed. `text`: pasted text.
+ * `notes`: entries added one at a time (`/记住`, `/收录`, the dashboard). `learned`: accepted
+ * candidates from chats.
+ */
+export type DocKind = 'file' | 'url' | 'text' | 'notes' | 'learned';
+export type DocStatus = 'pending' | 'parsing' | 'embedding' | 'ready' | 'error';
+export interface KnowledgeDoc {
+  id: string;
+  base: string;
+  title: string;
+  kind: DocKind;
+  /** File name or URL. */
+  source: string;
+  format: string;
+  status: DocStatus;
+  error: string;
+  /** SHA-256 of the content, so the same file is not added twice. */
+  hash: string;
+  chars: number;
+  chunks: number;
+  createdAt: number;
+  updatedAt: number;
+}
+interface ChunkRow {
+  id: string;
+  base: string;
+  doc: string;
+  seq: number;
+  title: string;
+  text: string;
+  /** `Float32Array` bytes, normalized; empty when not embedded. */
+  vector: ArrayBuffer;
+  /** Changed by hand; reparsing the document replaces it anyway. */
+  edited: boolean;
+  /** For notes: who added it, where. */
+  meta: Record<string, any>;
+  createdAt: number;
+  updatedAt: number;
+}
+export type KnowledgeChunk = Omit<ChunkRow, 'vector'> & {
+  embedded: boolean;
+};
+export type CandidateStatus = 'pending' | 'accepted' | 'rejected';
+interface CandidateRow {
+  id: number;
+  base: string;
+  bot: string;
+  channel: string;
+  question: string;
+  answer: string;
+  status: CandidateStatus;
+  /** The chunk an accepted candidate became. */
+  chunk: string;
+  createdAt: number;
+  updatedAt: number;
+}
+export type KnowledgeCandidate = CandidateRow;
+export interface KnowledgeProgress {
+  base: string;
+  doc?: string;
+  /** `reindex` for a base being embedded again. */
+  stage: DocStatus | 'reindex' | 'done';
+  done: number;
+  total: number;
+}
+/** A search hit with the base and document it came from. */
+export interface KnowledgeHit extends SearchHit {
+  base: string;
+  baseName: string;
+  docTitle: string;
+}
+export interface KnowledgeSearch {
+  hits: KnowledgeHit[];
+  /** Searched by meaning too (at least one base has an embedding model that answered). */
+  semantic: boolean;
+  /** Problems that made a base fall back to keywords, such as the embedding model failing. */
+  warnings: string[];
+}
+/** A hit as a reply's provenance and a run's state keep it. */
+export interface KnowledgeRef {
+  base: string;
+  doc: string;
+  chunk: string;
+  title: string;
+}
+declare const baseFields: {
+  name: z.ZodString;
+  description: z.ZodOptional<z.ZodString>;
+  provider: z.ZodOptional<z.ZodString>;
+  model: z.ZodOptional<z.ZodString>;
+  chunkSize: z.ZodOptional<z.ZodNumber>;
+  chunkOverlap: z.ZodOptional<z.ZodNumber>;
+  topK: z.ZodOptional<z.ZodNumber>;
+  minScore: z.ZodOptional<z.ZodNumber>;
+  bots: z.ZodOptional<z.ZodArray<z.ZodString>>;
+  commands: z.ZodOptional<z.ZodObject<{
+    enabled: z.ZodOptional<z.ZodBoolean>;
+    role: z.ZodOptional<z.ZodEnum<{
+      master: "master";
+      admin: "admin";
+      user: "user";
+    }>>;
+  }, z.core.$strip>>;
+  learn: z.ZodOptional<z.ZodObject<{
+    enabled: z.ZodOptional<z.ZodBoolean>;
+    channels: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    every: z.ZodOptional<z.ZodNumber>;
+    autoAccept: z.ZodOptional<z.ZodBoolean>;
+    dailyLimit: z.ZodOptional<z.ZodNumber>;
+    provider: z.ZodOptional<z.ZodString>;
+    model: z.ZodOptional<z.ZodString>;
+  }, z.core.$strip>>;
+};
+/**
+ * Knowledge bases: documents split into chunks, searched by keywords and (with an embedding
+ * model) by meaning. Flows search them with a node, the Agent with `kb.search`, admins add to
+ * them from chat, and they can learn candidate knowledge from chats for review.
+ */
+export declare class Knowledge extends Service {
+  static inject: string[];
+  private home;
+  private indexes;
+  private queue;
+  private abort;
+  /** Base id → `chat key` → messages since the last learning. */
+  private counters;
+  private learning;
+  /** Chat key → when it was last learned from, so the same messages are not read twice. */
+  private learnState;
+  private usage;
+  constructor(ctx: Context);
+  private get logger();
+  private get db();
+  [Service.init](): AsyncGenerator<import("cordis").Disposable<Promise<void>> | (() => boolean), void, unknown>;
+  list(): Promise<import("minato").FlatPick<KnowledgeBase, any>[]>;
+  get(id: string): Promise<KnowledgeBase>;
+  /** Bases with their document, chunk and candidate counts, for the list page. */
+  summaries(): Promise<{
+    docs: number;
+    chunks: number;
+    embedded: number;
+    busy: boolean;
+    candidates: number;
+    bots: string[];
+    model: string;
+    id: string;
+    description: string;
+    createdAt: number;
+    name: string;
+    provider: string;
+    dimension: number;
+    chunkSize: number;
+    chunkOverlap: number;
+    topK: number;
+    minScore: number;
+    commands: {
+      enabled: boolean;
+      role: BotRole;
+    };
+    learn: LearnSettings;
+    updatedAt: number;
+  }[]>;
+  /** Embeds a probe text to learn the model's vector length; fails with a readable error. */
+  private probe;
+  create(input: z.infer<z.ZodObject<typeof baseFields>>): Promise<KnowledgeBase>;
+  update(id: string, patch: Partial<z.infer<z.ZodObject<typeof baseFields>>>): Promise<KnowledgeBase>;
+  remove(id: string): Promise<{
+    ok: boolean;
+  }>;
+  private changed;
+  private progress;
+  private fileDir;
+  private filePath;
+  docs(base: string): Promise<import("minato").FlatPick<KnowledgeDoc, any>[]>;
+  doc(id: string): Promise<KnowledgeDoc>;
+  /**
+   * Adds a document: a file (`data` in base64 with its `name`), pasted `text`, or a `url` to fetch.
+   * It is parsed, split and embedded in the background; `knowledge/progress` reports how it goes.
+   */
+  addDoc(input: {
+    base: string;
+    name?: string;
+    text?: string;
+    data?: string;
+    url?: string;
+  }): Promise<KnowledgeDoc>;
+  private fetchPage;
+  removeDoc(id: string): Promise<{
+    ok: boolean;
+  }>;
+  /** Parses a document again from its file (a web page is fetched again); edits to its chunks are lost. */
+  reparse(id: string): Promise<{
+    ok: boolean;
+  }>;
+  private enqueue;
+  private setStatus;
+  /** Embeds texts with a base's model, `onBatch` after each batch; nothing for keyword-only bases. */
+  private embedTexts;
+  /** What a chunk's embedding reads: its title and text. */
+  private embedText;
+  private process;
+  /** Embeds every chunk of a base again with its current model, e.g. after the model changed. */
+  reindex(id: string): Promise<unknown>;
+  chunks(input: {
+    base: string;
+    doc?: string;
+    offset?: number;
+    limit?: number;
+    query?: string;
+  }): Promise<{
+    total: number;
+    chunks: KnowledgeChunk[];
+  }>;
+  private chunkRow;
+  updateChunk(id: string, patch: {
+    title?: string;
+    text?: string;
+  }): Promise<KnowledgeChunk>;
+  removeChunk(id: string): Promise<{
+    ok: boolean;
+  }>;
+  /** Joins a chunk with the one after it in the same document. */
+  mergeChunk(id: string): Promise<KnowledgeChunk>;
+  private recount;
+  /** The document of single entries of a kind, made the first time one is added. */
+  private entriesDoc;
+  /** Adds one entry, such as a `/记住` note or an accepted candidate, as a chunk of its own. */
+  addEntry(baseId: string, entry: {
+    text: string;
+    title?: string;
+    kind?: 'notes' | 'learned';
+    meta?: Record<string, any>;
+  }): Promise<KnowledgeChunk>;
+  /** Removes notes that contain the text; returns how many. */
+  forget(baseId: string, text: string): Promise<number>;
+  private index;
+  /**
+   * Searches bases (every one when none are given) by keywords and, where the base has an
+   * embedding model, by meaning. A failing embedding model falls back to keywords with a warning.
+   */
+  search(input: {
+    bases?: string[];
+    query: string;
+    topK?: number;
+    minScore?: number;
+    signal?: AbortSignal;
+  }): Promise<KnowledgeSearch>;
+  /** Chunks and links between similar ones, for the similarity map. */
+  graph(input: {
+    base: string;
+    neighbors?: number;
+    minSimilarity?: number;
+    limit?: number;
+  }): Promise<{
+    byVector: boolean;
+    total: number;
+    nodes: {
+      id: string;
+      doc: string;
+      docTitle: string;
+      title: string;
+      text: string;
+    }[];
+    edges: {
+      source: string;
+      target: string;
+      similarity: number;
+    }[];
+  }>;
+  /** Flows that ask an LLM, for "use in a flow", with whether they already search this base. */
+  flowChoices(base?: string): {
+    id: string;
+    name: string;
+    bots: string[];
+    llmChat: boolean;
+    uses: boolean;
+  }[];
+  /**
+   * Puts a knowledge search in front of a flow's first "Ask the LLM" node and passes its passages
+   * to it; a flow that already has a search node gets these bases added to it.
+   */
+  useInFlow(flowId: string, bases: string[]): Promise<{
+    flow: string;
+    node: string;
+    added: boolean;
+  }>;
+  /** Bases a bot may add to from chat, oldest first. */
+  private basesFor;
+  private defineCommands;
+  private metaOf;
+  /** The base a chat command writes to, after checking the sender may use it; tells them when not. */
+  private allowed;
+  private onMessage;
+  private useQuota;
+  /**
+   * Reads a chat's messages since it was last learned from and asks the LLM for knowledge worth
+   * keeping. What it finds waits for review, or is added straight away with auto-accept.
+   */
+  learn(baseId: string, botId: string, chat: string): Promise<{
+    candidates: KnowledgeCandidate[];
+    skipped?: string;
+  }>;
+  /** Leaves out candidates already waiting for review or already known to the base. */
+  private newCandidates;
+  candidates(base: string, status?: CandidateStatus): Promise<import("minato").FlatPick<CandidateRow, any>[]>;
+  private candidate;
+  /** Adds a candidate (as edited) to the base's "learned from chats" document. */
+  accept(id: number, patch?: {
+    question?: string;
+    answer?: string;
+  }): Promise<CandidateRow>;
+  reject(id: number): Promise<CandidateRow>;
+  private defineActions;
+}
+export { Candidate, ChatLineLike, ChunkOptions, DocFormat, FORMATS, IndexedChunk, Knowledge as default, LEARN_PROMPT, SearchHit, SearchIndex, SearchOptions, TextChunk, candidateText, chunkText, dot, extractText, formatOf, htmlToText, normalize, parseCandidates, passages, tokenize, transcript };