knowmind 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +17 -0
- package/README.md +70 -0
- package/bin/knowmind.js +224 -0
- package/package.json +51 -0
- package/src/client.js +115 -0
- package/src/config.js +118 -0
- package/src/extract-statements.js +134 -0
- package/src/local-embedding.js +122 -0
- package/src/mcp-stdio.js +481 -0
- package/src/sync.js +128 -0
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pre-Ingest-Extraktor für Preference-, Fact- und Update-Statements.
|
|
3
|
+
*
|
|
4
|
+
* Knowmind-CLI ergänzt jedes Upload um kurze, getaggte Zusatz-Chunks aus
|
|
5
|
+
* dem User-Anteil des Textes. Die Recall-Pipeline auf dem Server boostet
|
|
6
|
+
* diese Chunks gezielt, wenn die spätere Query nach genau diesem Statement-
|
|
7
|
+
* Typ fragt (siehe services/memory/app/recall_pipeline.py:fuse_and_rerank).
|
|
8
|
+
*
|
|
9
|
+
* Bewusst Regex statt LLM:
|
|
10
|
+
* - Reproduzierbar — kein Zufalls-Output
|
|
11
|
+
* - Schnell — kein Round-Trip, keine API-Kosten
|
|
12
|
+
* - Funktioniert für die häufigsten Pattern in Englisch und Deutsch
|
|
13
|
+
*
|
|
14
|
+
* Spiegelt die Logik aus services/memory/app/extractor.py — wenn der CLI
|
|
15
|
+
* die Extraction macht, läuft der Server-Fallback nicht doppelt.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
const PREFERENCE_PATTERNS = [
|
|
19
|
+
/\b(i (?:really )?(?:prefer|love|like|enjoy|adore|fancy)\b[^.!?]{4,200})/gi,
|
|
20
|
+
/\b(i (?:am|'m) into\b[^.!?]{4,200})/gi,
|
|
21
|
+
/\b(my (?:favou?rite|preferred)\b[^.!?]{4,200})/gi,
|
|
22
|
+
/\b(i (?:can'?t stand|hate|dislike|despise|avoid)\b[^.!?]{4,200})/gi,
|
|
23
|
+
/\b(i (?:always|usually|never|tend to)\b[^.!?]{4,200})/gi,
|
|
24
|
+
/\b(i (?:want|need|would like|wish|hope) to\b[^.!?]{4,200})/gi,
|
|
25
|
+
/\b(ich (?:mag|liebe|bevorzuge|hasse|vermeide)\b[^.!?]{4,200})/gi,
|
|
26
|
+
/\b(mein lieblings\b[^.!?]{4,200})/gi,
|
|
27
|
+
/\b(ich (?:bin|möchte|will)\b[^.!?]{4,200})/gi,
|
|
28
|
+
];
|
|
29
|
+
|
|
30
|
+
const FACT_PATTERNS = [
|
|
31
|
+
/\b(i (?:am|'m) (?:a|an|the)\b[^.!?]{4,150})/gi,
|
|
32
|
+
/\b(i (?:live|work|grew up|was born) (?:in|at|near|as)\b[^.!?]{4,200})/gi,
|
|
33
|
+
/\b(my (?:name|age|job|wife|husband|partner|daughter|son|brother|sister|mother|father|dog|cat|car|home|house|apartment|address) (?:is|was|will be)\b[^.!?]{2,200})/gi,
|
|
34
|
+
/\b(i have (?:a|an|two|three|four|five|six|seven|several|many)\b[^.!?]{4,200})/gi,
|
|
35
|
+
/\b(i (?:work|worked) (?:at|for|with|as)\b[^.!?]{4,200})/gi,
|
|
36
|
+
/\b(i'?(?:ve| have) been\b[^.!?]{4,200})/gi,
|
|
37
|
+
/\b(i (?:graduated|studied|majored|earned|completed|finished|received|got|hold)\b[^.!?]{4,200})/gi,
|
|
38
|
+
/\b(i (?:learned|trained|specialized|certified)\b[^.!?]{4,200})/gi,
|
|
39
|
+
/\b(my (?:degree|major|education|background|profession|career|salary|company|employer|school|university|college|alma mater) (?:is|was|in)\b[^.!?]{2,200})/gi,
|
|
40
|
+
/\b(i (?:weigh|measure) [0-9]+[^.!?]{0,80})/gi,
|
|
41
|
+
/\b(my (?:weight|height|blood pressure|score|rate|count|record|target|budget|salary|rent|mortgage) (?:is|was|of) [^.!?]{2,150})/gi,
|
|
42
|
+
/\b(i (?:set|achieved|finished|completed|won|reached) (?:a|an|the|my)?\s*(?:personal best|record|goal|milestone|time)\b[^.!?]{2,200})/gi,
|
|
43
|
+
/\b(ich (?:bin|war|heiße|wohne|arbeite|lebe|studiere|studierte|wiege)\b[^.!?]{4,200})/gi,
|
|
44
|
+
/\b(mein (?:name|alter|beruf|haus|auto|hund|katze|partner|gewicht|gehalt|abschluss|studium) (?:ist|war)\b[^.!?]{2,200})/gi,
|
|
45
|
+
];
|
|
46
|
+
|
|
47
|
+
const UPDATE_PATTERNS = [
|
|
48
|
+
/\b(actually,?\s[^.!?]{4,200})/gi,
|
|
49
|
+
/\b(i (?:changed my mind|no longer|don'?t .* anymore)\b[^.!?]{4,200})/gi,
|
|
50
|
+
/\b((?:now|recently|since) i (?:prefer|like|love|work|live|use)\b[^.!?]{4,200})/gi,
|
|
51
|
+
/\b(i (?:moved|switched|changed) to\b[^.!?]{4,200})/gi,
|
|
52
|
+
/\b(inzwischen (?:bevorzuge|mag|wohne|arbeite) ich\b[^.!?]{4,200})/gi,
|
|
53
|
+
/\b(ich habe (?:gewechselt|umgezogen|gekündigt)\b[^.!?]{4,200})/gi,
|
|
54
|
+
];
|
|
55
|
+
|
|
56
|
+
const USER_MARKER = /^\s*(?:\[user\]|user\s*:|frage\s*:)/i;
|
|
57
|
+
const ROLE_MARKER = /^\s*(?:\[[a-z_]+\]|[a-z_]+\s*:)/i;
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Liefert nur den User-Anteil eines Chat-Logs. Wenn keine Rollen-Markierung
|
|
61
|
+
* vorkommt (klassische Markdown-Notiz), wird der gesamte Text behandelt.
|
|
62
|
+
*/
|
|
63
|
+
function extractUserBlocks(text) {
|
|
64
|
+
if (!ROLE_MARKER.test(text)) return text;
|
|
65
|
+
const out = [];
|
|
66
|
+
let currentRole = null;
|
|
67
|
+
let buf = [];
|
|
68
|
+
for (const line of text.split(/\r?\n/)) {
|
|
69
|
+
if (USER_MARKER.test(line)) {
|
|
70
|
+
if (currentRole === "user" && buf.length) out.push(buf.join("\n"));
|
|
71
|
+
currentRole = "user";
|
|
72
|
+
buf = [];
|
|
73
|
+
} else if (ROLE_MARKER.test(line)) {
|
|
74
|
+
if (currentRole === "user" && buf.length) out.push(buf.join("\n"));
|
|
75
|
+
currentRole = "other";
|
|
76
|
+
buf = [];
|
|
77
|
+
} else if (currentRole === "user") {
|
|
78
|
+
buf.push(line);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
if (currentRole === "user" && buf.length) out.push(buf.join("\n"));
|
|
82
|
+
return out.join("\n");
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Findet alle Statements im User-Anteil. Liefert [{text, type}], wobei type
|
|
87
|
+
* ∈ {preference, fact, update}.
|
|
88
|
+
*/
|
|
89
|
+
export function extractUserStatements(text) {
|
|
90
|
+
if (!text || !text.trim()) return [];
|
|
91
|
+
const userText = extractUserBlocks(text);
|
|
92
|
+
if (!userText.trim()) return [];
|
|
93
|
+
|
|
94
|
+
const out = [];
|
|
95
|
+
const seen = new Set();
|
|
96
|
+
|
|
97
|
+
const groups = [
|
|
98
|
+
["preference", PREFERENCE_PATTERNS],
|
|
99
|
+
["fact", FACT_PATTERNS],
|
|
100
|
+
["update", UPDATE_PATTERNS],
|
|
101
|
+
];
|
|
102
|
+
|
|
103
|
+
for (const [type, patterns] of groups) {
|
|
104
|
+
for (const pat of patterns) {
|
|
105
|
+
const matches = userText.matchAll(pat);
|
|
106
|
+
for (const m of matches) {
|
|
107
|
+
const start = m.index;
|
|
108
|
+
let end = m.index + m[0].length;
|
|
109
|
+
const tail = userText.slice(end, end + 180);
|
|
110
|
+
const stop = tail.search(/[.!?]/);
|
|
111
|
+
if (stop !== -1) end += stop + 1;
|
|
112
|
+
const snippet = userText.slice(start, end).trim().replace(/\s+/g, " ");
|
|
113
|
+
const key = snippet.toLowerCase();
|
|
114
|
+
if (snippet.length >= 8 && snippet.length <= 280 && !seen.has(key)) {
|
|
115
|
+
out.push({ text: snippet, type });
|
|
116
|
+
seen.add(key);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
return out;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Konstruiert Pre-Ingest-Zusatz-Chunks aus dem Roh-Text.
|
|
126
|
+
* Liefert {content, metadata}-Paare. Hard cap maxChunks pro Dokument.
|
|
127
|
+
*/
|
|
128
|
+
export function synthChunks(text, maxChunks = 24) {
|
|
129
|
+
const statements = extractUserStatements(text).slice(0, maxChunks);
|
|
130
|
+
return statements.map(({ text: snippet, type }) => ({
|
|
131
|
+
content: `[${type}] ${snippet}`,
|
|
132
|
+
metadata: { type, extracted: true },
|
|
133
|
+
}));
|
|
134
|
+
}
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lokale Embedding-Berechnung im Client.
|
|
3
|
+
*
|
|
4
|
+
* Skalierungs-Pattern: Knowmind speichert + sucht, der Client rechnet
|
|
5
|
+
* Embeddings selbst. Server-CPU bleibt frei, jeder Kunde bringt seine
|
|
6
|
+
* eigene CPU mit.
|
|
7
|
+
*
|
|
8
|
+
* Modell: Xenova/multilingual-e5-large (1024d, mehrsprachig, ONNX-quantisiert).
|
|
9
|
+
* Beim ersten Aufruf wird das Modell nach ~/.knowmind/models/ heruntergeladen
|
|
10
|
+
* (rund 400 MB nach Komprimierung). Danach läuft alles offline.
|
|
11
|
+
*
|
|
12
|
+
* Wird nicht eager geladen — nur wenn der CLI tatsächlich einen
|
|
13
|
+
* embedding-basierten Befehl ausführt (store, upload). recall, health
|
|
14
|
+
* und stats brauchen das Modell nicht.
|
|
15
|
+
*/
|
|
16
|
+
import { homedir } from "node:os";
|
|
17
|
+
import { join } from "node:path";
|
|
18
|
+
import { mkdirSync, existsSync } from "node:fs";
|
|
19
|
+
|
|
20
|
+
const MODEL_ID = process.env.KNOWMIND_EMBED_MODEL || "Xenova/multilingual-e5-large";
|
|
21
|
+
const TARGET_DIM = 1024;
|
|
22
|
+
const CHUNK_TARGET = 1200; // Zeichen pro Chunk — identisch zum Server-Default
|
|
23
|
+
|
|
24
|
+
const CACHE_DIR = join(homedir(), ".knowmind", "models");
|
|
25
|
+
if (!existsSync(CACHE_DIR)) {
|
|
26
|
+
mkdirSync(CACHE_DIR, { recursive: true });
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
let pipelinePromise = null;
|
|
30
|
+
|
|
31
|
+
async function loadPipeline() {
|
|
32
|
+
if (pipelinePromise) return pipelinePromise;
|
|
33
|
+
pipelinePromise = (async () => {
|
|
34
|
+
const { pipeline, env } = await import("@huggingface/transformers");
|
|
35
|
+
// Cache-Verzeichnis explizit setzen — sonst landet das Modell in
|
|
36
|
+
// unklaren Pfaden und der Cache wirkt nicht zwischen CLI-Aufrufen.
|
|
37
|
+
env.cacheDir = CACHE_DIR;
|
|
38
|
+
env.allowLocalModels = true;
|
|
39
|
+
// ONNX-Quantisierte Variante laden — kleiner und auf CPU spürbar schneller.
|
|
40
|
+
const extractor = await pipeline("feature-extraction", MODEL_ID, {
|
|
41
|
+
dtype: "q8",
|
|
42
|
+
});
|
|
43
|
+
return extractor;
|
|
44
|
+
})();
|
|
45
|
+
return pipelinePromise;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** Modell explizit vorab laden — sinnvoll für CLI-Init oder lange Sessions. */
|
|
49
|
+
export async function warmup() {
|
|
50
|
+
await loadPipeline();
|
|
51
|
+
return { model: MODEL_ID, cacheDir: CACHE_DIR };
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Zerlegt Text in Chunks identisch zum Server-Default (1200 Zeichen).
|
|
56
|
+
*/
|
|
57
|
+
export function chunkText(text, size = CHUNK_TARGET) {
|
|
58
|
+
const out = [];
|
|
59
|
+
for (let i = 0; i < text.length; i += size) {
|
|
60
|
+
const seg = text.slice(i, i + size);
|
|
61
|
+
if (seg.trim()) out.push(seg);
|
|
62
|
+
}
|
|
63
|
+
return out.length > 0 ? out : [""];
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Pre-Ingest-Pipeline: zerlegt Text in Standard-Chunks UND extrahiert
|
|
68
|
+
* heuristisch Preference-/Fact-/Update-Statements als Zusatz-Chunks mit
|
|
69
|
+
* metadata.type. Liefert {chunks: [{content, metadata?}]}, das direkt an
|
|
70
|
+
* /api/documents geschickt werden kann.
|
|
71
|
+
*/
|
|
72
|
+
export async function prepareChunks(text) {
|
|
73
|
+
const { synthChunks } = await import("./extract-statements.js");
|
|
74
|
+
const baseChunks = chunkText(text).map((content) => ({ content }));
|
|
75
|
+
const extras = synthChunks(text);
|
|
76
|
+
return [...baseChunks, ...extras];
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Einzelnes Embedding für eine Query oder Passage.
|
|
81
|
+
* mode='passage' (Default) prefixt mit "passage: ", 'query' mit "query: ".
|
|
82
|
+
* Das ist Pflicht beim e5-Modell.
|
|
83
|
+
*/
|
|
84
|
+
export async function embed(text, mode = "passage") {
|
|
85
|
+
const extractor = await loadPipeline();
|
|
86
|
+
const prefix = mode === "query" ? "query: " : "passage: ";
|
|
87
|
+
const out = await extractor(prefix + text, { pooling: "mean", normalize: true });
|
|
88
|
+
const vec = Array.from(out.data);
|
|
89
|
+
if (vec.length !== TARGET_DIM) {
|
|
90
|
+
throw new Error(
|
|
91
|
+
`Embedding-Dim ${vec.length} ≠ erwartet ${TARGET_DIM}. Modell stimmt nicht.`,
|
|
92
|
+
);
|
|
93
|
+
}
|
|
94
|
+
return vec;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Batch-Embedding für mehrere Passagen.
|
|
99
|
+
* Wenn der Aufrufer eine ganze Notiz mit vielen Chunks hat, soll er das
|
|
100
|
+
* in einem Schwung machen, damit das Modell die Inferenz batched.
|
|
101
|
+
*/
|
|
102
|
+
export async function embedBatch(texts, mode = "passage") {
|
|
103
|
+
if (!texts || texts.length === 0) return [];
|
|
104
|
+
const extractor = await loadPipeline();
|
|
105
|
+
const prefix = mode === "query" ? "query: " : "passage: ";
|
|
106
|
+
const inputs = texts.map((t) => prefix + t);
|
|
107
|
+
const out = await extractor(inputs, { pooling: "mean", normalize: true });
|
|
108
|
+
// out.dims = [batch, dim]
|
|
109
|
+
const dim = out.dims[1];
|
|
110
|
+
if (dim !== TARGET_DIM) {
|
|
111
|
+
throw new Error(`Embedding-Dim ${dim} ≠ erwartet ${TARGET_DIM}.`);
|
|
112
|
+
}
|
|
113
|
+
const data = Array.from(out.data);
|
|
114
|
+
const vectors = [];
|
|
115
|
+
for (let i = 0; i < texts.length; i++) {
|
|
116
|
+
vectors.push(data.slice(i * dim, (i + 1) * dim));
|
|
117
|
+
}
|
|
118
|
+
return vectors;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
export const MODEL = MODEL_ID;
|
|
122
|
+
export const EMBED_DIM = TARGET_DIM;
|