notoken-core 1.6.0 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config/ascii-art.json +12 -0
- package/config/chat-responses.json +1019 -0
- package/config/cheat-sheets.json +94 -0
- package/config/concept-clusters.json +31 -0
- package/config/daily-tips.json +105 -0
- package/config/entities.json +93 -0
- package/config/history-today.json +9762 -0
- package/config/image-prompts.json +20 -0
- package/config/intent-vectors.json +1 -0
- package/config/intents.json +5749 -85
- package/config/ollama-models.json +193 -0
- package/config/rules.json +32 -1
- package/config/startup-quotes.json +45 -0
- package/dist/automation/discordPatchright.d.ts +35 -0
- package/dist/automation/discordPatchright.js +437 -0
- package/dist/automation/discordSetup.d.ts +31 -0
- package/dist/automation/discordSetup.js +338 -0
- package/dist/automation/smAutomation.d.ts +82 -0
- package/dist/automation/smAutomation.js +448 -0
- package/dist/conversation/coreference.js +44 -4
- package/dist/conversation/pendingActions.d.ts +55 -0
- package/dist/conversation/pendingActions.js +127 -0
- package/dist/conversation/store.d.ts +72 -0
- package/dist/conversation/store.js +140 -1
- package/dist/conversation/topicTracker.d.ts +36 -0
- package/dist/conversation/topicTracker.js +141 -0
- package/dist/execution/ssh.d.ts +42 -1
- package/dist/execution/ssh.js +538 -3
- package/dist/handlers/executor.d.ts +2 -0
- package/dist/handlers/executor.js +4669 -31
- package/dist/index.d.ts +39 -5
- package/dist/index.js +56 -4
- package/dist/nlp/batchParser.d.ts +30 -0
- package/dist/nlp/batchParser.js +77 -0
- package/dist/nlp/conceptExpansion.d.ts +54 -0
- package/dist/nlp/conceptExpansion.js +136 -0
- package/dist/nlp/conceptRouter.d.ts +49 -0
- package/dist/nlp/conceptRouter.js +302 -0
- package/dist/nlp/confidenceCalibrator.d.ts +62 -0
- package/dist/nlp/confidenceCalibrator.js +116 -0
- package/dist/nlp/correctionLearner.d.ts +45 -0
- package/dist/nlp/correctionLearner.js +207 -0
- package/dist/nlp/entitySpellCorrect.d.ts +35 -0
- package/dist/nlp/entitySpellCorrect.js +141 -0
- package/dist/nlp/knowledgeGraph.d.ts +70 -0
- package/dist/nlp/knowledgeGraph.js +380 -0
- package/dist/nlp/llmFallback.d.ts +47 -0
- package/dist/nlp/llmFallback.js +175 -36
- package/dist/nlp/llmParser.d.ts +5 -1
- package/dist/nlp/llmParser.js +43 -24
- package/dist/nlp/multiClassifier.js +91 -6
- package/dist/nlp/multiIntent.d.ts +43 -0
- package/dist/nlp/multiIntent.js +154 -0
- package/dist/nlp/parseIntent.d.ts +6 -1
- package/dist/nlp/parseIntent.js +199 -6
- package/dist/nlp/ruleParser.js +348 -0
- package/dist/nlp/semanticSimilarity.d.ts +30 -0
- package/dist/nlp/semanticSimilarity.js +174 -0
- package/dist/nlp/vocabularyBuilder.d.ts +43 -0
- package/dist/nlp/vocabularyBuilder.js +224 -0
- package/dist/nlp/wikidata.d.ts +49 -0
- package/dist/nlp/wikidata.js +228 -0
- package/dist/policy/confirm.d.ts +10 -0
- package/dist/policy/confirm.js +39 -0
- package/dist/policy/safety.js +6 -4
- package/dist/types/intent.d.ts +8 -0
- package/dist/types/intent.js +1 -0
- package/dist/utils/achievements.d.ts +38 -0
- package/dist/utils/achievements.js +126 -0
- package/dist/utils/aliases.d.ts +5 -0
- package/dist/utils/aliases.js +39 -0
- package/dist/utils/analysis.js +71 -15
- package/dist/utils/bookmarks.d.ts +13 -0
- package/dist/utils/bookmarks.js +51 -0
- package/dist/utils/browser.d.ts +64 -0
- package/dist/utils/browser.js +364 -0
- package/dist/utils/commandHistory.d.ts +20 -0
- package/dist/utils/commandHistory.js +108 -0
- package/dist/utils/completer.d.ts +17 -0
- package/dist/utils/completer.js +79 -0
- package/dist/utils/config.js +32 -2
- package/dist/utils/dbQuery.d.ts +25 -0
- package/dist/utils/dbQuery.js +248 -0
- package/dist/utils/devTools.d.ts +35 -0
- package/dist/utils/devTools.js +95 -0
- package/dist/utils/discordDiag.d.ts +35 -0
- package/dist/utils/discordDiag.js +834 -0
- package/dist/utils/diskCleanup.d.ts +36 -0
- package/dist/utils/diskCleanup.js +775 -0
- package/dist/utils/entityResolver.d.ts +107 -0
- package/dist/utils/entityResolver.js +468 -0
- package/dist/utils/imageGen.d.ts +92 -0
- package/dist/utils/imageGen.js +2031 -0
- package/dist/utils/installTracker.d.ts +57 -0
- package/dist/utils/installTracker.js +160 -0
- package/dist/utils/multiExec.d.ts +21 -0
- package/dist/utils/multiExec.js +141 -0
- package/dist/utils/openclawDiag.d.ts +127 -0
- package/dist/utils/openclawDiag.js +1535 -0
- package/dist/utils/openclawLogParser.d.ts +65 -0
- package/dist/utils/openclawLogParser.js +168 -0
- package/dist/utils/output.js +4 -0
- package/dist/utils/platform.js +2 -1
- package/dist/utils/progressReporter.d.ts +50 -0
- package/dist/utils/progressReporter.js +58 -0
- package/dist/utils/projectDetect.d.ts +44 -0
- package/dist/utils/projectDetect.js +319 -0
- package/dist/utils/projectScanner.d.ts +44 -0
- package/dist/utils/projectScanner.js +312 -0
- package/dist/utils/shellCompat.d.ts +78 -0
- package/dist/utils/shellCompat.js +186 -0
- package/dist/utils/smartArchive.d.ts +16 -0
- package/dist/utils/smartArchive.js +172 -0
- package/dist/utils/smartRetry.d.ts +26 -0
- package/dist/utils/smartRetry.js +114 -0
- package/dist/utils/snippets.d.ts +13 -0
- package/dist/utils/snippets.js +53 -0
- package/dist/utils/stabilityMatrixManager.d.ts +80 -0
- package/dist/utils/stabilityMatrixManager.js +268 -0
- package/dist/utils/teachMode.d.ts +41 -0
- package/dist/utils/teachMode.js +100 -0
- package/dist/utils/timer.d.ts +22 -0
- package/dist/utils/timer.js +52 -0
- package/dist/utils/updater.d.ts +1 -0
- package/dist/utils/updater.js +1 -1
- package/dist/utils/userContext.d.ts +57 -0
- package/dist/utils/userContext.js +133 -0
- package/dist/utils/version.d.ts +20 -0
- package/dist/utils/version.js +212 -0
- package/package.json +6 -3
package/dist/nlp/llmFallback.js
CHANGED
|
@@ -10,6 +10,18 @@
|
|
|
10
10
|
import { execSync } from "node:child_process";
|
|
11
11
|
import { loadIntents } from "../utils/config.js";
|
|
12
12
|
import { detectLocalPlatform } from "../utils/platform.js";
|
|
13
|
+
/** Conversation history for multi-turn LLM disambiguation */
|
|
14
|
+
const _llmConversation = [];
|
|
15
|
+
/** Add a turn to the LLM conversation for multi-turn context */
|
|
16
|
+
export function addLLMContext(role, content) {
|
|
17
|
+
_llmConversation.push({ role, content });
|
|
18
|
+
if (_llmConversation.length > 10)
|
|
19
|
+
_llmConversation.splice(0, _llmConversation.length - 10);
|
|
20
|
+
}
|
|
21
|
+
/** Clear LLM conversation when topic changes */
|
|
22
|
+
export function clearLLMContext() { _llmConversation.length = 0; }
|
|
23
|
+
/** Get conversation for context in multi-turn */
|
|
24
|
+
export function getLLMContext() { return [..._llmConversation]; }
|
|
13
25
|
/**
|
|
14
26
|
* Check if any LLM is configured.
|
|
15
27
|
*/
|
|
@@ -18,7 +30,7 @@ import { detectLocalPlatform } from "../utils/platform.js";
|
|
|
18
30
|
* Order: explicit config → auto-detect Ollama → nothing.
|
|
19
31
|
*/
|
|
20
32
|
export function isLLMConfigured() {
|
|
21
|
-
return !!(process.env.NOTOKEN_LLM_ENDPOINT || process.env.NOTOKEN_LLM_CLI || detectOllama());
|
|
33
|
+
return !!(process.env.NOTOKEN_LLM_ENDPOINT || process.env.NOTOKEN_LLM_CLI || detectOllama() || detectCodex());
|
|
22
34
|
}
|
|
23
35
|
/** Which LLM backend is active? */
|
|
24
36
|
export function getLLMBackend() {
|
|
@@ -28,8 +40,25 @@ export function getLLMBackend() {
|
|
|
28
40
|
return "api";
|
|
29
41
|
if (detectOllama())
|
|
30
42
|
return "ollama";
|
|
43
|
+
if (detectCodex())
|
|
44
|
+
return "codex";
|
|
31
45
|
return null;
|
|
32
46
|
}
|
|
47
|
+
let codexChecked = false;
|
|
48
|
+
let codexAvailable = false;
|
|
49
|
+
function detectCodex() {
|
|
50
|
+
if (codexChecked)
|
|
51
|
+
return codexAvailable;
|
|
52
|
+
codexChecked = true;
|
|
53
|
+
try {
|
|
54
|
+
execSync("command -v codex", { timeout: 1000, stdio: "pipe" });
|
|
55
|
+
codexAvailable = true;
|
|
56
|
+
}
|
|
57
|
+
catch {
|
|
58
|
+
codexAvailable = false;
|
|
59
|
+
}
|
|
60
|
+
return codexAvailable;
|
|
61
|
+
}
|
|
33
62
|
let ollamaChecked = false;
|
|
34
63
|
let ollamaAvailable = false;
|
|
35
64
|
function detectOllama() {
|
|
@@ -67,6 +96,12 @@ export async function llmFallback(rawText, context) {
|
|
|
67
96
|
if (apiResult)
|
|
68
97
|
return apiResult;
|
|
69
98
|
}
|
|
99
|
+
// Try Codex (auto-detected local)
|
|
100
|
+
if (detectCodex()) {
|
|
101
|
+
const codexResult = await tryLLMCli(rawText, { ...context, _cli: "codex" });
|
|
102
|
+
if (codexResult)
|
|
103
|
+
return codexResult;
|
|
104
|
+
}
|
|
70
105
|
// Try Ollama (auto-detected local)
|
|
71
106
|
if (detectOllama()) {
|
|
72
107
|
const ollamaResult = await tryOllama(rawText, context);
|
|
@@ -75,6 +110,55 @@ export async function llmFallback(rawText, context) {
|
|
|
75
110
|
}
|
|
76
111
|
return null;
|
|
77
112
|
}
|
|
113
|
+
/**
|
|
114
|
+
* Multi-turn LLM disambiguation.
|
|
115
|
+
*
|
|
116
|
+
* 1. Ask the LLM what to do
|
|
117
|
+
* 2. If it needs more info → run gatherCommands → feed results back
|
|
118
|
+
* 3. Repeat up to maxTurns times
|
|
119
|
+
* 4. Return the final result
|
|
120
|
+
*/
|
|
121
|
+
export async function llmMultiTurn(rawText, context, options) {
|
|
122
|
+
const maxTurns = options?.maxTurns ?? 3;
|
|
123
|
+
const onProgress = options?.onProgress ?? (() => { });
|
|
124
|
+
// Turn 1: initial LLM call
|
|
125
|
+
onProgress("Asking LLM to interpret...");
|
|
126
|
+
let result = await llmFallback(rawText, context);
|
|
127
|
+
if (!result)
|
|
128
|
+
return null;
|
|
129
|
+
// Multi-turn loop: if LLM needs more info, run commands and ask again
|
|
130
|
+
for (let turn = 1; turn < maxTurns && result?.needsMoreInfo && result.gatherCommands?.length; turn++) {
|
|
131
|
+
const commands = result.gatherCommands;
|
|
132
|
+
onProgress(`Turn ${turn + 1}: Running ${commands.length} command(s) to gather info...`);
|
|
133
|
+
// Run each gather command
|
|
134
|
+
const commandResults = [];
|
|
135
|
+
for (const cmd of commands) {
|
|
136
|
+
onProgress(` Running: ${cmd.command}`);
|
|
137
|
+
try {
|
|
138
|
+
const { execSync } = await import("node:child_process");
|
|
139
|
+
const output = execSync(cmd.command, {
|
|
140
|
+
encoding: "utf-8",
|
|
141
|
+
timeout: 15_000,
|
|
142
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
143
|
+
}).trim();
|
|
144
|
+
commandResults.push({ ...cmd, output: output.substring(0, 1000) });
|
|
145
|
+
}
|
|
146
|
+
catch (err) {
|
|
147
|
+
const e = err;
|
|
148
|
+
commandResults.push({ ...cmd, output: `ERROR: ${e.message?.split("\n")[0] ?? "failed"}` });
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
// Build follow-up prompt with command outputs
|
|
152
|
+
const outputSummary = commandResults.map(r => `Command: ${r.command}\nPurpose: ${r.purpose}\nOutput:\n${r.output}`).join("\n\n");
|
|
153
|
+
const followUpText = `${rawText}\n\n--- COMMAND OUTPUTS ---\n${outputSummary}\n\nBased on these results, complete this JSON template. Replace FILL with values:\n\`\`\`json\n{"understood": FILL, "restatement": "FILL", "suggestedIntents": [{"intent": "FILL", "fields": {}, "confidence": FILL, "reasoning": "FILL"}], "needsMoreInfo": false}\n\`\`\``;
|
|
154
|
+
onProgress("Analyzing results...");
|
|
155
|
+
result = await llmFallback(followUpText, context);
|
|
156
|
+
// Track conversation
|
|
157
|
+
addLLMContext("user", rawText);
|
|
158
|
+
addLLMContext("assistant", JSON.stringify(commandResults));
|
|
159
|
+
}
|
|
160
|
+
return result;
|
|
161
|
+
}
|
|
78
162
|
async function tryLLMCli(rawText, context) {
|
|
79
163
|
const cli = process.env.NOTOKEN_LLM_CLI;
|
|
80
164
|
if (!cli)
|
|
@@ -91,6 +175,10 @@ async function tryLLMCli(rawText, context) {
|
|
|
91
175
|
execSync("command -v chatgpt", { stdio: "pipe" });
|
|
92
176
|
cmd = `chatgpt ${JSON.stringify(prompt)}`;
|
|
93
177
|
}
|
|
178
|
+
else if (cli === "codex") {
|
|
179
|
+
execSync("command -v codex", { stdio: "pipe" });
|
|
180
|
+
cmd = `codex ${JSON.stringify(prompt)}`;
|
|
181
|
+
}
|
|
94
182
|
else {
|
|
95
183
|
return null;
|
|
96
184
|
}
|
|
@@ -166,16 +254,21 @@ async function tryOllama(rawText, context) {
|
|
|
166
254
|
const prompt = buildPrompt(rawText, context);
|
|
167
255
|
const model = process.env.NOTOKEN_OLLAMA_MODEL ?? "llama3.2";
|
|
168
256
|
try {
|
|
169
|
-
const
|
|
257
|
+
const controller = new AbortController();
|
|
258
|
+
const timeout = setTimeout(() => controller.abort(), 90_000);
|
|
259
|
+
// Use 127.0.0.1 explicitly — Node 18 fetch resolves localhost to IPv6 ::1 which Ollama doesn't listen on
|
|
260
|
+
const response = await fetch("http://127.0.0.1:11434/api/generate", {
|
|
170
261
|
method: "POST",
|
|
171
262
|
headers: { "Content-Type": "application/json" },
|
|
263
|
+
signal: controller.signal,
|
|
172
264
|
body: JSON.stringify({
|
|
173
265
|
model,
|
|
174
266
|
prompt,
|
|
175
267
|
stream: false,
|
|
176
|
-
options: { temperature: 0.1, num_predict:
|
|
268
|
+
options: { temperature: 0.1, num_predict: 512 },
|
|
177
269
|
}),
|
|
178
270
|
});
|
|
271
|
+
clearTimeout(timeout);
|
|
179
272
|
if (!response.ok)
|
|
180
273
|
return null;
|
|
181
274
|
const data = (await response.json());
|
|
@@ -184,7 +277,8 @@ async function tryOllama(rawText, context) {
|
|
|
184
277
|
return null;
|
|
185
278
|
return parseResponse(text);
|
|
186
279
|
}
|
|
187
|
-
catch {
|
|
280
|
+
catch (err) {
|
|
281
|
+
console.error(`\x1b[2m[llm-ollama] ${err.message?.substring(0, 100)}\x1b[0m`);
|
|
188
282
|
return null;
|
|
189
283
|
}
|
|
190
284
|
}
|
|
@@ -207,7 +301,7 @@ export function isOllamaInstalled() {
|
|
|
207
301
|
*/
|
|
208
302
|
export async function getOllamaModels() {
|
|
209
303
|
try {
|
|
210
|
-
const response = await fetch("http://
|
|
304
|
+
const response = await fetch("http://127.0.0.1:11434/api/tags");
|
|
211
305
|
if (!response.ok)
|
|
212
306
|
return [];
|
|
213
307
|
const data = (await response.json());
|
|
@@ -219,50 +313,95 @@ export async function getOllamaModels() {
|
|
|
219
313
|
}
|
|
220
314
|
function buildPrompt(rawText, context) {
|
|
221
315
|
const intents = loadIntents();
|
|
222
|
-
const intentSummary = intents.map((i) => {
|
|
223
|
-
const fields = Object.entries(i.fields)
|
|
224
|
-
.map(([k, v]) => `${k}:${v.type}${v.required ? "*" : ""}`)
|
|
225
|
-
.join(", ");
|
|
226
|
-
return ` ${i.name}: ${i.description} [${fields}]`;
|
|
227
|
-
}).join("\n");
|
|
228
316
|
const platform = detectLocalPlatform();
|
|
229
|
-
|
|
317
|
+
// Group intents by domain — much shorter than listing all 298
|
|
318
|
+
const domains = new Map();
|
|
319
|
+
for (const i of intents) {
|
|
320
|
+
const domain = i.name.split(".")[0];
|
|
321
|
+
if (!domains.has(domain))
|
|
322
|
+
domains.set(domain, []);
|
|
323
|
+
domains.get(domain).push(`${i.name}: ${i.description}`);
|
|
324
|
+
}
|
|
325
|
+
// Only include top-level summary + relevant domains based on user input
|
|
326
|
+
const relevantDomains = [];
|
|
327
|
+
const inputLower = rawText.toLowerCase();
|
|
328
|
+
for (const [domain, items] of domains) {
|
|
329
|
+
// Include domains that might be relevant to the query
|
|
330
|
+
const domainKeywords = {
|
|
331
|
+
service: ["service", "restart", "start", "stop", "status", "running"],
|
|
332
|
+
server: ["server", "cpu", "memory", "disk", "load", "uptime"],
|
|
333
|
+
docker: ["docker", "container", "image", "compose"],
|
|
334
|
+
network: ["network", "ip", "port", "ping", "dns", "curl", "speed"],
|
|
335
|
+
git: ["git", "commit", "push", "pull", "branch", "merge"],
|
|
336
|
+
deploy: ["deploy", "release", "rollback"],
|
|
337
|
+
logs: ["log", "error", "tail", "search"],
|
|
338
|
+
security: ["security", "attack", "firewall", "scan", "block"],
|
|
339
|
+
disk: ["disk", "space", "cleanup", "scan", "drive"],
|
|
340
|
+
db: ["database", "mysql", "postgres", "query", "sql"],
|
|
341
|
+
openclaw: ["openclaw", "claw", "gateway", "discord"],
|
|
342
|
+
ollama: ["ollama", "llm", "model"],
|
|
343
|
+
ai: ["image", "generate", "stable diffusion"],
|
|
344
|
+
files: ["file", "find", "copy", "move", "delete"],
|
|
345
|
+
process: ["process", "kill", "pid"],
|
|
346
|
+
user: ["user", "who", "login"],
|
|
347
|
+
backup: ["backup", "restore", "snapshot"],
|
|
348
|
+
notoken: ["notoken", "status", "version", "update", "help"],
|
|
349
|
+
};
|
|
350
|
+
const keywords = domainKeywords[domain] ?? [domain];
|
|
351
|
+
if (keywords.some(k => inputLower.includes(k)) || items.length <= 5) {
|
|
352
|
+
relevantDomains.push(`\n [${domain}] (${items.length} commands)\n${items.slice(0, 8).map(i => ` ${i}`).join("\n")}`);
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
// If no relevant domains found, include a general summary
|
|
356
|
+
if (relevantDomains.length === 0) {
|
|
357
|
+
for (const [domain, items] of [...domains].slice(0, 10)) {
|
|
358
|
+
relevantDomains.push(` [${domain}]: ${items.slice(0, 3).map(i => i.split(":")[0]).join(", ")}...`);
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
// Recent conversation context
|
|
362
|
+
const recentIntents = context.recentIntents ?? [];
|
|
363
|
+
const recentContext = recentIntents.length > 0
|
|
364
|
+
? `\nRECENT COMMANDS (what the user has been doing):\n ${recentIntents.slice(0, 5).join(", ")}\n`
|
|
365
|
+
: "";
|
|
366
|
+
// Known entities
|
|
367
|
+
const entities = context.knownEntities ?? [];
|
|
368
|
+
const entityContext = entities.length > 0
|
|
369
|
+
? `\nKNOWN ENTITIES:\n ${entities.slice(0, 10).map(e => `${e.entity} (${e.type})`).join(", ")}\n`
|
|
370
|
+
: "";
|
|
371
|
+
return `You are NoToken, a server operations CLI assistant. The user said something my NLP couldn't parse. Help me understand what they want.
|
|
230
372
|
|
|
231
373
|
ENVIRONMENT:
|
|
232
|
-
OS: ${platform.distro}${platform.isWSL ? " (WSL)" : ""}
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
Shell: ${platform.shell}
|
|
236
|
-
Package manager: ${platform.packageManager}
|
|
237
|
-
Init system: ${platform.initSystem}
|
|
238
|
-
|
|
374
|
+
OS: ${platform.distro}${platform.isWSL ? " (WSL)" : ""} | Shell: ${platform.shell}
|
|
375
|
+
Package manager: ${platform.packageManager} | Init: ${platform.initSystem}
|
|
376
|
+
${recentContext}${entityContext}
|
|
239
377
|
USER INPUT: "${rawText}"
|
|
240
378
|
|
|
241
|
-
|
|
242
|
-
${
|
|
379
|
+
AVAILABLE COMMANDS (grouped by domain — ${intents.length} total):
|
|
380
|
+
${relevantDomains.join("\n")}
|
|
243
381
|
|
|
244
|
-
|
|
245
|
-
${intentSummary}
|
|
382
|
+
Complete this JSON template. Replace every FILL with actual values based on the user's input. Remove fields you don't need. Output ONLY the completed JSON:
|
|
246
383
|
|
|
247
|
-
|
|
384
|
+
\`\`\`json
|
|
248
385
|
{
|
|
249
|
-
"understood":
|
|
250
|
-
"restatement": "
|
|
386
|
+
"understood": FILL_true_or_false,
|
|
387
|
+
"restatement": "FILL_what_user_wants_in_plain_english",
|
|
251
388
|
"suggestedIntents": [
|
|
252
|
-
{
|
|
253
|
-
"intent": "intent.name from list above",
|
|
254
|
-
"fields": { "field": "value" },
|
|
255
|
-
"confidence": 0.0-1.0,
|
|
256
|
-
"reasoning": "why this intent"
|
|
257
|
-
}
|
|
389
|
+
{"intent": "FILL_best_matching_command_from_list_above", "fields": {}, "confidence": FILL_0_to_1, "reasoning": "FILL_why_this_command"}
|
|
258
390
|
],
|
|
259
|
-
"
|
|
260
|
-
|
|
391
|
+
"needsMoreInfo": FILL_true_if_you_need_to_run_commands_first,
|
|
392
|
+
"gatherCommands": [
|
|
393
|
+
{"command": "FILL_shell_command_to_run", "purpose": "FILL_why_run_this"}
|
|
261
394
|
],
|
|
262
|
-
"
|
|
395
|
+
"shellCommands": ["FILL_raw_shell_command_if_no_intent_fits"],
|
|
396
|
+
"missingInfo": ["FILL_question_to_ask_user_if_unclear"]
|
|
263
397
|
}
|
|
398
|
+
\`\`\`
|
|
264
399
|
|
|
265
|
-
|
|
400
|
+
Rules:
|
|
401
|
+
- If one of my commands fits, use suggestedIntents and set needsMoreInfo=false.
|
|
402
|
+
- If you need to investigate first, set needsMoreInfo=true and list gatherCommands with real shell commands (uptime, df -h, ps aux, curl, etc).
|
|
403
|
+
- I will run those commands and send you the output for a second round.
|
|
404
|
+
- Remove empty arrays. Output ONLY the JSON.`;
|
|
266
405
|
}
|
|
267
406
|
function parseResponse(raw) {
|
|
268
407
|
try {
|
package/dist/nlp/llmParser.d.ts
CHANGED
|
@@ -5,4 +5,8 @@ import type { DynamicIntent } from "../types/intent.js";
|
|
|
5
5
|
* Sends the raw text + context to an LLM and asks for structured JSON.
|
|
6
6
|
* Set NOTOKEN_LLM_ENDPOINT and optionally NOTOKEN_LLM_API_KEY in env.
|
|
7
7
|
*/
|
|
8
|
-
export declare function parseByLLM(rawText: string
|
|
8
|
+
export declare function parseByLLM(rawText: string, nearMisses?: Array<{
|
|
9
|
+
intent: string;
|
|
10
|
+
score: number;
|
|
11
|
+
source: string;
|
|
12
|
+
}>): Promise<DynamicIntent | null>;
|
package/dist/nlp/llmParser.js
CHANGED
|
@@ -7,14 +7,14 @@ import { loadRules } from "../utils/config.js";
|
|
|
7
7
|
* Sends the raw text + context to an LLM and asks for structured JSON.
|
|
8
8
|
* Set NOTOKEN_LLM_ENDPOINT and optionally NOTOKEN_LLM_API_KEY in env.
|
|
9
9
|
*/
|
|
10
|
-
export async function parseByLLM(rawText) {
|
|
10
|
+
export async function parseByLLM(rawText, nearMisses) {
|
|
11
11
|
const endpoint = process.env.NOTOKEN_LLM_ENDPOINT;
|
|
12
12
|
if (!endpoint)
|
|
13
13
|
return null;
|
|
14
14
|
const apiKey = process.env.NOTOKEN_LLM_API_KEY ?? "";
|
|
15
15
|
const rules = loadRules();
|
|
16
16
|
const intents = loadIntents();
|
|
17
|
-
const systemPrompt = buildSystemPrompt(intents, rules);
|
|
17
|
+
const systemPrompt = buildSystemPrompt(intents, rules, nearMisses);
|
|
18
18
|
const userPrompt = `Parse this command into structured intent JSON:\n\n"${rawText}"`;
|
|
19
19
|
try {
|
|
20
20
|
const response = await fetch(endpoint, {
|
|
@@ -59,35 +59,54 @@ export async function parseByLLM(rawText) {
|
|
|
59
59
|
return null;
|
|
60
60
|
}
|
|
61
61
|
}
|
|
62
|
-
function buildSystemPrompt(intents, rules) {
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
.
|
|
62
|
+
function buildSystemPrompt(intents, rules, nearMisses) {
|
|
63
|
+
// Build a concise intent list grouped by domain
|
|
64
|
+
const domains = new Map();
|
|
65
|
+
for (const i of intents) {
|
|
66
|
+
const domain = i.name.split(".")[0];
|
|
67
|
+
if (!domains.has(domain))
|
|
68
|
+
domains.set(domain, []);
|
|
69
|
+
const fields = Object.entries(i.fields).map(([k, v]) => `${k}:${v.type}`).join(", ");
|
|
70
|
+
domains.get(domain).push(`${i.name}${fields ? ` [${fields}]` : ""}`);
|
|
71
|
+
}
|
|
72
|
+
// If we have near-misses, show their full details + related intents
|
|
73
|
+
let nearMissSection = "";
|
|
74
|
+
if (nearMisses && nearMisses.length > 0) {
|
|
75
|
+
const nearMissDetails = nearMisses
|
|
76
|
+
.filter((v, i, a) => a.findIndex(x => x.intent === v.intent) === i) // dedup
|
|
77
|
+
.slice(0, 5)
|
|
78
|
+
.map(nm => {
|
|
79
|
+
const def = intents.find(i => i.name === nm.intent);
|
|
80
|
+
const fields = def ? Object.entries(def.fields).map(([k, v]) => `${k}:${v.type}`).join(", ") : "";
|
|
81
|
+
return ` - ${nm.intent} (${(nm.score * 100).toFixed(0)}% from ${nm.source}): ${def?.description ?? ""}${fields ? ` [${fields}]` : ""}`;
|
|
82
|
+
}).join("\n");
|
|
83
|
+
// Also include related intents from the same domains
|
|
84
|
+
const nearDomains = new Set(nearMisses.map(nm => nm.intent.split(".")[0]));
|
|
85
|
+
const relatedIntents = [...nearDomains].flatMap(d => (domains.get(d) ?? []).slice(0, 5)).join("\n ");
|
|
86
|
+
nearMissSection = `\nNEAR MATCHES (my classifiers think it might be one of these — pick the best or suggest another):
|
|
87
|
+
${nearMissDetails}
|
|
88
|
+
|
|
89
|
+
RELATED COMMANDS in those domains:
|
|
90
|
+
${relatedIntents}\n`;
|
|
91
|
+
}
|
|
92
|
+
// Compact domain summary for everything else
|
|
93
|
+
const domainSummary = [...domains].map(([d, items]) => ` [${d}]: ${items.slice(0, 4).join(", ")}${items.length > 4 ? ` +${items.length - 4} more` : ""}`).join("\n");
|
|
71
94
|
const envs = Object.keys(rules.environmentAliases).join(", ");
|
|
72
95
|
const services = Object.keys(rules.serviceAliases).join(", ");
|
|
73
|
-
return `You are
|
|
74
|
-
Parse the user's natural language
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
${
|
|
96
|
+
return `You are NoToken, a server operations CLI command parser.
|
|
97
|
+
Parse the user's natural language into a structured JSON intent.
|
|
98
|
+
${nearMissSection}
|
|
99
|
+
ALL AVAILABLE COMMANDS (${intents.length} total):
|
|
100
|
+
${domainSummary}
|
|
78
101
|
|
|
79
102
|
Known environments: ${envs}
|
|
80
103
|
Known services: ${services}
|
|
81
104
|
|
|
82
|
-
Return ONLY valid JSON
|
|
83
|
-
|
|
84
|
-
- "confidence": 0.0 to 1.0
|
|
85
|
-
- "fields": object with all relevant fields for that intent
|
|
86
|
-
|
|
87
|
-
Example: {"intent": "service.restart", "confidence": 0.9, "fields": {"service": "nginx", "environment": "prod"}}
|
|
105
|
+
Return ONLY valid JSON:
|
|
106
|
+
{"intent": "domain.command", "confidence": 0.0-1.0, "fields": {"field": "value"}}
|
|
88
107
|
|
|
89
|
-
If
|
|
90
|
-
Return ONLY
|
|
108
|
+
If unclear, return: {"intent": "unknown", "confidence": 0.1, "fields": {"reason": "...", "clarification": "What did you mean? Did you want to..."}}
|
|
109
|
+
Return ONLY JSON, no markdown.`;
|
|
91
110
|
}
|
|
92
111
|
function extractContent(data) {
|
|
93
112
|
if (data.choices && Array.isArray(data.choices)) {
|
|
@@ -1,9 +1,14 @@
|
|
|
1
1
|
import { loadIntents, loadRules } from "../utils/config.js";
|
|
2
2
|
import { semanticParse, fuzzyMatch } from "./semantic.js";
|
|
3
3
|
import { parseByRules } from "./ruleParser.js";
|
|
4
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
5
|
+
import { resolve, dirname } from "node:path";
|
|
6
|
+
import { fileURLToPath } from "node:url";
|
|
7
|
+
import { expandQuery } from "./conceptExpansion.js";
|
|
4
8
|
const CLASSIFIER_WEIGHTS = {
|
|
5
9
|
synonym: 1.0,
|
|
6
10
|
semantic: 0.8,
|
|
11
|
+
vector: 0.7,
|
|
7
12
|
context: 0.6,
|
|
8
13
|
fuzzy: 0.5,
|
|
9
14
|
};
|
|
@@ -12,8 +17,24 @@ const CLASSIFIER_WEIGHTS = {
|
|
|
12
17
|
*/
|
|
13
18
|
export function classifyMulti(rawText, recentIntents) {
|
|
14
19
|
const votes = [];
|
|
15
|
-
//
|
|
20
|
+
// 0. Expand query with synonym clusters for better matching
|
|
21
|
+
// "reboot the server" → "reboot the server restart cycle reload bounce"
|
|
22
|
+
let expandedText = rawText;
|
|
23
|
+
try {
|
|
24
|
+
expandedText = expandQuery(rawText);
|
|
25
|
+
}
|
|
26
|
+
catch { /* concept expansion not available */ }
|
|
27
|
+
// 1. Synonym classifier — run on both original AND expanded text
|
|
16
28
|
votes.push(...classifySynonym(rawText));
|
|
29
|
+
if (expandedText !== rawText) {
|
|
30
|
+
// Run again on expanded text but with lower weight
|
|
31
|
+
const expandedVotes = classifySynonym(expandedText);
|
|
32
|
+
for (const v of expandedVotes) {
|
|
33
|
+
v.confidence *= 0.7; // Expansion matches are less certain
|
|
34
|
+
v.reason += " (expanded)";
|
|
35
|
+
}
|
|
36
|
+
votes.push(...expandedVotes);
|
|
37
|
+
}
|
|
17
38
|
// 2. Semantic classifier (compromise-powered)
|
|
18
39
|
votes.push(...classifySemantic(rawText));
|
|
19
40
|
// 3. Context classifier (recent history)
|
|
@@ -22,19 +43,23 @@ export function classifyMulti(rawText, recentIntents) {
|
|
|
22
43
|
}
|
|
23
44
|
// 4. Fuzzy classifier (keyboard distance)
|
|
24
45
|
votes.push(...classifyFuzzy(rawText));
|
|
25
|
-
//
|
|
46
|
+
// 5. Vector classifier (precomputed TF-IDF cosine similarity)
|
|
47
|
+
votes.push(...classifyVector(rawText));
|
|
48
|
+
// Merge votes: max weighted score + bonus for agreement
|
|
26
49
|
const scoreMap = new Map();
|
|
27
50
|
for (const vote of votes) {
|
|
28
51
|
const weight = CLASSIFIER_WEIGHTS[vote.classifier] ?? 1.0;
|
|
29
|
-
const
|
|
30
|
-
existing
|
|
52
|
+
const weighted = vote.confidence * weight;
|
|
53
|
+
const existing = scoreMap.get(vote.intent) ?? { maxWeighted: 0, totalWeighted: 0, count: 0 };
|
|
54
|
+
existing.maxWeighted = Math.max(existing.maxWeighted, weighted);
|
|
55
|
+
existing.totalWeighted += weighted;
|
|
31
56
|
existing.count += 1;
|
|
32
57
|
scoreMap.set(vote.intent, existing);
|
|
33
58
|
}
|
|
34
59
|
const scores = Array.from(scoreMap.entries())
|
|
35
|
-
.map(([intent, {
|
|
60
|
+
.map(([intent, { maxWeighted, count }]) => ({
|
|
36
61
|
intent,
|
|
37
|
-
score:
|
|
62
|
+
score: maxWeighted + Math.min(0.15, (count - 1) * 0.05),
|
|
38
63
|
votes: count,
|
|
39
64
|
}))
|
|
40
65
|
.sort((a, b) => b.score - a.score);
|
|
@@ -179,3 +204,63 @@ function scoreEntityMatch(parse, def) {
|
|
|
179
204
|
}
|
|
180
205
|
return matches / total;
|
|
181
206
|
}
|
|
207
|
+
let _vectorData = null;
|
|
208
|
+
function loadVectors() {
|
|
209
|
+
if (_vectorData)
|
|
210
|
+
return _vectorData;
|
|
211
|
+
const paths = [
|
|
212
|
+
resolve(dirname(fileURLToPath(import.meta.url)), "../../config/intent-vectors.json"),
|
|
213
|
+
resolve(process.cwd(), "config/intent-vectors.json"),
|
|
214
|
+
];
|
|
215
|
+
for (const p of paths) {
|
|
216
|
+
if (existsSync(p)) {
|
|
217
|
+
try {
|
|
218
|
+
_vectorData = JSON.parse(readFileSync(p, "utf-8"));
|
|
219
|
+
return _vectorData;
|
|
220
|
+
}
|
|
221
|
+
catch { /* skip */ }
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
return null;
|
|
225
|
+
}
|
|
226
|
+
const VECTOR_STOP = new Set(["a", "an", "the", "is", "it", "in", "on", "to", "for", "of", "and", "or", "my", "me", "i", "we", "you", "do", "does", "did", "be", "am", "are", "was", "were", "have", "has", "had", "this", "that", "what", "which", "who", "how", "where", "when", "why", "not", "no", "but", "if", "so", "at", "by", "with", "from", "up", "out", "can", "could", "would", "should", "will", "may", "might", "just", "about", "all", "please"]);
|
|
227
|
+
function classifyVector(rawText) {
|
|
228
|
+
const data = loadVectors();
|
|
229
|
+
if (!data)
|
|
230
|
+
return [];
|
|
231
|
+
const tokens = rawText.toLowerCase().replace(/[^a-z0-9_.\-\/]/g, " ").split(/\s+/).filter((w) => w.length > 1 && !VECTOR_STOP.has(w));
|
|
232
|
+
if (tokens.length === 0)
|
|
233
|
+
return [];
|
|
234
|
+
const vocabIndex = new Map(data.vocab.map((v, i) => [v, i]));
|
|
235
|
+
const inputVec = {};
|
|
236
|
+
let magnitude = 0;
|
|
237
|
+
const tf = new Map();
|
|
238
|
+
for (const t of tokens)
|
|
239
|
+
tf.set(t, (tf.get(t) ?? 0) + 1);
|
|
240
|
+
for (const [term, count] of tf) {
|
|
241
|
+
const idx = vocabIndex.get(term);
|
|
242
|
+
if (idx !== undefined) {
|
|
243
|
+
inputVec[idx] = count;
|
|
244
|
+
magnitude += count * count;
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
magnitude = Math.sqrt(magnitude);
|
|
248
|
+
if (magnitude === 0)
|
|
249
|
+
return [];
|
|
250
|
+
for (const idx of Object.keys(inputVec))
|
|
251
|
+
inputVec[Number(idx)] /= magnitude;
|
|
252
|
+
const votes = [];
|
|
253
|
+
for (const [intentName, intentVec] of Object.entries(data.vectors)) {
|
|
254
|
+
let dot = 0;
|
|
255
|
+
for (const [idx, val] of Object.entries(inputVec)) {
|
|
256
|
+
const iv = intentVec[idx];
|
|
257
|
+
if (iv)
|
|
258
|
+
dot += val * iv;
|
|
259
|
+
}
|
|
260
|
+
if (dot > 0.1) {
|
|
261
|
+
votes.push({ classifier: "vector", intent: intentName, confidence: Math.min(0.95, dot), reason: `TF-IDF cosine: ${dot.toFixed(3)}` });
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
votes.sort((a, b) => b.confidence - a.confidence);
|
|
265
|
+
return votes.slice(0, 3);
|
|
266
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Multi-intent parser.
|
|
3
|
+
*
|
|
4
|
+
* Splits compound sentences into individual intents and creates a plan.
|
|
5
|
+
*
|
|
6
|
+
* "check if the firewall is blocking port 443 and also check dns for my domain"
|
|
7
|
+
* → Step 1: firewall.list (check port 443)
|
|
8
|
+
* Step 2: dns.lookup (check domain)
|
|
9
|
+
*
|
|
10
|
+
* "show me disk usage, check memory, and list running containers"
|
|
11
|
+
* → Step 1: server.check_disk
|
|
12
|
+
* Step 2: server.check_memory
|
|
13
|
+
* Step 3: docker.list
|
|
14
|
+
*
|
|
15
|
+
* Splitting rules:
|
|
16
|
+
* - Split on: "and", "also", "then", "after that", ",", ";"
|
|
17
|
+
* - But NOT inside quoted strings or after "and" that joins nouns ("cats and dogs")
|
|
18
|
+
* - Each part is parsed independently through rule parser + concept router
|
|
19
|
+
* - Only creates a plan if 2+ distinct intents are found
|
|
20
|
+
*/
|
|
21
|
+
export interface PlanStep {
|
|
22
|
+
intent: string;
|
|
23
|
+
rawText: string;
|
|
24
|
+
confidence: number;
|
|
25
|
+
description: string;
|
|
26
|
+
requiresConfirmation: boolean;
|
|
27
|
+
riskLevel: string;
|
|
28
|
+
}
|
|
29
|
+
export interface MultiIntentPlan {
|
|
30
|
+
steps: PlanStep[];
|
|
31
|
+
originalText: string;
|
|
32
|
+
isSingleIntent: boolean;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Split a compound sentence into parts.
|
|
36
|
+
*/
|
|
37
|
+
export declare function splitCompoundSentence(text: string): string[];
|
|
38
|
+
/**
|
|
39
|
+
* Parse a potentially compound sentence into a multi-step plan.
|
|
40
|
+
* Returns a single-step plan if only one intent is found.
|
|
41
|
+
*/
|
|
42
|
+
export declare function parseMultiIntent(rawText: string): MultiIntentPlan;
|
|
43
|
+
export declare function formatPlanSteps(plan: MultiIntentPlan): string;
|