@otto-code/brain 0.8.2 → 0.8.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/lifecycle.js +16 -2
- package/dist/config/schema.d.ts +12 -0
- package/dist/config/schema.js +1 -0
- package/dist/config/store.js +24 -2
- package/dist/models/download.js +15 -5
- package/dist/models/enrich.js +1 -0
- package/dist/service/activity.d.ts +22 -2
- package/dist/service/activity.js +109 -23
- package/dist/service/host-api.d.ts +26 -0
- package/dist/service/host-api.js +88 -1
- package/dist/service/router.d.ts +19 -4
- package/dist/service/router.js +91 -39
- package/dist/service/scheduler.d.ts +9 -1
- package/dist/service/scheduler.js +15 -3
- package/dist/service/serve.d.ts +2 -1
- package/dist/service/serve.js +63 -24
- package/dist/service/status-events.d.ts +88 -0
- package/dist/service/status-events.js +256 -0
- package/dist/service/supervisor.d.ts +2 -2
- package/dist/service/supervisor.js +11 -5
- package/dist/sysmon.d.ts +1 -1
- package/dist/sysmon.js +43 -8
- package/dist/types.d.ts +2 -0
- package/package.json +2 -2
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* they are always explicit user actions, never auto-started.
|
|
6
6
|
*/
|
|
7
7
|
import { spawn } from "node:child_process";
|
|
8
|
-
import { openSync } from "node:fs";
|
|
8
|
+
import { existsSync, openSync } from "node:fs";
|
|
9
9
|
import http from "node:http";
|
|
10
10
|
import https from "node:https";
|
|
11
11
|
import { loadBrainConfig } from "../config/index.js";
|
|
@@ -30,7 +30,9 @@ export async function runServeCommand(options, _command) {
|
|
|
30
30
|
});
|
|
31
31
|
const scheme = handle.secure ? "https" : "http";
|
|
32
32
|
process.stdout.write(`router listening on ${scheme}://${handle.displayHost}:${handle.port}\n`);
|
|
33
|
-
process.stdout.write(
|
|
33
|
+
process.stdout.write(handle.model
|
|
34
|
+
? `ready: ${handle.model.displayName}, ${vram.formatGiB(handle.supervisor.vramAtReadyBytes ?? 0)} VRAM in use\n`
|
|
35
|
+
: "ready: no model loaded; use the Library tab or `otto brain pull` to download one\n");
|
|
34
36
|
process.stdout.write("press Ctrl+C to stop\n");
|
|
35
37
|
const shutdown = async () => {
|
|
36
38
|
process.stdout.write("\nstopping…\n");
|
|
@@ -83,7 +85,19 @@ export async function runStartCommand(options, command) {
|
|
|
83
85
|
});
|
|
84
86
|
}
|
|
85
87
|
const { logFile } = resolveBrainPaths();
|
|
88
|
+
// process.argv[1] is the entry script of whatever host is running us (the npm
|
|
89
|
+
// CLI's bin, the desktop bundle's dist/index.js, bin/otto-brain). If it is not
|
|
90
|
+
// a file, we are running somewhere that does not lay argv out like Node - the
|
|
91
|
+
// detached child would silently get a verb where the script belongs and parse
|
|
92
|
+
// as garbage, so say so instead.
|
|
86
93
|
const entry = process.argv[1];
|
|
94
|
+
if (!entry || !existsSync(entry)) {
|
|
95
|
+
throw new CommandError({
|
|
96
|
+
code: "NO_ENTRYPOINT",
|
|
97
|
+
message: "cannot start the brain detached: this host does not expose a CLI entry script",
|
|
98
|
+
details: "run `otto brain serve` in the foreground, or use the npm CLI (npm i -g @otto-code/cli)",
|
|
99
|
+
});
|
|
100
|
+
}
|
|
87
101
|
const args = [...invocationVerbPrefix(command), "serve"];
|
|
88
102
|
if (options.model)
|
|
89
103
|
args.push("--model", options.model);
|
package/dist/config/schema.d.ts
CHANGED
|
@@ -818,6 +818,7 @@ export declare const CatalogModelSchema: z.ZodObject<{
|
|
|
818
818
|
moe: z.ZodOptional<z.ZodBoolean>;
|
|
819
819
|
vision: z.ZodOptional<z.ZodBoolean>;
|
|
820
820
|
thinking: z.ZodOptional<z.ZodBoolean>;
|
|
821
|
+
reasoningEfforts: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
821
822
|
contextMax: z.ZodOptional<z.ZodNumber>;
|
|
822
823
|
useCases: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
823
824
|
tier: z.ZodOptional<z.ZodString>;
|
|
@@ -835,6 +836,7 @@ export declare const CatalogModelSchema: z.ZodObject<{
|
|
|
835
836
|
moe: z.ZodOptional<z.ZodBoolean>;
|
|
836
837
|
vision: z.ZodOptional<z.ZodBoolean>;
|
|
837
838
|
thinking: z.ZodOptional<z.ZodBoolean>;
|
|
839
|
+
reasoningEfforts: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
838
840
|
contextMax: z.ZodOptional<z.ZodNumber>;
|
|
839
841
|
useCases: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
840
842
|
tier: z.ZodOptional<z.ZodString>;
|
|
@@ -852,6 +854,7 @@ export declare const CatalogModelSchema: z.ZodObject<{
|
|
|
852
854
|
moe: z.ZodOptional<z.ZodBoolean>;
|
|
853
855
|
vision: z.ZodOptional<z.ZodBoolean>;
|
|
854
856
|
thinking: z.ZodOptional<z.ZodBoolean>;
|
|
857
|
+
reasoningEfforts: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
855
858
|
contextMax: z.ZodOptional<z.ZodNumber>;
|
|
856
859
|
useCases: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
857
860
|
tier: z.ZodOptional<z.ZodString>;
|
|
@@ -876,6 +879,7 @@ export declare const CatalogSchema: z.ZodObject<{
|
|
|
876
879
|
moe: z.ZodOptional<z.ZodBoolean>;
|
|
877
880
|
vision: z.ZodOptional<z.ZodBoolean>;
|
|
878
881
|
thinking: z.ZodOptional<z.ZodBoolean>;
|
|
882
|
+
reasoningEfforts: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
879
883
|
contextMax: z.ZodOptional<z.ZodNumber>;
|
|
880
884
|
useCases: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
881
885
|
tier: z.ZodOptional<z.ZodString>;
|
|
@@ -893,6 +897,7 @@ export declare const CatalogSchema: z.ZodObject<{
|
|
|
893
897
|
moe: z.ZodOptional<z.ZodBoolean>;
|
|
894
898
|
vision: z.ZodOptional<z.ZodBoolean>;
|
|
895
899
|
thinking: z.ZodOptional<z.ZodBoolean>;
|
|
900
|
+
reasoningEfforts: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
896
901
|
contextMax: z.ZodOptional<z.ZodNumber>;
|
|
897
902
|
useCases: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
898
903
|
tier: z.ZodOptional<z.ZodString>;
|
|
@@ -910,6 +915,7 @@ export declare const CatalogSchema: z.ZodObject<{
|
|
|
910
915
|
moe: z.ZodOptional<z.ZodBoolean>;
|
|
911
916
|
vision: z.ZodOptional<z.ZodBoolean>;
|
|
912
917
|
thinking: z.ZodOptional<z.ZodBoolean>;
|
|
918
|
+
reasoningEfforts: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
913
919
|
contextMax: z.ZodOptional<z.ZodNumber>;
|
|
914
920
|
useCases: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
915
921
|
tier: z.ZodOptional<z.ZodString>;
|
|
@@ -933,6 +939,7 @@ export declare const CatalogSchema: z.ZodObject<{
|
|
|
933
939
|
moe: z.ZodOptional<z.ZodBoolean>;
|
|
934
940
|
vision: z.ZodOptional<z.ZodBoolean>;
|
|
935
941
|
thinking: z.ZodOptional<z.ZodBoolean>;
|
|
942
|
+
reasoningEfforts: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
936
943
|
contextMax: z.ZodOptional<z.ZodNumber>;
|
|
937
944
|
useCases: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
938
945
|
tier: z.ZodOptional<z.ZodString>;
|
|
@@ -950,6 +957,7 @@ export declare const CatalogSchema: z.ZodObject<{
|
|
|
950
957
|
moe: z.ZodOptional<z.ZodBoolean>;
|
|
951
958
|
vision: z.ZodOptional<z.ZodBoolean>;
|
|
952
959
|
thinking: z.ZodOptional<z.ZodBoolean>;
|
|
960
|
+
reasoningEfforts: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
953
961
|
contextMax: z.ZodOptional<z.ZodNumber>;
|
|
954
962
|
useCases: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
955
963
|
tier: z.ZodOptional<z.ZodString>;
|
|
@@ -967,6 +975,7 @@ export declare const CatalogSchema: z.ZodObject<{
|
|
|
967
975
|
moe: z.ZodOptional<z.ZodBoolean>;
|
|
968
976
|
vision: z.ZodOptional<z.ZodBoolean>;
|
|
969
977
|
thinking: z.ZodOptional<z.ZodBoolean>;
|
|
978
|
+
reasoningEfforts: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
970
979
|
contextMax: z.ZodOptional<z.ZodNumber>;
|
|
971
980
|
useCases: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
972
981
|
tier: z.ZodOptional<z.ZodString>;
|
|
@@ -990,6 +999,7 @@ export declare const CatalogSchema: z.ZodObject<{
|
|
|
990
999
|
moe: z.ZodOptional<z.ZodBoolean>;
|
|
991
1000
|
vision: z.ZodOptional<z.ZodBoolean>;
|
|
992
1001
|
thinking: z.ZodOptional<z.ZodBoolean>;
|
|
1002
|
+
reasoningEfforts: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
993
1003
|
contextMax: z.ZodOptional<z.ZodNumber>;
|
|
994
1004
|
useCases: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
995
1005
|
tier: z.ZodOptional<z.ZodString>;
|
|
@@ -1007,6 +1017,7 @@ export declare const CatalogSchema: z.ZodObject<{
|
|
|
1007
1017
|
moe: z.ZodOptional<z.ZodBoolean>;
|
|
1008
1018
|
vision: z.ZodOptional<z.ZodBoolean>;
|
|
1009
1019
|
thinking: z.ZodOptional<z.ZodBoolean>;
|
|
1020
|
+
reasoningEfforts: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
1010
1021
|
contextMax: z.ZodOptional<z.ZodNumber>;
|
|
1011
1022
|
useCases: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
1012
1023
|
tier: z.ZodOptional<z.ZodString>;
|
|
@@ -1024,6 +1035,7 @@ export declare const CatalogSchema: z.ZodObject<{
|
|
|
1024
1035
|
moe: z.ZodOptional<z.ZodBoolean>;
|
|
1025
1036
|
vision: z.ZodOptional<z.ZodBoolean>;
|
|
1026
1037
|
thinking: z.ZodOptional<z.ZodBoolean>;
|
|
1038
|
+
reasoningEfforts: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
1027
1039
|
contextMax: z.ZodOptional<z.ZodNumber>;
|
|
1028
1040
|
useCases: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
1029
1041
|
tier: z.ZodOptional<z.ZodString>;
|
package/dist/config/schema.js
CHANGED
|
@@ -172,6 +172,7 @@ export const CatalogModelSchema = z
|
|
|
172
172
|
moe: z.boolean().optional(),
|
|
173
173
|
vision: z.boolean().optional(),
|
|
174
174
|
thinking: z.boolean().optional(),
|
|
175
|
+
reasoningEfforts: z.array(z.string()).optional(),
|
|
175
176
|
contextMax: z.number().optional(),
|
|
176
177
|
useCases: z.array(z.string()).optional(),
|
|
177
178
|
tier: z.string().optional(),
|
package/dist/config/store.js
CHANGED
|
@@ -42,8 +42,30 @@ export function saveProfilesStore(store, paths = resolveBrainPaths()) {
|
|
|
42
42
|
// -------------------------------------------------------------------- catalog
|
|
43
43
|
export function loadCatalog(paths = resolveBrainPaths()) {
|
|
44
44
|
const current = readJson(paths.catalogFile, CatalogSchema);
|
|
45
|
-
if (current)
|
|
46
|
-
|
|
45
|
+
if (current) {
|
|
46
|
+
// The catalog is intentionally persisted so local Brain homes survive
|
|
47
|
+
// package updates, but additive metadata still needs to reach existing
|
|
48
|
+
// homes. Backfill only fields introduced by the seed catalog; user-owned
|
|
49
|
+
// catalog values remain authoritative.
|
|
50
|
+
const legacy = readJson(path.join(packageRoot(), "config", "downloads.json"), CatalogSchema);
|
|
51
|
+
if (!legacy)
|
|
52
|
+
return current;
|
|
53
|
+
const seedById = new Map(legacy.models.map((model) => [model.id, model]));
|
|
54
|
+
let changed = false;
|
|
55
|
+
const models = current.models.map((model) => {
|
|
56
|
+
const seed = seedById.get(model.id);
|
|
57
|
+
if (model.reasoningEfforts === undefined && seed?.reasoningEfforts !== undefined) {
|
|
58
|
+
changed = true;
|
|
59
|
+
return { ...model, reasoningEfforts: seed.reasoningEfforts };
|
|
60
|
+
}
|
|
61
|
+
return model;
|
|
62
|
+
});
|
|
63
|
+
if (!changed)
|
|
64
|
+
return current;
|
|
65
|
+
const merged = { ...current, models };
|
|
66
|
+
writeJson(paths.catalogFile, merged);
|
|
67
|
+
return merged;
|
|
68
|
+
}
|
|
47
69
|
const legacy = path.join(packageRoot(), "config", "downloads.json");
|
|
48
70
|
const migrated = readJson(legacy, CatalogSchema);
|
|
49
71
|
if (migrated) {
|
package/dist/models/download.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* `<managedModelsDir>/<publisher>/<repo>/<file>` to mirror the LM Studio layout
|
|
5
5
|
* the scanner already understands.
|
|
6
6
|
*/
|
|
7
|
-
import { createWriteStream, existsSync, mkdirSync } from "node:fs";
|
|
7
|
+
import { createWriteStream, existsSync, mkdirSync, rmSync } from "node:fs";
|
|
8
8
|
import path from "node:path";
|
|
9
9
|
import { Readable } from "node:stream";
|
|
10
10
|
import { pipeline } from "node:stream/promises";
|
|
@@ -41,6 +41,10 @@ function authHeaders(token) {
|
|
|
41
41
|
* killed download never leaves a truncated file that looks complete.
|
|
42
42
|
*/
|
|
43
43
|
async function streamRepoFile(url, destPath, label, token, onProgress, received) {
|
|
44
|
+
const tmp = `${destPath}.part`;
|
|
45
|
+
// Remove leftovers from an earlier interrupted attempt before starting a
|
|
46
|
+
// fresh request, including when this request fails before opening a stream.
|
|
47
|
+
rmSync(tmp, { force: true });
|
|
44
48
|
mkdirSync(path.dirname(destPath), { recursive: true });
|
|
45
49
|
if (existsSync(destPath))
|
|
46
50
|
return false;
|
|
@@ -54,10 +58,16 @@ async function streamRepoFile(url, destPath, label, token, onProgress, received)
|
|
|
54
58
|
received.bytes += chunk.length;
|
|
55
59
|
onProgress?.({ file: label, receivedBytes: received.bytes, totalBytes });
|
|
56
60
|
});
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
+
try {
|
|
62
|
+
await pipeline(body, createWriteStream(tmp));
|
|
63
|
+
const { renameSync } = await import("node:fs");
|
|
64
|
+
renameSync(tmp, destPath);
|
|
65
|
+
}
|
|
66
|
+
finally {
|
|
67
|
+
// Cancellation kills the CLI child while the stream is still writing. Do
|
|
68
|
+
// not leave a truncated `.part` behind for the next quant attempt.
|
|
69
|
+
rmSync(tmp, { force: true });
|
|
70
|
+
}
|
|
61
71
|
return true;
|
|
62
72
|
}
|
|
63
73
|
/** Download the model file; returns the local path it was written to. */
|
package/dist/models/enrich.js
CHANGED
|
@@ -58,6 +58,13 @@ export declare function withActivity<T>(kind: BrainActivityKind, options: {
|
|
|
58
58
|
*/
|
|
59
59
|
export declare function chunkHasReasoning(text: string): boolean;
|
|
60
60
|
export declare function chunkHasContent(text: string): boolean;
|
|
61
|
+
export type InferenceStage = "processing" | "thinking" | "generating";
|
|
62
|
+
export interface InferenceActivitySnapshot {
|
|
63
|
+
activeRequests: number;
|
|
64
|
+
processing: number;
|
|
65
|
+
thinking: number;
|
|
66
|
+
generating: number;
|
|
67
|
+
}
|
|
61
68
|
/**
|
|
62
69
|
* Which in-flight completions are currently mid-thought.
|
|
63
70
|
*
|
|
@@ -68,16 +75,29 @@ export declare function chunkHasContent(text: string): boolean;
|
|
|
68
75
|
* if more reasoning follows - a stream that is producing readable output should
|
|
69
76
|
* not report as though it were still silent.
|
|
70
77
|
*
|
|
71
|
-
* A
|
|
72
|
-
*
|
|
78
|
+
* A map rather than one global phase because llama-server runs several slots at
|
|
79
|
+
* once. One request can be processing a prompt while another thinks and a third
|
|
80
|
+
* generates content; the aggregate counts must preserve all three.
|
|
73
81
|
*/
|
|
74
82
|
export declare class ReasoningTracker {
|
|
75
83
|
#private;
|
|
84
|
+
/**
|
|
85
|
+
* Watch stage counts, not the per-chunk traffic behind them.
|
|
86
|
+
*
|
|
87
|
+
* The status event stream needs to publish request start, the moment a model
|
|
88
|
+
* goes silent to think and the moment it starts answering. Repeated chunks in
|
|
89
|
+
* one stage do not notify; slot sampling owns bounded token-rate updates.
|
|
90
|
+
*/
|
|
91
|
+
onChange(listener: () => void): () => void;
|
|
92
|
+
/** A completion was dispatched to llama-server and awaits its first output delta. */
|
|
93
|
+
begin(requestId: string): void;
|
|
76
94
|
/** Note a chunk of `requestId`'s stream. Cheap enough to call per chunk. */
|
|
77
95
|
observe(requestId: string, text: string): void;
|
|
78
96
|
/** Forget the request. Must be called on end *and* on error, or the flag sticks. */
|
|
79
97
|
end(requestId: string): void;
|
|
80
98
|
get active(): boolean;
|
|
81
99
|
get count(): number;
|
|
100
|
+
/** Aggregate request stages. Counts stay exact even with several parallel slots. */
|
|
101
|
+
get snapshot(): InferenceActivitySnapshot;
|
|
82
102
|
}
|
|
83
103
|
//# sourceMappingURL=activity.d.ts.map
|
package/dist/service/activity.js
CHANGED
|
@@ -3,10 +3,16 @@ var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (
|
|
|
3
3
|
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
|
|
4
4
|
return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
|
|
5
5
|
};
|
|
6
|
-
var
|
|
6
|
+
var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
|
|
7
|
+
if (kind === "m") throw new TypeError("Private method is not writable");
|
|
8
|
+
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
|
|
9
|
+
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
|
|
10
|
+
return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
|
|
11
|
+
};
|
|
12
|
+
var _ReasoningTracker_instances, _ReasoningTracker_requests, _ReasoningTracker_tails, _ReasoningTracker_inlineReasoning, _ReasoningTracker_listeners, _ReasoningTracker_lastSnapshot, _ReasoningTracker_announce;
|
|
7
13
|
/**
|
|
8
|
-
* What long-running work currently owns the brain, and
|
|
9
|
-
*
|
|
14
|
+
* What long-running work currently owns the brain, and which stage each live
|
|
15
|
+
* inference request has reached.
|
|
10
16
|
*
|
|
11
17
|
* Two trackers with deliberately different lifetimes:
|
|
12
18
|
*
|
|
@@ -18,11 +24,11 @@ var _ReasoningTracker_reasoning, _ReasoningTracker_content;
|
|
|
18
24
|
* was killed with Ctrl-C never gets to clean up after itself, and a status
|
|
19
25
|
* that stays stuck on "calibrating" forever is worse than no status at all.
|
|
20
26
|
*
|
|
21
|
-
* - **
|
|
27
|
+
* - **Inference** is per-request and lives only as long as the stream does, so
|
|
22
28
|
* it is plain in-process state on the router. It never touches disk.
|
|
23
29
|
*
|
|
24
|
-
* Both
|
|
25
|
-
*
|
|
30
|
+
* Both ride on host status: ops under `activity`, inference under `inference`.
|
|
31
|
+
* The client uses those independent signals to drive the Overview and rail.
|
|
26
32
|
*/
|
|
27
33
|
import { existsSync, readFileSync, rmSync } from "node:fs";
|
|
28
34
|
import { resolveBrainPaths } from "../config/paths.js";
|
|
@@ -164,10 +170,14 @@ function clampProgress(progress) {
|
|
|
164
170
|
* would cost more than the signal is worth.
|
|
165
171
|
*/
|
|
166
172
|
export function chunkHasReasoning(text) {
|
|
167
|
-
return
|
|
173
|
+
return (/"type"\s*:\s*"(?:thinking|reasoning)_delta"/u.test(text) ||
|
|
174
|
+
/"(?:thinking|reasoning_content)"\s*:\s*"[^"]/u.test(text));
|
|
168
175
|
}
|
|
169
176
|
export function chunkHasContent(text) {
|
|
170
|
-
return
|
|
177
|
+
return (/"type"\s*:\s*"text_delta"/u.test(text) ||
|
|
178
|
+
/"content"\s*:\s*"[^"]/u.test(text) ||
|
|
179
|
+
/"tool_calls"\s*:\s*\[\s*\{/u.test(text) ||
|
|
180
|
+
/"type"\s*:\s*"(?:tool_use|input_json_delta)"/u.test(text));
|
|
171
181
|
}
|
|
172
182
|
/**
|
|
173
183
|
* Which in-flight completions are currently mid-thought.
|
|
@@ -179,38 +189,114 @@ export function chunkHasContent(text) {
|
|
|
179
189
|
* if more reasoning follows - a stream that is producing readable output should
|
|
180
190
|
* not report as though it were still silent.
|
|
181
191
|
*
|
|
182
|
-
* A
|
|
183
|
-
*
|
|
192
|
+
* A map rather than one global phase because llama-server runs several slots at
|
|
193
|
+
* once. One request can be processing a prompt while another thinks and a third
|
|
194
|
+
* generates content; the aggregate counts must preserve all three.
|
|
184
195
|
*/
|
|
185
196
|
export class ReasoningTracker {
|
|
186
197
|
constructor() {
|
|
187
|
-
|
|
188
|
-
|
|
198
|
+
_ReasoningTracker_instances.add(this);
|
|
199
|
+
_ReasoningTracker_requests.set(this, new Map());
|
|
200
|
+
/** Tail of the last transport chunk, so a field name split by TCP is still detected. */
|
|
201
|
+
_ReasoningTracker_tails.set(this, new Map());
|
|
202
|
+
/** Models/runtimes that leave reasoning inline as `<think>…</think>`. */
|
|
203
|
+
_ReasoningTracker_inlineReasoning.set(this, new Set());
|
|
204
|
+
_ReasoningTracker_listeners.set(this, new Set());
|
|
205
|
+
_ReasoningTracker_lastSnapshot.set(this, "0:0:0:0");
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Watch stage counts, not the per-chunk traffic behind them.
|
|
209
|
+
*
|
|
210
|
+
* The status event stream needs to publish request start, the moment a model
|
|
211
|
+
* goes silent to think and the moment it starts answering. Repeated chunks in
|
|
212
|
+
* one stage do not notify; slot sampling owns bounded token-rate updates.
|
|
213
|
+
*/
|
|
214
|
+
onChange(listener) {
|
|
215
|
+
__classPrivateFieldGet(this, _ReasoningTracker_listeners, "f").add(listener);
|
|
216
|
+
return () => __classPrivateFieldGet(this, _ReasoningTracker_listeners, "f").delete(listener);
|
|
217
|
+
}
|
|
218
|
+
/** A completion was dispatched to llama-server and awaits its first output delta. */
|
|
219
|
+
begin(requestId) {
|
|
220
|
+
if (__classPrivateFieldGet(this, _ReasoningTracker_requests, "f").has(requestId))
|
|
221
|
+
return;
|
|
222
|
+
__classPrivateFieldGet(this, _ReasoningTracker_requests, "f").set(requestId, "processing");
|
|
223
|
+
__classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_announce).call(this);
|
|
189
224
|
}
|
|
190
225
|
/** Note a chunk of `requestId`'s stream. Cheap enough to call per chunk. */
|
|
191
226
|
observe(requestId, text) {
|
|
192
|
-
|
|
227
|
+
const current = __classPrivateFieldGet(this, _ReasoningTracker_requests, "f").get(requestId);
|
|
228
|
+
if (current === "generating")
|
|
193
229
|
return;
|
|
194
|
-
if (
|
|
195
|
-
__classPrivateFieldGet(this,
|
|
196
|
-
|
|
230
|
+
if (!current)
|
|
231
|
+
__classPrivateFieldGet(this, _ReasoningTracker_requests, "f").set(requestId, "processing");
|
|
232
|
+
// Node can split an SSE JSON field name at any byte. Keeping a small tail
|
|
233
|
+
// makes stage recognition independent of transport chunk boundaries without
|
|
234
|
+
// parsing or retaining the generated content itself.
|
|
235
|
+
const combined = `${__classPrivateFieldGet(this, _ReasoningTracker_tails, "f").get(requestId) ?? ""}${text}`;
|
|
236
|
+
__classPrivateFieldGet(this, _ReasoningTracker_tails, "f").set(requestId, combined.slice(-128));
|
|
237
|
+
if (__classPrivateFieldGet(this, _ReasoningTracker_inlineReasoning, "f").has(requestId)) {
|
|
238
|
+
if (combined.includes("</think>")) {
|
|
239
|
+
__classPrivateFieldGet(this, _ReasoningTracker_inlineReasoning, "f").delete(requestId);
|
|
240
|
+
__classPrivateFieldGet(this, _ReasoningTracker_requests, "f").set(requestId, "generating");
|
|
241
|
+
__classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_announce).call(this);
|
|
242
|
+
}
|
|
243
|
+
return;
|
|
244
|
+
}
|
|
245
|
+
if (combined.includes("<think>")) {
|
|
246
|
+
__classPrivateFieldGet(this, _ReasoningTracker_inlineReasoning, "f").add(requestId);
|
|
247
|
+
__classPrivateFieldGet(this, _ReasoningTracker_requests, "f").set(requestId, "thinking");
|
|
248
|
+
__classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_announce).call(this);
|
|
249
|
+
return;
|
|
250
|
+
}
|
|
251
|
+
if (chunkHasContent(combined)) {
|
|
252
|
+
__classPrivateFieldGet(this, _ReasoningTracker_requests, "f").set(requestId, "generating");
|
|
253
|
+
__classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_announce).call(this);
|
|
197
254
|
return;
|
|
198
255
|
}
|
|
199
|
-
if (chunkHasReasoning(
|
|
200
|
-
__classPrivateFieldGet(this,
|
|
256
|
+
if (chunkHasReasoning(combined) && current !== "thinking") {
|
|
257
|
+
__classPrivateFieldGet(this, _ReasoningTracker_requests, "f").set(requestId, "thinking");
|
|
258
|
+
__classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_announce).call(this);
|
|
201
259
|
}
|
|
202
260
|
}
|
|
203
261
|
/** Forget the request. Must be called on end *and* on error, or the flag sticks. */
|
|
204
262
|
end(requestId) {
|
|
205
|
-
__classPrivateFieldGet(this,
|
|
206
|
-
__classPrivateFieldGet(this,
|
|
263
|
+
__classPrivateFieldGet(this, _ReasoningTracker_requests, "f").delete(requestId);
|
|
264
|
+
__classPrivateFieldGet(this, _ReasoningTracker_tails, "f").delete(requestId);
|
|
265
|
+
__classPrivateFieldGet(this, _ReasoningTracker_inlineReasoning, "f").delete(requestId);
|
|
266
|
+
__classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_announce).call(this);
|
|
207
267
|
}
|
|
208
268
|
get active() {
|
|
209
|
-
return
|
|
269
|
+
return this.snapshot.thinking > 0;
|
|
210
270
|
}
|
|
211
271
|
get count() {
|
|
212
|
-
return
|
|
272
|
+
return this.snapshot.thinking;
|
|
273
|
+
}
|
|
274
|
+
/** Aggregate request stages. Counts stay exact even with several parallel slots. */
|
|
275
|
+
get snapshot() {
|
|
276
|
+
const result = {
|
|
277
|
+
activeRequests: __classPrivateFieldGet(this, _ReasoningTracker_requests, "f").size,
|
|
278
|
+
processing: 0,
|
|
279
|
+
thinking: 0,
|
|
280
|
+
generating: 0,
|
|
281
|
+
};
|
|
282
|
+
for (const stage of __classPrivateFieldGet(this, _ReasoningTracker_requests, "f").values())
|
|
283
|
+
result[stage] += 1;
|
|
284
|
+
return result;
|
|
213
285
|
}
|
|
214
286
|
}
|
|
215
|
-
|
|
287
|
+
_ReasoningTracker_requests = new WeakMap(), _ReasoningTracker_tails = new WeakMap(), _ReasoningTracker_inlineReasoning = new WeakMap(), _ReasoningTracker_listeners = new WeakMap(), _ReasoningTracker_lastSnapshot = new WeakMap(), _ReasoningTracker_instances = new WeakSet(), _ReasoningTracker_announce = function _ReasoningTracker_announce() {
|
|
288
|
+
const snapshot = this.snapshot;
|
|
289
|
+
const key = `${snapshot.activeRequests}:${snapshot.processing}:${snapshot.thinking}:${snapshot.generating}`;
|
|
290
|
+
if (key === __classPrivateFieldGet(this, _ReasoningTracker_lastSnapshot, "f"))
|
|
291
|
+
return;
|
|
292
|
+
__classPrivateFieldSet(this, _ReasoningTracker_lastSnapshot, key, "f");
|
|
293
|
+
for (const listener of __classPrivateFieldGet(this, _ReasoningTracker_listeners, "f")) {
|
|
294
|
+
try {
|
|
295
|
+
listener();
|
|
296
|
+
}
|
|
297
|
+
catch {
|
|
298
|
+
// Status reporting must never break a proxied completion.
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
};
|
|
216
302
|
//# sourceMappingURL=activity.js.map
|
|
@@ -26,7 +26,16 @@ import type { RankedModel } from "../ops/results.js";
|
|
|
26
26
|
import type { GpuInfo, Model } from "../types.js";
|
|
27
27
|
import * as vram from "../vram.js";
|
|
28
28
|
import type { SystemSample } from "../sysmon.js";
|
|
29
|
+
import type { BrainStatusPublisher } from "./status-events.js";
|
|
29
30
|
import type { Supervisor } from "./supervisor.js";
|
|
31
|
+
/**
|
|
32
|
+
* The management API's own version, additive to the capability flags.
|
|
33
|
+
*
|
|
34
|
+
* Capabilities answer "can this brain do X"; this answers "which generation of
|
|
35
|
+
* the API is this" for the rare change that no single flag describes. A daemon
|
|
36
|
+
* reads both and never requires an exact package-version match.
|
|
37
|
+
*/
|
|
38
|
+
export declare const HOST_API_VERSION = 2;
|
|
30
39
|
/**
|
|
31
40
|
* What this brain can serve. The daemon folds this into `brain.host.status` and
|
|
32
41
|
* Otto gates each tab on it, because the daemon and the brain version
|
|
@@ -52,6 +61,17 @@ export interface HostCapabilities {
|
|
|
52
61
|
rename: boolean;
|
|
53
62
|
/** POST /__host/model/rename/reset */
|
|
54
63
|
reset: boolean;
|
|
64
|
+
/**
|
|
65
|
+
* GET /__host/events: a live SSE stream of complete status snapshots.
|
|
66
|
+
*
|
|
67
|
+
* The one capability a daemon reads *before* deciding how to watch this brain.
|
|
68
|
+
* False (including on every brain that predates the stream) means the daemon
|
|
69
|
+
* keeps polling `/__host/status`, which is why nothing about the older
|
|
70
|
+
* management API had to change for this to ship.
|
|
71
|
+
*/
|
|
72
|
+
events: boolean;
|
|
73
|
+
/** Bounded live inference stages, token counts and throughput on status events. */
|
|
74
|
+
liveInference: boolean;
|
|
55
75
|
/** Whether writes are currently permitted (allowRemoteConfig). */
|
|
56
76
|
writable: boolean;
|
|
57
77
|
}
|
|
@@ -72,6 +92,12 @@ export interface HostApiDeps {
|
|
|
72
92
|
/** The managed models directory, for disk accounting. Null when unresolvable. */
|
|
73
93
|
getModelsDir: () => string | null;
|
|
74
94
|
sampleResources: () => Promise<SystemSample>;
|
|
95
|
+
/**
|
|
96
|
+
* The live status source behind `GET /__host/events`. Absent (or not yet
|
|
97
|
+
* carrying a snapshot source) means this brain does not advertise events and
|
|
98
|
+
* its daemon keeps polling status.
|
|
99
|
+
*/
|
|
100
|
+
statusEvents?: BrainStatusPublisher | null;
|
|
75
101
|
}
|
|
76
102
|
/** One row of the model inventory: the scan, metadata, profile and score joined. */
|
|
77
103
|
export interface InventoryRow {
|
package/dist/service/host-api.js
CHANGED
|
@@ -7,6 +7,23 @@ import { errorMessage, readJsonBody, sendError, sendJson } from "./http-util.js"
|
|
|
7
7
|
const MAX_PATCH_BYTES = 256 * 1024;
|
|
8
8
|
const MAX_DISPLAY_NAME = 200;
|
|
9
9
|
const DEFAULT_LOG_LINES = 200;
|
|
10
|
+
/**
|
|
11
|
+
* The management API's own version, additive to the capability flags.
|
|
12
|
+
*
|
|
13
|
+
* Capabilities answer "can this brain do X"; this answers "which generation of
|
|
14
|
+
* the API is this" for the rare change that no single flag describes. A daemon
|
|
15
|
+
* reads both and never requires an exact package-version match.
|
|
16
|
+
*/
|
|
17
|
+
export const HOST_API_VERSION = 2;
|
|
18
|
+
/**
|
|
19
|
+
* How often the SSE stream writes a comment line when nothing has changed.
|
|
20
|
+
*
|
|
21
|
+
* This is transport keepalive, not a status event: a proxy or a NAT table with
|
|
22
|
+
* an idle timeout would otherwise silently drop a stream from a brain that is
|
|
23
|
+
* simply sitting still, and the daemon would report an unreachable brain that is
|
|
24
|
+
* fine. Comments are ignored by every SSE parser, so no reader sees them.
|
|
25
|
+
*/
|
|
26
|
+
const SSE_KEEPALIVE_MS = 20000;
|
|
10
27
|
function stateOf(supervisor, model) {
|
|
11
28
|
if (!supervisor.model || supervisor.model.id !== model.id)
|
|
12
29
|
return "not-loaded";
|
|
@@ -105,6 +122,11 @@ export function createHostApi(deps) {
|
|
|
105
122
|
inventory: true,
|
|
106
123
|
rename: true,
|
|
107
124
|
reset: true,
|
|
125
|
+
// Read live rather than captured: the publisher is inert until the router
|
|
126
|
+
// installs its snapshot source, and advertising a stream we cannot serve
|
|
127
|
+
// would make a daemon stop polling and see nothing.
|
|
128
|
+
events: Boolean(deps.statusEvents?.ready),
|
|
129
|
+
liveInference: Boolean(deps.statusEvents?.ready),
|
|
108
130
|
writable: deps.getAllowWrite(),
|
|
109
131
|
});
|
|
110
132
|
/** Refuse a write unless the owner opted into remote configuration. */
|
|
@@ -216,7 +238,13 @@ export function createHostApi(deps) {
|
|
|
216
238
|
return;
|
|
217
239
|
}
|
|
218
240
|
updateDisplayName(model.id, displayName);
|
|
219
|
-
|
|
241
|
+
// The catalog is kept in memory between requests. Refresh it here so
|
|
242
|
+
// the next inventory request, and all model lookups, see the persisted
|
|
243
|
+
// name immediately instead of reverting to the scan-derived name until
|
|
244
|
+
// the brain is restarted. Reset already follows this pattern below.
|
|
245
|
+
const catalog = deps.rescan();
|
|
246
|
+
const updated = resolveModel(catalog, model.id);
|
|
247
|
+
sendJson(res, { displayName: updated ? updated.displayName : displayName });
|
|
220
248
|
});
|
|
221
249
|
};
|
|
222
250
|
const handleReset = (req, res, model) => {
|
|
@@ -333,6 +361,56 @@ export function createHostApi(deps) {
|
|
|
333
361
|
command: deps.supervisor.command,
|
|
334
362
|
});
|
|
335
363
|
};
|
|
364
|
+
/**
|
|
365
|
+
* Stream complete status snapshots as SSE.
|
|
366
|
+
*
|
|
367
|
+
* Authentication is the listener's, not this route's: `withAuth` in serve.ts
|
|
368
|
+
* gates every `/__host/*` path with the same token and TLS policy, so an
|
|
369
|
+
* unauthenticated caller never reaches this function.
|
|
370
|
+
*
|
|
371
|
+
* The stream is unidirectional and outlives the request, which is exactly why
|
|
372
|
+
* SSE rather than a socket the brain would have to dial back to a daemon: a
|
|
373
|
+
* remote brain has no idea where its daemon is, and the daemon already knows
|
|
374
|
+
* how to reach the brain over an authenticated HTTP(S) endpoint.
|
|
375
|
+
*/
|
|
376
|
+
const handleEvents = (req, res, publisher) => {
|
|
377
|
+
res.writeHead(200, {
|
|
378
|
+
"content-type": "text/event-stream",
|
|
379
|
+
"cache-control": "no-cache, no-transform",
|
|
380
|
+
connection: "keep-alive",
|
|
381
|
+
// Tells nginx-shaped intermediaries not to buffer, which would defeat the
|
|
382
|
+
// whole point by holding each snapshot until the response ended.
|
|
383
|
+
"x-accel-buffering": "no",
|
|
384
|
+
});
|
|
385
|
+
res.flushHeaders?.();
|
|
386
|
+
const write = (snapshot) => {
|
|
387
|
+
if (res.writableEnded || res.destroyed)
|
|
388
|
+
return;
|
|
389
|
+
res.write(`event: status\ndata: ${JSON.stringify(snapshot)}\n\n`);
|
|
390
|
+
};
|
|
391
|
+
let unsubscribe = () => { };
|
|
392
|
+
const keepalive = setInterval(() => {
|
|
393
|
+
if (res.writableEnded || res.destroyed)
|
|
394
|
+
return;
|
|
395
|
+
res.write(": keepalive\n\n");
|
|
396
|
+
}, SSE_KEEPALIVE_MS);
|
|
397
|
+
keepalive.unref?.();
|
|
398
|
+
const teardown = () => {
|
|
399
|
+
clearInterval(keepalive);
|
|
400
|
+
unsubscribe();
|
|
401
|
+
};
|
|
402
|
+
// The publisher ends the response on host shutdown: an open SSE response is
|
|
403
|
+
// an open connection, and `server.close()` waits for those.
|
|
404
|
+
unsubscribe = publisher.subscribe(write, () => {
|
|
405
|
+
clearInterval(keepalive);
|
|
406
|
+
if (!res.writableEnded && !res.destroyed)
|
|
407
|
+
res.end();
|
|
408
|
+
});
|
|
409
|
+
// Both ends matter: `close` on the request covers a client that walked away,
|
|
410
|
+
// and `close` on the response covers the service shutting the socket down.
|
|
411
|
+
req.on("close", teardown);
|
|
412
|
+
res.on("close", teardown);
|
|
413
|
+
};
|
|
336
414
|
function handleHostApi(req, res) {
|
|
337
415
|
const raw = req.url || "";
|
|
338
416
|
if (!raw.startsWith("/__host/"))
|
|
@@ -345,6 +423,15 @@ export function createHostApi(deps) {
|
|
|
345
423
|
sendJson(res, capabilities());
|
|
346
424
|
return true;
|
|
347
425
|
}
|
|
426
|
+
if (route === "/__host/events" && method === "GET") {
|
|
427
|
+
const publisher = deps.statusEvents;
|
|
428
|
+
if (!publisher?.ready) {
|
|
429
|
+
sendError(res, 404, "this brain does not serve a status event stream");
|
|
430
|
+
return true;
|
|
431
|
+
}
|
|
432
|
+
handleEvents(req, res, publisher);
|
|
433
|
+
return true;
|
|
434
|
+
}
|
|
348
435
|
if (route === "/__host/logs" && method === "GET") {
|
|
349
436
|
handleLogs(res, params);
|
|
350
437
|
return true;
|