@hraness/sys1 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +677 -0
- package/dist/backends.d.ts +54 -0
- package/dist/cli.js +8254 -0
- package/dist/client.d.ts +52 -0
- package/dist/client.js +667 -0
- package/dist/config.d.ts +126 -0
- package/dist/daemon.d.ts +57 -0
- package/dist/defaults.d.ts +20 -0
- package/dist/doctor.d.ts +29 -0
- package/dist/engine-worker.js +256 -0
- package/dist/gateway.d.ts +36 -0
- package/dist/http.d.ts +9 -0
- package/dist/index.d.ts +32 -0
- package/dist/index.js +3520 -0
- package/dist/kev.d.ts +5 -0
- package/dist/local/decide.d.ts +63 -0
- package/dist/local/engine-types.d.ts +36 -0
- package/dist/local/engine.d.ts +29 -0
- package/dist/local/input.d.ts +4 -0
- package/dist/local/runner.d.ts +71 -0
- package/dist/local/store.d.ts +159 -0
- package/dist/packs/core/fixtures/calibration.jsonl +40 -0
- package/dist/packs/core/fixtures/heldout.jsonl +40 -0
- package/dist/packs/core/pack.yaml +50 -0
- package/dist/profile.d.ts +77 -0
- package/dist/protocol.d.ts +148 -0
- package/dist/qualification.d.ts +22 -0
- package/dist/response.d.ts +10 -0
- package/dist/router.d.ts +62 -0
- package/dist/runtime.d.ts +22 -0
- package/package.json +82 -0
package/dist/config.d.ts
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
export declare const ROUTING_POLICIES: readonly ["auto", "prefer-local", "prefer-hosted", "local-only", "hosted-only"];
|
|
3
|
+
export declare const routingPolicySchema: z.ZodEnum<{
|
|
4
|
+
auto: "auto";
|
|
5
|
+
"prefer-local": "prefer-local";
|
|
6
|
+
"prefer-hosted": "prefer-hosted";
|
|
7
|
+
"local-only": "local-only";
|
|
8
|
+
"hosted-only": "hosted-only";
|
|
9
|
+
}>;
|
|
10
|
+
export type RoutingPolicy = z.infer<typeof routingPolicySchema>;
|
|
11
|
+
export declare function isLoopbackHost(host: string): boolean;
|
|
12
|
+
export declare const loopbackHostSchema: z.ZodString;
|
|
13
|
+
export declare const localBackendSchema: z.ZodObject<{
|
|
14
|
+
name: z.ZodString;
|
|
15
|
+
base_url: z.ZodURL;
|
|
16
|
+
model: z.ZodString;
|
|
17
|
+
adapter: z.ZodOptional<z.ZodEnum<{
|
|
18
|
+
systemone: "systemone";
|
|
19
|
+
kev: "kev";
|
|
20
|
+
}>>;
|
|
21
|
+
size_b: z.ZodOptional<z.ZodNumber>;
|
|
22
|
+
cost_rank: z.ZodOptional<z.ZodNumber>;
|
|
23
|
+
capabilities: z.ZodOptional<z.ZodObject<{
|
|
24
|
+
max_options: z.ZodOptional<z.ZodNumber>;
|
|
25
|
+
max_questions: z.ZodOptional<z.ZodNumber>;
|
|
26
|
+
}, z.core.$strip>>;
|
|
27
|
+
enabled: z.ZodDefault<z.ZodBoolean>;
|
|
28
|
+
}, z.core.$strip>;
|
|
29
|
+
export declare const configSchema: z.ZodObject<{
|
|
30
|
+
version: z.ZodLiteral<1>;
|
|
31
|
+
gateway: z.ZodPrefault<z.ZodObject<{
|
|
32
|
+
host: z.ZodDefault<z.ZodString>;
|
|
33
|
+
port: z.ZodDefault<z.ZodNumber>;
|
|
34
|
+
request_timeout_ms: z.ZodDefault<z.ZodNumber>;
|
|
35
|
+
probe_timeout_ms: z.ZodDefault<z.ZodNumber>;
|
|
36
|
+
}, z.core.$strip>>;
|
|
37
|
+
routing: z.ZodPrefault<z.ZodObject<{
|
|
38
|
+
policy: z.ZodDefault<z.ZodEnum<{
|
|
39
|
+
auto: "auto";
|
|
40
|
+
"prefer-local": "prefer-local";
|
|
41
|
+
"prefer-hosted": "prefer-hosted";
|
|
42
|
+
"local-only": "local-only";
|
|
43
|
+
"hosted-only": "hosted-only";
|
|
44
|
+
}>>;
|
|
45
|
+
}, z.core.$strip>>;
|
|
46
|
+
hosted: z.ZodPrefault<z.ZodObject<{
|
|
47
|
+
enabled: z.ZodDefault<z.ZodBoolean>;
|
|
48
|
+
base_url: z.ZodDefault<z.ZodURL>;
|
|
49
|
+
model: z.ZodDefault<z.ZodString>;
|
|
50
|
+
api_key_env: z.ZodDefault<z.ZodString>;
|
|
51
|
+
}, z.core.$strip>>;
|
|
52
|
+
local: z.ZodPrefault<z.ZodObject<{
|
|
53
|
+
enabled: z.ZodDefault<z.ZodBoolean>;
|
|
54
|
+
model: z.ZodDefault<z.ZodString>;
|
|
55
|
+
context_tokens: z.ZodDefault<z.ZodNumber>;
|
|
56
|
+
eval_timeout_ms: z.ZodDefault<z.ZodNumber>;
|
|
57
|
+
max_loaded_models: z.ZodDefault<z.ZodNumber>;
|
|
58
|
+
}, z.core.$strip>>;
|
|
59
|
+
backends: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
60
|
+
name: z.ZodString;
|
|
61
|
+
base_url: z.ZodURL;
|
|
62
|
+
model: z.ZodString;
|
|
63
|
+
adapter: z.ZodOptional<z.ZodEnum<{
|
|
64
|
+
systemone: "systemone";
|
|
65
|
+
kev: "kev";
|
|
66
|
+
}>>;
|
|
67
|
+
size_b: z.ZodOptional<z.ZodNumber>;
|
|
68
|
+
cost_rank: z.ZodOptional<z.ZodNumber>;
|
|
69
|
+
capabilities: z.ZodOptional<z.ZodObject<{
|
|
70
|
+
max_options: z.ZodOptional<z.ZodNumber>;
|
|
71
|
+
max_questions: z.ZodOptional<z.ZodNumber>;
|
|
72
|
+
}, z.core.$strip>>;
|
|
73
|
+
enabled: z.ZodDefault<z.ZodBoolean>;
|
|
74
|
+
}, z.core.$strip>>>;
|
|
75
|
+
}, z.core.$strip>;
|
|
76
|
+
export type LocalBackendConfig = z.infer<typeof localBackendSchema>;
|
|
77
|
+
export type Sys1Config = z.infer<typeof configSchema>;
|
|
78
|
+
export declare const DEFAULT_CONFIG: Sys1Config;
|
|
79
|
+
export declare function sys1Home(env?: NodeJS.ProcessEnv): string;
|
|
80
|
+
export declare function configPath(home: string): string;
|
|
81
|
+
export declare function pidPath(home: string): string;
|
|
82
|
+
export declare function logPath(home: string): string;
|
|
83
|
+
export type ConfigLoadResult = {
|
|
84
|
+
ok: true;
|
|
85
|
+
config: Sys1Config;
|
|
86
|
+
path: string;
|
|
87
|
+
existed: boolean;
|
|
88
|
+
} | {
|
|
89
|
+
ok: false;
|
|
90
|
+
path: string;
|
|
91
|
+
message: string;
|
|
92
|
+
};
|
|
93
|
+
export declare function loadConfig(home: string): ConfigLoadResult;
|
|
94
|
+
export declare function saveConfig(home: string, config: Sys1Config): string;
|
|
95
|
+
export declare const SETTABLE_KEYS: {
|
|
96
|
+
readonly "routing.policy": z.ZodEnum<{
|
|
97
|
+
auto: "auto";
|
|
98
|
+
"prefer-local": "prefer-local";
|
|
99
|
+
"prefer-hosted": "prefer-hosted";
|
|
100
|
+
"local-only": "local-only";
|
|
101
|
+
"hosted-only": "hosted-only";
|
|
102
|
+
}>;
|
|
103
|
+
readonly "gateway.host": z.ZodString;
|
|
104
|
+
readonly "gateway.port": z.ZodCoercedNumber<unknown>;
|
|
105
|
+
readonly "gateway.request_timeout_ms": z.ZodCoercedNumber<unknown>;
|
|
106
|
+
readonly "gateway.probe_timeout_ms": z.ZodCoercedNumber<unknown>;
|
|
107
|
+
readonly "hosted.base_url": z.ZodURL;
|
|
108
|
+
readonly "hosted.model": z.ZodString;
|
|
109
|
+
readonly "hosted.api_key_env": z.ZodString;
|
|
110
|
+
readonly "local.enabled": z.ZodPipe<z.ZodEnum<{
|
|
111
|
+
false: "false";
|
|
112
|
+
true: "true";
|
|
113
|
+
}>, z.ZodTransform<boolean, "false" | "true">>;
|
|
114
|
+
readonly "local.model": z.ZodString;
|
|
115
|
+
readonly "local.context_tokens": z.ZodCoercedNumber<unknown>;
|
|
116
|
+
readonly "local.eval_timeout_ms": z.ZodCoercedNumber<unknown>;
|
|
117
|
+
readonly "local.max_loaded_models": z.ZodCoercedNumber<unknown>;
|
|
118
|
+
};
|
|
119
|
+
export type SettableKey = keyof typeof SETTABLE_KEYS;
|
|
120
|
+
export declare function setConfigValue(config: Sys1Config, key: SettableKey, rawValue: string): {
|
|
121
|
+
ok: true;
|
|
122
|
+
config: Sys1Config;
|
|
123
|
+
} | {
|
|
124
|
+
ok: false;
|
|
125
|
+
message: string;
|
|
126
|
+
};
|
package/dist/daemon.d.ts
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { type Sys1Config } from "./config.ts";
|
|
3
|
+
declare const pidFileSchema: z.ZodObject<{
|
|
4
|
+
pid: z.ZodNumber;
|
|
5
|
+
port: z.ZodNumber;
|
|
6
|
+
host: z.ZodString;
|
|
7
|
+
instance: z.ZodString;
|
|
8
|
+
started_at: z.ZodString;
|
|
9
|
+
}, z.core.$strip>;
|
|
10
|
+
export type PidFile = z.infer<typeof pidFileSchema>;
|
|
11
|
+
export declare function readPidFile(home: string): PidFile | null;
|
|
12
|
+
export declare function processAlive(pid: number): boolean;
|
|
13
|
+
export declare function gatewayUrl(host: string, port: number): string;
|
|
14
|
+
export declare function healthz(host: string, port: number, timeoutMs?: number, fetchFn?: typeof fetch): Promise<boolean>;
|
|
15
|
+
export type DaemonState = {
|
|
16
|
+
state: "running";
|
|
17
|
+
pid: number;
|
|
18
|
+
port: number;
|
|
19
|
+
host: string;
|
|
20
|
+
started_at: string;
|
|
21
|
+
} | {
|
|
22
|
+
state: "stale_pidfile";
|
|
23
|
+
pid: number;
|
|
24
|
+
} | {
|
|
25
|
+
state: "foreign_listener";
|
|
26
|
+
port: number;
|
|
27
|
+
} | {
|
|
28
|
+
state: "stopped";
|
|
29
|
+
};
|
|
30
|
+
export declare function daemonStatus(home: string, config: Sys1Config, fetchFn?: typeof fetch): Promise<DaemonState>;
|
|
31
|
+
export interface UpResult {
|
|
32
|
+
ok: boolean;
|
|
33
|
+
message: string;
|
|
34
|
+
pid?: number;
|
|
35
|
+
url?: string;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Spawn a detached `sys1 serve` child and wait for its health check. The
|
|
39
|
+
* child writes the pid file itself after binding; this function only polls.
|
|
40
|
+
*/
|
|
41
|
+
export declare function daemonUp(options: {
|
|
42
|
+
home: string;
|
|
43
|
+
config: Sys1Config;
|
|
44
|
+
cliEntry: string;
|
|
45
|
+
env: NodeJS.ProcessEnv;
|
|
46
|
+
fetchFn?: typeof fetch;
|
|
47
|
+
waitMs?: number;
|
|
48
|
+
}): Promise<UpResult>;
|
|
49
|
+
export interface DownResult {
|
|
50
|
+
ok: boolean;
|
|
51
|
+
message: string;
|
|
52
|
+
}
|
|
53
|
+
export declare function daemonDown(home: string, config: Sys1Config, fetchFn?: typeof fetch): Promise<DownResult>;
|
|
54
|
+
/** Called by the detached `serve --daemon-child` process after binding. */
|
|
55
|
+
export declare function writePidFile(home: string, pid: number, host: string, port: number, instance: string): void;
|
|
56
|
+
export declare function clearPidFile(home: string, pid: number): void;
|
|
57
|
+
export {};
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
export declare const LOCAL_MODEL_TIERS: readonly ["compact", "quality"];
|
|
2
|
+
export type LocalModelTier = (typeof LOCAL_MODEL_TIERS)[number];
|
|
3
|
+
export declare const DEFAULT_LOCAL_MODELS: Record<LocalModelTier, string>;
|
|
4
|
+
export interface PlatformRecommendation {
|
|
5
|
+
platform: string;
|
|
6
|
+
arch: string;
|
|
7
|
+
target: string;
|
|
8
|
+
supported: boolean;
|
|
9
|
+
acceleration: string;
|
|
10
|
+
tier: LocalModelTier;
|
|
11
|
+
model: string | null;
|
|
12
|
+
/** Model decision quality remains experimental, independent of runtime support. */
|
|
13
|
+
experimental?: boolean;
|
|
14
|
+
reason: string;
|
|
15
|
+
}
|
|
16
|
+
export declare function platformRecommendation(options?: {
|
|
17
|
+
platform?: string;
|
|
18
|
+
arch?: string;
|
|
19
|
+
tier?: LocalModelTier;
|
|
20
|
+
}): PlatformRecommendation;
|
package/dist/doctor.d.ts
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { type Sys1Config } from "./config.ts";
|
|
2
|
+
import { type DaemonState } from "./daemon.ts";
|
|
3
|
+
import { type NativeRuntimeProbe } from "./local/engine.ts";
|
|
4
|
+
import type { JsonValue } from "./protocol.ts";
|
|
5
|
+
export type DoctorStatus = "pass" | "warn" | "fail";
|
|
6
|
+
export interface DoctorCheck {
|
|
7
|
+
id: string;
|
|
8
|
+
status: DoctorStatus;
|
|
9
|
+
summary: string;
|
|
10
|
+
detail?: JsonValue;
|
|
11
|
+
}
|
|
12
|
+
export interface DoctorReport {
|
|
13
|
+
ok: boolean;
|
|
14
|
+
version: 1;
|
|
15
|
+
checks: DoctorCheck[];
|
|
16
|
+
counts: {
|
|
17
|
+
pass: number;
|
|
18
|
+
warn: number;
|
|
19
|
+
fail: number;
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
export interface DoctorOptions {
|
|
23
|
+
home: string;
|
|
24
|
+
env?: NodeJS.ProcessEnv;
|
|
25
|
+
runtimeVersion?: string;
|
|
26
|
+
nativeProbe?: () => Promise<NativeRuntimeProbe>;
|
|
27
|
+
daemonProbe?: (home: string, config: Sys1Config) => Promise<DaemonState>;
|
|
28
|
+
}
|
|
29
|
+
export declare function runDoctor(options: DoctorOptions): Promise<DoctorReport>;
|
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
// src/local/engine-worker.ts
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
|
|
4
|
+
// src/local/engine-types.ts
|
|
5
|
+
class EngineUnavailableError extends Error {
|
|
6
|
+
detail;
|
|
7
|
+
constructor(message, detail) {
|
|
8
|
+
super(message);
|
|
9
|
+
this.name = "EngineUnavailableError";
|
|
10
|
+
this.detail = detail;
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
var ENGINE_IPC_LIMITS = {
|
|
14
|
+
requestBytes: 131072,
|
|
15
|
+
responseBytes: 16 * 1048576,
|
|
16
|
+
maxEntries: 300000,
|
|
17
|
+
maxTokenBytes: 4096,
|
|
18
|
+
maxPromptBytes: 65536
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
// src/local/input.ts
|
|
22
|
+
class LocalInputError extends Error {
|
|
23
|
+
constructor(message) {
|
|
24
|
+
super(message);
|
|
25
|
+
this.name = "LocalInputError";
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// src/local/engine-core.ts
|
|
30
|
+
async function loadNodeLlamaCpp() {
|
|
31
|
+
const specifier = "node-llama-cpp";
|
|
32
|
+
return await import(specifier);
|
|
33
|
+
}
|
|
34
|
+
async function runNativeProbe(loadModule = loadNodeLlamaCpp) {
|
|
35
|
+
let llama = null;
|
|
36
|
+
try {
|
|
37
|
+
const mod = await loadModule();
|
|
38
|
+
const supported = await mod.getLlamaGpuTypes("supported");
|
|
39
|
+
llama = await mod.getLlama({ gpu: "auto", logLevel: "fatal", build: "never", skipDownload: true });
|
|
40
|
+
return {
|
|
41
|
+
ok: true,
|
|
42
|
+
backend: llama.gpu === false || llama.gpu === undefined ? "cpu" : llama.gpu,
|
|
43
|
+
gpu_offloading: llama.supportsGpuOffloading ?? false,
|
|
44
|
+
supported_backends: supported.map((backend) => backend === false ? "cpu" : backend)
|
|
45
|
+
};
|
|
46
|
+
} catch (error) {
|
|
47
|
+
return {
|
|
48
|
+
ok: false,
|
|
49
|
+
message: error instanceof Error ? error.message : "native runtime unavailable"
|
|
50
|
+
};
|
|
51
|
+
} finally {
|
|
52
|
+
if (llama !== null)
|
|
53
|
+
await llama.dispose().catch(() => {});
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
class NativeLlamaEngine {
|
|
58
|
+
loadModule;
|
|
59
|
+
modelId;
|
|
60
|
+
options;
|
|
61
|
+
llama = null;
|
|
62
|
+
model = null;
|
|
63
|
+
context = null;
|
|
64
|
+
chatWrapper = null;
|
|
65
|
+
disposed = false;
|
|
66
|
+
constructor(options, loadModule = loadNodeLlamaCpp) {
|
|
67
|
+
this.loadModule = loadModule;
|
|
68
|
+
this.options = options;
|
|
69
|
+
this.modelId = options.modelId;
|
|
70
|
+
}
|
|
71
|
+
async ensureLoaded() {
|
|
72
|
+
if (this.context !== null)
|
|
73
|
+
return;
|
|
74
|
+
let mod;
|
|
75
|
+
try {
|
|
76
|
+
mod = await this.loadModule();
|
|
77
|
+
} catch {
|
|
78
|
+
throw new EngineUnavailableError("node-llama-cpp is not installed", "reinstall @hraness/sys1 or run `bun add node-llama-cpp`");
|
|
79
|
+
}
|
|
80
|
+
try {
|
|
81
|
+
this.llama = await mod.getLlama({ gpu: "auto", logLevel: "fatal", build: "never", skipDownload: true });
|
|
82
|
+
} catch (error) {
|
|
83
|
+
if (mod.NoBinaryFoundError !== undefined && error instanceof mod.NoBinaryFoundError) {
|
|
84
|
+
throw new EngineUnavailableError("llama.cpp binaries are not installed", "run `bun pm trust node-llama-cpp && bun install` or `sys1 doctor` for setup help");
|
|
85
|
+
}
|
|
86
|
+
throw error;
|
|
87
|
+
}
|
|
88
|
+
try {
|
|
89
|
+
this.model = await this.llama.loadModel({ modelPath: this.options.modelPath });
|
|
90
|
+
this.chatWrapper = mod.resolveChatWrapper(this.model, {
|
|
91
|
+
customWrapperSettings: { qwen: { thoughts: "discourage" } }
|
|
92
|
+
});
|
|
93
|
+
this.context = await this.model.createContext({ contextSize: this.options.contextSize });
|
|
94
|
+
} catch (error) {
|
|
95
|
+
if (this.context !== null)
|
|
96
|
+
await this.context.dispose().catch(() => {});
|
|
97
|
+
if (this.model !== null)
|
|
98
|
+
await this.model.dispose().catch(() => {});
|
|
99
|
+
if (this.llama !== null)
|
|
100
|
+
await this.llama.dispose().catch(() => {});
|
|
101
|
+
this.context = null;
|
|
102
|
+
this.model = null;
|
|
103
|
+
this.chatWrapper = null;
|
|
104
|
+
this.llama = null;
|
|
105
|
+
throw new EngineUnavailableError("local model failed to load", error instanceof Error ? error.message : this.modelId);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
async firstTokenDistribution(prompt) {
|
|
109
|
+
if (this.disposed)
|
|
110
|
+
throw new EngineUnavailableError("engine is disposed", this.modelId);
|
|
111
|
+
await this.ensureLoaded();
|
|
112
|
+
const model = this.model;
|
|
113
|
+
const context = this.context;
|
|
114
|
+
const chatWrapper = this.chatWrapper;
|
|
115
|
+
if (model === null || context === null || chatWrapper === null) {
|
|
116
|
+
throw new EngineUnavailableError("engine failed to initialize", this.modelId);
|
|
117
|
+
}
|
|
118
|
+
const rendered = chatWrapper.generateContextState({
|
|
119
|
+
chatHistory: [
|
|
120
|
+
{
|
|
121
|
+
type: "system",
|
|
122
|
+
text: "You are a precise decision engine. Follow the requested answer format exactly and do not explain."
|
|
123
|
+
},
|
|
124
|
+
{ type: "user", text: prompt },
|
|
125
|
+
{ type: "model", response: [] }
|
|
126
|
+
]
|
|
127
|
+
});
|
|
128
|
+
const tokens = rendered.contextText.tokenize(model.tokenizer);
|
|
129
|
+
if (tokens.length >= context.contextSize) {
|
|
130
|
+
throw new LocalInputError("prompt tokens exceed the model context capacity");
|
|
131
|
+
}
|
|
132
|
+
const sequence = context.getSequence();
|
|
133
|
+
try {
|
|
134
|
+
const generator = sequence.evaluateWithMetadata(tokens, { probabilities: true }, { temperature: 0, yieldEogToken: true });
|
|
135
|
+
const first = await generator.next();
|
|
136
|
+
await generator.return(undefined).catch(() => {});
|
|
137
|
+
const probs = first.done ? undefined : first.value.probabilities;
|
|
138
|
+
if (probs === undefined) {
|
|
139
|
+
throw new EngineUnavailableError("model produced no distribution", this.modelId);
|
|
140
|
+
}
|
|
141
|
+
const entries = [];
|
|
142
|
+
for (const [token, p] of probs) {
|
|
143
|
+
if (typeof p !== "number" || !(p > 0))
|
|
144
|
+
continue;
|
|
145
|
+
entries.push([model.detokenize([token]), p]);
|
|
146
|
+
}
|
|
147
|
+
return { entries, inputTokens: tokens.length };
|
|
148
|
+
} finally {
|
|
149
|
+
await sequence.dispose().catch(() => {});
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
async dispose() {
|
|
153
|
+
this.disposed = true;
|
|
154
|
+
if (this.context !== null)
|
|
155
|
+
await this.context.dispose().catch(() => {});
|
|
156
|
+
if (this.model !== null)
|
|
157
|
+
await this.model.dispose().catch(() => {});
|
|
158
|
+
if (this.llama !== null)
|
|
159
|
+
await this.llama.dispose().catch(() => {});
|
|
160
|
+
this.context = null;
|
|
161
|
+
this.model = null;
|
|
162
|
+
this.chatWrapper = null;
|
|
163
|
+
this.llama = null;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// src/local/engine-ipc.ts
|
|
168
|
+
import { write } from "node:fs";
|
|
169
|
+
import { setTimeout as delay } from "node:timers/promises";
|
|
170
|
+
async function writeEngineMessage(value) {
|
|
171
|
+
const wire = Buffer.from(`${JSON.stringify(value)}
|
|
172
|
+
`);
|
|
173
|
+
if (wire.length > ENGINE_IPC_LIMITS.responseBytes)
|
|
174
|
+
throw new Error("worker_response_limit");
|
|
175
|
+
let offset = 0;
|
|
176
|
+
while (offset < wire.length) {
|
|
177
|
+
const written = await new Promise((resolve, reject) => {
|
|
178
|
+
write(1, wire, offset, wire.length - offset, null, (error, count) => {
|
|
179
|
+
if (error !== null) {
|
|
180
|
+
if (error.code === "EAGAIN" || error.code === "EWOULDBLOCK")
|
|
181
|
+
resolve(0);
|
|
182
|
+
else
|
|
183
|
+
reject(error);
|
|
184
|
+
} else
|
|
185
|
+
resolve(count);
|
|
186
|
+
});
|
|
187
|
+
});
|
|
188
|
+
if (written === 0)
|
|
189
|
+
await delay(1);
|
|
190
|
+
offset += written;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// src/local/engine-worker.ts
|
|
195
|
+
var requestSchema = z.discriminatedUnion("op", [
|
|
196
|
+
z.object({ id: z.number().int().positive(), op: z.literal("probe") }).strict(),
|
|
197
|
+
z.object({
|
|
198
|
+
id: z.number().int().positive(),
|
|
199
|
+
op: z.literal("evaluate"),
|
|
200
|
+
options: z.object({
|
|
201
|
+
modelPath: z.string().min(1).max(4096),
|
|
202
|
+
modelId: z.string().min(1).max(128),
|
|
203
|
+
contextSize: z.number().int().min(256).max(65536),
|
|
204
|
+
evalTimeoutMs: z.number().int().min(1).max(300000)
|
|
205
|
+
}).strict(),
|
|
206
|
+
prompt: z.string().refine((text) => Buffer.byteLength(text) <= ENGINE_IPC_LIMITS.maxPromptBytes)
|
|
207
|
+
}).strict()
|
|
208
|
+
]);
|
|
209
|
+
var engine;
|
|
210
|
+
var identity;
|
|
211
|
+
var busy = false;
|
|
212
|
+
var chunks = [];
|
|
213
|
+
var bytes = 0;
|
|
214
|
+
async function handle(wire) {
|
|
215
|
+
const parsed = requestSchema.safeParse(JSON.parse(wire));
|
|
216
|
+
if (!parsed.success)
|
|
217
|
+
process.exit(1);
|
|
218
|
+
const request = parsed.data;
|
|
219
|
+
try {
|
|
220
|
+
if (request.op === "probe") {
|
|
221
|
+
const result = await runNativeProbe();
|
|
222
|
+
await writeEngineMessage({ id: request.id, kind: "probe", value: result.ok ? result : { ok: false, message: "native runtime unavailable" } });
|
|
223
|
+
} else {
|
|
224
|
+
const nextIdentity = JSON.stringify(request.options);
|
|
225
|
+
if (identity !== undefined && nextIdentity !== identity)
|
|
226
|
+
process.exit(1);
|
|
227
|
+
identity = nextIdentity;
|
|
228
|
+
engine ??= new NativeLlamaEngine(request.options);
|
|
229
|
+
await writeEngineMessage({ id: request.id, kind: "distribution", value: await engine.firstTokenDistribution(request.prompt) });
|
|
230
|
+
}
|
|
231
|
+
} catch (error) {
|
|
232
|
+
await writeEngineMessage({ id: request.id, kind: error instanceof LocalInputError ? "unsupported" : "error", detail: error instanceof LocalInputError ? "context_limit" : "native_inference_failed" });
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
process.stdin.on("data", (chunk) => {
|
|
236
|
+
if (busy)
|
|
237
|
+
process.exit(1);
|
|
238
|
+
bytes += chunk.byteLength;
|
|
239
|
+
if (bytes > ENGINE_IPC_LIMITS.requestBytes)
|
|
240
|
+
process.exit(1);
|
|
241
|
+
chunks.push(chunk);
|
|
242
|
+
const newline = chunk.indexOf(10);
|
|
243
|
+
if (newline < 0)
|
|
244
|
+
return;
|
|
245
|
+
if (newline !== chunk.length - 1)
|
|
246
|
+
process.exit(1);
|
|
247
|
+
const wire = Buffer.concat(chunks, bytes).toString("utf8");
|
|
248
|
+
chunks = [];
|
|
249
|
+
bytes = 0;
|
|
250
|
+
busy = true;
|
|
251
|
+
handle(wire).then(() => {
|
|
252
|
+
busy = false;
|
|
253
|
+
}, () => process.exit(1));
|
|
254
|
+
});
|
|
255
|
+
process.stdin.on("end", () => process.exit(0));
|
|
256
|
+
process.stdin.on("error", () => process.exit(1));
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { type Sys1Config } from "./config.ts";
|
|
2
|
+
import { LocalRunner } from "./local/runner.ts";
|
|
3
|
+
export declare const SYS1_VERSION = "0.17.0";
|
|
4
|
+
export interface GatewayDeps {
|
|
5
|
+
config: Sys1Config;
|
|
6
|
+
env: NodeJS.ProcessEnv;
|
|
7
|
+
fetchFn?: typeof fetch;
|
|
8
|
+
/**
|
|
9
|
+
* Re-read config per request so `sys1 config set` / `backend add` take
|
|
10
|
+
* effect on a running daemon. May throw; a thrown read answers 503.
|
|
11
|
+
*/
|
|
12
|
+
reloadConfig?: () => Sys1Config;
|
|
13
|
+
/**
|
|
14
|
+
* State directory. Required for builtin local models; when absent the
|
|
15
|
+
* gateway only serves URL-registered and hosted backends.
|
|
16
|
+
*/
|
|
17
|
+
home?: string;
|
|
18
|
+
/** Injectable local runner (tests substitute a fake engine factory). */
|
|
19
|
+
localRunner?: LocalRunner;
|
|
20
|
+
/** Daemon ownership proof. Standalone embedded gateways omit this. */
|
|
21
|
+
daemon?: {
|
|
22
|
+
instance: string;
|
|
23
|
+
onShutdown: () => void | Promise<void>;
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
export declare function createFetchHandler(deps: GatewayDeps): (req: Request) => Promise<Response>;
|
|
27
|
+
export interface RunningGateway {
|
|
28
|
+
url: string;
|
|
29
|
+
port: number;
|
|
30
|
+
stop: () => Promise<void>;
|
|
31
|
+
}
|
|
32
|
+
/** Network admission wrapper, shared by the listener and boundary tests. */
|
|
33
|
+
export declare function createNetworkFetchHandler(deps: GatewayDeps): (req: Request) => Promise<Response>;
|
|
34
|
+
export declare function startGateway(deps: GatewayDeps & {
|
|
35
|
+
port?: number;
|
|
36
|
+
}): RunningGateway;
|
package/dist/http.d.ts
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/** A body exceeded its byte limit before it finished streaming. */
|
|
2
|
+
export declare class HttpBodyLimitError extends Error {
|
|
3
|
+
readonly maxBytes: number;
|
|
4
|
+
constructor(maxBytes: number);
|
|
5
|
+
}
|
|
6
|
+
/** Read at most maxBytes, canceling the stream on overflow or caller abort. */
|
|
7
|
+
export declare function readBoundedText(source: {
|
|
8
|
+
body: ReadableStream<Uint8Array> | null;
|
|
9
|
+
}, maxBytes: number, signal?: AbortSignal): Promise<string>;
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
export { PROTOCOL_LIMITS, answerSchema, choiceAnswerSchema, choiceQuestionSchema, entrySchema, errorBody, noulAnswerSchema, noulQuestionSchema, questionSchema, scoreAnswerSchema, scoreQuestionSchema, serializedBytes, systemOneRequestSchema, systemOneResponseSchema, } from "./protocol.ts";
|
|
2
|
+
export type { Answer, ChoiceAnswer, ChoiceQuestion, EntryType, GatewayError, JsonValue, NoulAnswer, NoulQuestion, Question, ScoreAnswer, ScoreQuestion, SystemOneRequest, SystemOneResponse, } from "./protocol.ts";
|
|
3
|
+
export { DEFAULT_CONFIG, ROUTING_POLICIES, SETTABLE_KEYS, configSchema, isLoopbackHost, loadConfig, localBackendSchema, loopbackHostSchema, routingPolicySchema, saveConfig, setConfigValue, sys1Home, } from "./config.ts";
|
|
4
|
+
export type { LocalBackendConfig, RoutingPolicy, SettableKey, Sys1Config, } from "./config.ts";
|
|
5
|
+
export { chooseBackend, requestNeeds } from "./router.ts";
|
|
6
|
+
export type { BackendCandidate, BackendCapabilities, BackendKind, RequestNeeds, RouteChoice, } from "./router.ts";
|
|
7
|
+
export { HOSTED_BACKEND_NAME, extractBackendCapabilities, extractModelIds, forwardToBackend, probeAll, probeBackend, runtimeBackends, } from "./backends.ts";
|
|
8
|
+
export type { ForwardResult, ProbeResult, RuntimeBackend } from "./backends.ts";
|
|
9
|
+
export { DEFAULT_LOCAL_MODELS, LOCAL_MODEL_TIERS, platformRecommendation, } from "./defaults.ts";
|
|
10
|
+
export type { LocalModelTier, PlatformRecommendation } from "./defaults.ts";
|
|
11
|
+
export { qualifyBackend } from "./qualification.ts";
|
|
12
|
+
export type { BackendQualificationCheck, BackendQualificationOptions, BackendQualificationReport, BackendQualificationStatus, } from "./qualification.ts";
|
|
13
|
+
export { SYS1_VERSION, createFetchHandler, startGateway } from "./gateway.ts";
|
|
14
|
+
export type { GatewayDeps, RunningGateway } from "./gateway.ts";
|
|
15
|
+
export { runDoctor } from "./doctor.ts";
|
|
16
|
+
export type { DoctorCheck, DoctorOptions, DoctorReport, DoctorStatus, } from "./doctor.ts";
|
|
17
|
+
export { clearPidFile, daemonDown, daemonStatus, daemonUp, healthz, processAlive, readPidFile, writePidFile, } from "./daemon.ts";
|
|
18
|
+
export type { DaemonState, DownResult, PidFile, UpResult } from "./daemon.ts";
|
|
19
|
+
export { DECIDE_LIMITS, DECISION_LABELS, aggregateMass, answerLabels, confidenceOf, decisionPrompt, outcomeFromMass, toLocalAnswer, toLocalResponse, } from "./local/decide.ts";
|
|
20
|
+
export type { LabelMass, LocalAnswer, LocalResponse, QuestionOutcome, } from "./local/decide.ts";
|
|
21
|
+
export { EngineUnavailableError, LlamaEngine, probeNativeRuntime, } from "./local/engine.ts";
|
|
22
|
+
export type { DecisionEngine, FirstTokenDistribution, LlamaEngineOptions, NativeRuntimeProbe, } from "./local/engine.ts";
|
|
23
|
+
export { BUILTIN_PREFIX, LocalRunner, builtinCandidates, builtinName, defaultEngineFactory, } from "./local/runner.ts";
|
|
24
|
+
export type { BuiltinCandidate, DecideResult, EngineFactory, LocalAdapter, LocalQuestionDiagnostic, RunnerOptions, } from "./local/runner.ts";
|
|
25
|
+
export { MODEL_KINDS, MODEL_LIMITS, MODEL_REGISTRY, ModelStoreError, findInstalled, findRegistry, inspectGgufFile, installedModels, loadManifest, loadManifestChecked, manifestPath, modelFilePath, modelsDir, pullModel, removeModel, resolvePullTarget, saveManifest, storeBytes, verifyModel, } from "./local/store.ts";
|
|
26
|
+
export type { GgufInspection, InstalledModel, Manifest, ManifestLoadResult, ModelKind, PullOptions, PullResult, PullTarget, RegistryEntry, VerifyModelResult, } from "./local/store.ts";
|
|
27
|
+
export { createClient, DEFAULT_BASE_URL, Sys1ClientError } from "./client.ts";
|
|
28
|
+
export type { ClientErrorCode, ClientOptions, EvaluationOptions, EvaluationResult, RouteMetadata, Sys1Client, } from "./client.ts";
|
|
29
|
+
export { createRouter } from "./runtime.ts";
|
|
30
|
+
export type { RouterOptions, EmbeddedRouter } from "./runtime.ts";
|
|
31
|
+
export { createProfile, Sys1ProfileError } from "./profile.ts";
|
|
32
|
+
export type { DecisionProfile, ProfileDefinition, ReadonlyProfileDefinition, ProfileErrorCode, } from "./profile.ts";
|