@alilis/k-hat 0.2.6 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -138
- package/dist/{cli.js → cli/cli.js} +64 -12
- package/dist/{config.js → core/config.js} +3 -0
- package/dist/core/probe.js +15 -0
- package/dist/{key-protector.js → data/key-protector.js} +1 -1
- package/dist/{portable-vault.js → data/portable-vault.js} +2 -2
- package/dist/{store.js → data/store.js} +11 -4
- package/dist/{vault.js → data/vault.js} +1 -1
- package/dist/{admin.js → network/admin.js} +17 -17
- package/dist/{selector.js → network/selector.js} +13 -1
- package/dist/network/server.js +383 -0
- package/dist/service.js +283 -0
- package/dist/{tui-state.js → ui/tui/tui-state.js} +4 -2
- package/dist/{tui.js → ui/tui/tui.js} +7 -5
- package/dist/{web-ui.js → ui/web-ui.js} +42 -11
- package/package.json +6 -5
- package/dist/server.js +0 -238
- /package/dist/{logger.js → core/logger.js} +0 -0
- /package/dist/{router.js → core/router.js} +0 -0
- /package/dist/{types.js → core/types.js} +0 -0
- /package/dist/{dpapi.js → data/dpapi.js} +0 -0
- /package/dist/{tui-client.js → ui/tui/tui-client.js} +0 -0
- /package/dist/{tui-main.js → ui/tui/tui-main.js} +0 -0
- /package/dist/{tui-types.js → ui/tui/tui-types.js} +0 -0
|
@@ -0,0 +1,383 @@
|
|
|
1
|
+
import { createServer } from 'node:http';
|
|
2
|
+
import { watch } from 'node:fs';
|
|
3
|
+
import { readJsonFile, saveJsonAtomic, validateConfig, providerUrl, defaultTimeouts } from '../core/config.js';
|
|
4
|
+
import { LogWriter } from '../core/logger.js';
|
|
5
|
+
import { join, basename } from 'node:path';
|
|
6
|
+
import { findRoute, routeUpstreamModel, resolveProvider, isRouteDisabled } from '../core/router.js';
|
|
7
|
+
import { WeightedSelector } from './selector.js';
|
|
8
|
+
import { handleAdmin } from '../network/admin.js';
|
|
9
|
+
import { enableKey } from '../data/store.js';
|
|
10
|
+
import { probeKey } from '../core/probe.js';
|
|
11
|
+
import { ACCESS_TOKEN_REF } from '../data/vault.js';
|
|
12
|
+
const RETRYABLE = new Set([401, 402, 429]);
|
|
13
|
+
const HOP_BY_HOP = new Set(['content-length', 'transfer-encoding', 'connection']);
|
|
14
|
+
const ANTHROPIC_VERSION = '2023-06-01';
|
|
15
|
+
/** 429 cooldown: first wait, doubling per consecutive cycle, capped. */
|
|
16
|
+
const COOLDOWN_BASE_MS = 30_000;
|
|
17
|
+
const COOLDOWN_MAX_MS = 10 * 60_000;
|
|
18
|
+
/** An upstream Retry-After is honored, but never beyond this cap. */
|
|
19
|
+
const RETRY_AFTER_MAX_MS = 30 * 60_000;
|
|
20
|
+
/** Background probe cadence for permanently unavailable keys (cooldown keys revive on their own). */
|
|
21
|
+
const PROBE_INTERVAL_MS = 5 * 60_000;
|
|
22
|
+
/** Upper bound for buffering a non-streaming response to read its usage. */
|
|
23
|
+
const USAGE_BUFFER_LIMIT = 16 * 1024 * 1024;
|
|
24
|
+
/** Entry paths accepted by the proxy, mapped to the protocol family they speak. */
|
|
25
|
+
const ENDPOINTS = {
|
|
26
|
+
'/v1/chat/completions': 'openai',
|
|
27
|
+
'/v1/responses': 'openai',
|
|
28
|
+
'/v1/messages': 'anthropic'
|
|
29
|
+
};
|
|
30
|
+
function modelList(config) {
|
|
31
|
+
const seen = new Set();
|
|
32
|
+
const data = config.routes
|
|
33
|
+
.filter((route) => route.enabled !== false && !seen.has(route.model) && seen.add(route.model))
|
|
34
|
+
.map((route) => ({ id: route.model, object: 'model', created: 0, owned_by: route.provider }));
|
|
35
|
+
return { object: 'list', data };
|
|
36
|
+
}
|
|
37
|
+
/** Rewrite the client-facing auth into the upstream auth convention for the target protocol. */
|
|
38
|
+
function upstreamHeaders(protocol, secret, accept) {
|
|
39
|
+
if (protocol === 'anthropic')
|
|
40
|
+
return { 'content-type': 'application/json', accept: accept ?? '*/*', 'x-api-key': secret, 'anthropic-version': ANTHROPIC_VERSION };
|
|
41
|
+
return { 'content-type': 'application/json', accept: accept ?? '*/*', authorization: `Bearer ${secret}` };
|
|
42
|
+
}
|
|
43
|
+
async function readBody(req, limit) {
|
|
44
|
+
const chunks = [];
|
|
45
|
+
let size = 0;
|
|
46
|
+
for await (const chunk of req) {
|
|
47
|
+
const part = Buffer.from(chunk);
|
|
48
|
+
size += part.length;
|
|
49
|
+
if (size > limit)
|
|
50
|
+
throw Object.assign(new Error('Request body too large'), { statusCode: 413 });
|
|
51
|
+
chunks.push(part);
|
|
52
|
+
}
|
|
53
|
+
return Buffer.concat(chunks);
|
|
54
|
+
}
|
|
55
|
+
function scanUsage(protocol, text) {
|
|
56
|
+
// Usage snapshots are cumulative within a stream (Anthropic message_start/message_delta each
|
|
57
|
+
// carry a running total), so the per-field max across snapshots is the stream total; summing
|
|
58
|
+
// would double-count.
|
|
59
|
+
let tokensIn = 0;
|
|
60
|
+
let tokensOut = 0;
|
|
61
|
+
for (const line of text.split('\n')) {
|
|
62
|
+
if (!line.startsWith('data: '))
|
|
63
|
+
continue;
|
|
64
|
+
try {
|
|
65
|
+
const payload = JSON.parse(line.slice(6));
|
|
66
|
+
// Anthropic's message_start nests usage under `message`; message_delta and OpenAI carry it at the top level.
|
|
67
|
+
const usage = payload.usage ?? payload.message?.usage;
|
|
68
|
+
if (!usage || typeof usage !== 'object')
|
|
69
|
+
continue;
|
|
70
|
+
if (protocol === 'openai') {
|
|
71
|
+
tokensIn = Math.max(tokensIn, Number(usage.prompt_tokens ?? usage.input_tokens ?? 0) || 0);
|
|
72
|
+
tokensOut = Math.max(tokensOut, Number(usage.completion_tokens ?? usage.output_tokens ?? 0) || 0);
|
|
73
|
+
}
|
|
74
|
+
else {
|
|
75
|
+
tokensIn = Math.max(tokensIn, Number(usage.input_tokens ?? 0) || 0);
|
|
76
|
+
tokensOut = Math.max(tokensOut, Number(usage.output_tokens ?? 0) || 0);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
catch { }
|
|
80
|
+
}
|
|
81
|
+
return { tokensIn, tokensOut };
|
|
82
|
+
}
|
|
83
|
+
function json(res, status, value) { res.writeHead(status, { 'content-type': 'application/json' }); res.end(JSON.stringify(value)); }
|
|
84
|
+
/** Abort reason carrier so timeouts map to 504 instead of a generic 502. */
|
|
85
|
+
function upstreamAbort(abort, message) {
|
|
86
|
+
const reason = new Error(message);
|
|
87
|
+
reason.upstreamTimeout = true;
|
|
88
|
+
abort.abort(reason);
|
|
89
|
+
}
|
|
90
|
+
/** Parse a Retry-After header (delta-seconds or HTTP-date) into a capped cooldown duration. */
|
|
91
|
+
function retryAfterMs(value) {
|
|
92
|
+
if (!value)
|
|
93
|
+
return undefined;
|
|
94
|
+
const seconds = Number(value);
|
|
95
|
+
if (Number.isFinite(seconds) && seconds >= 0)
|
|
96
|
+
return Math.min(seconds * 1000, RETRY_AFTER_MAX_MS);
|
|
97
|
+
const date = Date.parse(value);
|
|
98
|
+
if (!Number.isNaN(date))
|
|
99
|
+
return Math.min(Math.max(date - Date.now(), 0), RETRY_AFTER_MAX_MS);
|
|
100
|
+
return undefined;
|
|
101
|
+
}
|
|
102
|
+
export function createKhatServer(options) {
|
|
103
|
+
const states = options.states ?? {};
|
|
104
|
+
const secrets = { ...(options.secrets ?? {}) };
|
|
105
|
+
const currentSecret = (ref) => options.store?.vault.get(ref) ?? secrets[ref];
|
|
106
|
+
const selector = new WeightedSelector();
|
|
107
|
+
const statePath = options.statePath;
|
|
108
|
+
const logger = options.store ? new LogWriter(join(options.store.dir, 'logs')) : undefined;
|
|
109
|
+
const timeouts = () => ({ ...defaultTimeouts, ...options.config.timeouts });
|
|
110
|
+
const markFailure = async (providerId, keyId, status, retryAfterHeader) => {
|
|
111
|
+
const ref = `${providerId}/${keyId}`;
|
|
112
|
+
const at = new Date().toISOString();
|
|
113
|
+
if (status === 429) {
|
|
114
|
+
// 429 is transient: cool down and auto-revive at cooldownUntil instead of dying permanently.
|
|
115
|
+
const previous = states[ref];
|
|
116
|
+
const count = previous?.status === 'cooldown' ? (previous.cooldownCount ?? 0) + 1 : 1;
|
|
117
|
+
const waitMs = retryAfterMs(retryAfterHeader) ?? Math.min(COOLDOWN_BASE_MS * 2 ** (count - 1), COOLDOWN_MAX_MS);
|
|
118
|
+
states[ref] = { status: 'cooldown', cooldownUntil: new Date(Date.now() + waitMs).toISOString(), cooldownCount: count, lastError: { http: status, at } };
|
|
119
|
+
}
|
|
120
|
+
else {
|
|
121
|
+
// 401/402 mean the key itself is bad; recovery requires a probe, manual enable, or the background prober.
|
|
122
|
+
states[ref] = { status: 'unavailable', lastError: { http: status, at } };
|
|
123
|
+
}
|
|
124
|
+
if (statePath)
|
|
125
|
+
await saveJsonAtomic(statePath, { keys: states, counters: options.store?.counters ?? {} });
|
|
126
|
+
};
|
|
127
|
+
const markHealthy = async (ref) => {
|
|
128
|
+
// A success clears cooldown residue and resets the backoff counter.
|
|
129
|
+
delete states[ref];
|
|
130
|
+
if (statePath)
|
|
131
|
+
await saveJsonAtomic(statePath, { keys: states, counters: options.store?.counters ?? {} });
|
|
132
|
+
};
|
|
133
|
+
const server = createServer(async (req, res) => {
|
|
134
|
+
try {
|
|
135
|
+
if (req.url?.startsWith('/_keys')) {
|
|
136
|
+
if (!options.store)
|
|
137
|
+
return json(res, 503, { error: { message: 'Admin API is not available (no store attached)' } });
|
|
138
|
+
await handleAdmin(req, res, options.store, () => options.store?.vault.get(ACCESS_TOKEN_REF) ?? options.accessToken);
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
141
|
+
const currentAccessToken = options.store?.vault.get(ACCESS_TOKEN_REF) ?? options.accessToken;
|
|
142
|
+
if (!currentAccessToken || req.headers.authorization !== `Bearer ${currentAccessToken}`)
|
|
143
|
+
return json(res, 401, { error: { message: 'Unauthorized' } });
|
|
144
|
+
const pathname = new URL(req.url ?? '/', 'http://localhost').pathname;
|
|
145
|
+
if (req.method === 'GET' && (pathname === '/models' || pathname === '/v1/models'))
|
|
146
|
+
return json(res, 200, modelList(options.config));
|
|
147
|
+
if (req.method !== 'POST' || !(pathname in ENDPOINTS))
|
|
148
|
+
return json(res, 404, { error: { message: 'Not found' } });
|
|
149
|
+
const protocol = ENDPOINTS[pathname];
|
|
150
|
+
const body = await readBody(req, options.config.requestBodyLimitMB * 1024 * 1024);
|
|
151
|
+
let parsed;
|
|
152
|
+
try {
|
|
153
|
+
parsed = JSON.parse(body.toString('utf8'));
|
|
154
|
+
}
|
|
155
|
+
catch {
|
|
156
|
+
return json(res, 400, { error: { message: 'Invalid JSON' } });
|
|
157
|
+
}
|
|
158
|
+
if (!parsed.model || typeof parsed.model !== 'string')
|
|
159
|
+
return json(res, 400, { error: { message: 'model is required' } });
|
|
160
|
+
const route = findRoute(options.config, parsed.model);
|
|
161
|
+
const provider = resolveProvider(options.config, parsed.model);
|
|
162
|
+
if (!provider) {
|
|
163
|
+
if (isRouteDisabled(options.config, parsed.model))
|
|
164
|
+
return json(res, 403, { error: { message: `Route disabled for model: ${parsed.model}` } });
|
|
165
|
+
return json(res, 404, { error: { message: `No route for model: ${parsed.model}` } });
|
|
166
|
+
}
|
|
167
|
+
if (provider.protocol !== protocol)
|
|
168
|
+
return json(res, 400, { error: { message: `Model ${parsed.model} resolves to a ${provider.protocol} provider, but ${req.url} speaks ${protocol}` } });
|
|
169
|
+
const upstreamBody = JSON.stringify({ ...parsed, model: routeUpstreamModel(route) });
|
|
170
|
+
const tried = new Set();
|
|
171
|
+
while (true) {
|
|
172
|
+
const key = selector.select(provider.id, provider.keys.filter((item) => !tried.has(item.id)), states);
|
|
173
|
+
if (!key) {
|
|
174
|
+
const now = Date.now();
|
|
175
|
+
const waits = provider.keys
|
|
176
|
+
.map((item) => states[`${provider.id}/${item.id}`])
|
|
177
|
+
.filter((state) => state?.status === 'cooldown' && Date.parse(state.cooldownUntil ?? '') > now)
|
|
178
|
+
.map((state) => Date.parse(state.cooldownUntil));
|
|
179
|
+
const earliest = waits.length ? Math.min(...waits) : undefined;
|
|
180
|
+
const hint = earliest !== undefined ? ` (${waits.length} cooling down, earliest retry in ~${Math.ceil((earliest - now) / 1000)}s)` : '';
|
|
181
|
+
return json(res, 503, { error: { message: `All keys are unavailable${hint}`, keys: provider.keys.map((item) => ({ id: item.id, ...(states[`${provider.id}/${item.id}`] ?? { status: 'available' }) })) } });
|
|
182
|
+
}
|
|
183
|
+
tried.add(key.id);
|
|
184
|
+
const secret = currentSecret(key.vaultRef);
|
|
185
|
+
if (secret === undefined)
|
|
186
|
+
continue;
|
|
187
|
+
const abort = new AbortController();
|
|
188
|
+
const clientGone = () => abort.abort(new Error('client disconnected'));
|
|
189
|
+
res.on('close', clientGone);
|
|
190
|
+
const headerTimer = setTimeout(() => upstreamAbort(abort, 'upstream response header timeout'), timeouts().headerMs);
|
|
191
|
+
let upstream;
|
|
192
|
+
try {
|
|
193
|
+
upstream = await fetch(providerUrl(provider.baseUrl, pathname), { method: 'POST', headers: upstreamHeaders(protocol, secret, req.headers.accept), body: upstreamBody, signal: abort.signal });
|
|
194
|
+
}
|
|
195
|
+
catch (error) {
|
|
196
|
+
if (error?.upstreamTimeout)
|
|
197
|
+
error.statusCode = 504;
|
|
198
|
+
throw error;
|
|
199
|
+
}
|
|
200
|
+
finally {
|
|
201
|
+
clearTimeout(headerTimer);
|
|
202
|
+
}
|
|
203
|
+
if (RETRYABLE.has(upstream.status)) {
|
|
204
|
+
await markFailure(provider.id, key.id, upstream.status, upstream.headers.get('retry-after'));
|
|
205
|
+
if (tried.size < provider.keys.length) {
|
|
206
|
+
res.off('close', clientGone);
|
|
207
|
+
continue;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
const startedAt = Date.now();
|
|
211
|
+
let firstByteAt;
|
|
212
|
+
let bytes = 0;
|
|
213
|
+
let sseRemainder = '';
|
|
214
|
+
let tokensIn = 0;
|
|
215
|
+
let tokensOut = 0;
|
|
216
|
+
const responseHeaders = {};
|
|
217
|
+
upstream.headers.forEach((value, name) => { if (!HOP_BY_HOP.has(name))
|
|
218
|
+
responseHeaders[name] = value; });
|
|
219
|
+
res.writeHead(upstream.status, responseHeaders);
|
|
220
|
+
if (upstream.body) {
|
|
221
|
+
const idleTimer = setTimeout(() => upstreamAbort(abort, 'upstream stream idle timeout'), timeouts().streamIdleMs);
|
|
222
|
+
// Non-streaming JSON responses carry usage at the top level instead of in SSE lines;
|
|
223
|
+
// buffer them (bounded) so their tokens can be counted too.
|
|
224
|
+
const isEventStream = (upstream.headers.get('content-type') ?? '').toLowerCase().includes('text/event-stream');
|
|
225
|
+
let jsonBuffer = isEventStream ? undefined : [];
|
|
226
|
+
let jsonSize = 0;
|
|
227
|
+
try {
|
|
228
|
+
for await (const chunk of upstream.body) {
|
|
229
|
+
const buffer = Buffer.from(chunk);
|
|
230
|
+
if (firstByteAt === undefined)
|
|
231
|
+
firstByteAt = Date.now();
|
|
232
|
+
bytes += buffer.length;
|
|
233
|
+
if (isEventStream) {
|
|
234
|
+
const combined = sseRemainder + buffer.toString('utf8');
|
|
235
|
+
const lastNewline = combined.lastIndexOf('\n');
|
|
236
|
+
if (lastNewline >= 0) {
|
|
237
|
+
const usage = scanUsage(protocol, combined.slice(0, lastNewline + 1));
|
|
238
|
+
tokensIn = Math.max(tokensIn, usage.tokensIn);
|
|
239
|
+
tokensOut = Math.max(tokensOut, usage.tokensOut);
|
|
240
|
+
sseRemainder = combined.slice(lastNewline + 1);
|
|
241
|
+
}
|
|
242
|
+
else
|
|
243
|
+
sseRemainder = combined;
|
|
244
|
+
}
|
|
245
|
+
if (jsonBuffer) {
|
|
246
|
+
jsonSize += buffer.length;
|
|
247
|
+
if (jsonSize > USAGE_BUFFER_LIMIT)
|
|
248
|
+
jsonBuffer = undefined;
|
|
249
|
+
else
|
|
250
|
+
jsonBuffer.push(buffer);
|
|
251
|
+
}
|
|
252
|
+
res.write(buffer);
|
|
253
|
+
idleTimer.refresh();
|
|
254
|
+
}
|
|
255
|
+
if (isEventStream) {
|
|
256
|
+
const usage = scanUsage(protocol, sseRemainder);
|
|
257
|
+
tokensIn = Math.max(tokensIn, usage.tokensIn);
|
|
258
|
+
tokensOut = Math.max(tokensOut, usage.tokensOut);
|
|
259
|
+
}
|
|
260
|
+
if (jsonBuffer) {
|
|
261
|
+
try {
|
|
262
|
+
const payload = JSON.parse(Buffer.concat(jsonBuffer).toString('utf8'));
|
|
263
|
+
const jsonUsage = payload?.usage;
|
|
264
|
+
if (jsonUsage && typeof jsonUsage === 'object') {
|
|
265
|
+
tokensIn = Math.max(tokensIn, Number(jsonUsage.prompt_tokens ?? jsonUsage.input_tokens ?? 0) || 0);
|
|
266
|
+
tokensOut = Math.max(tokensOut, Number(jsonUsage.completion_tokens ?? jsonUsage.output_tokens ?? 0) || 0);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
catch { }
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
catch (error) {
|
|
273
|
+
if (error?.upstreamTimeout)
|
|
274
|
+
res.destroy(error);
|
|
275
|
+
throw error;
|
|
276
|
+
}
|
|
277
|
+
finally {
|
|
278
|
+
clearTimeout(idleTimer);
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
const durationMs = Date.now() - startedAt;
|
|
282
|
+
const ttfbMs = (firstByteAt ?? Date.now()) - startedAt;
|
|
283
|
+
const keyRef = `${provider.id}/${key.id}`;
|
|
284
|
+
if (upstream.ok && states[keyRef])
|
|
285
|
+
await markHealthy(keyRef);
|
|
286
|
+
options.store?.recordCounter(keyRef, { requests: 1, failed: upstream.ok ? 0 : 1, bytesOut: bytes, tokensIn, tokensOut });
|
|
287
|
+
try {
|
|
288
|
+
await logger?.append({ ts: new Date().toISOString(), event: 'forward', model: parsed.model, provider: provider.id, key: keyRef, status: upstream.status, ttfbMs, durationMs, bytes, tokensIn, tokensOut });
|
|
289
|
+
}
|
|
290
|
+
catch (error) {
|
|
291
|
+
console.error(`[khat] failed to write request log: ${error?.message ?? error}`);
|
|
292
|
+
}
|
|
293
|
+
res.end();
|
|
294
|
+
res.off('close', clientGone);
|
|
295
|
+
return;
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
catch (error) {
|
|
299
|
+
if (!res.headersSent)
|
|
300
|
+
json(res, error.statusCode ?? 502, { error: { message: error.message ?? 'Proxy error' } });
|
|
301
|
+
else
|
|
302
|
+
res.destroy(error);
|
|
303
|
+
}
|
|
304
|
+
});
|
|
305
|
+
const configWatcher = options.store ? watch(options.store.dir, (_event, filename) => {
|
|
306
|
+
if (filename?.toString() !== basename(options.store.configPath))
|
|
307
|
+
return;
|
|
308
|
+
void (async () => {
|
|
309
|
+
try {
|
|
310
|
+
const next = await readJsonFile(options.store.configPath);
|
|
311
|
+
if (next === undefined)
|
|
312
|
+
return;
|
|
313
|
+
const validated = validateConfig(next);
|
|
314
|
+
// The daemon is the single writer (ADR-0005): a disk copy older than the in-memory config
|
|
315
|
+
// can only be a stale echo of our own write, never a genuine external edit.
|
|
316
|
+
const diskRevision = validated.revision;
|
|
317
|
+
const memoryRevision = options.config.revision;
|
|
318
|
+
if (diskRevision !== undefined && memoryRevision !== undefined && diskRevision < memoryRevision)
|
|
319
|
+
return;
|
|
320
|
+
// The socket keeps the address it was started with; adopting a new bind/port here would
|
|
321
|
+
// make status report an endpoint the process is not actually listening on.
|
|
322
|
+
if (validated.bind !== options.config.bind || validated.port !== options.config.port) {
|
|
323
|
+
console.error(`[khat] bind/port change requires a restart; still listening on ${options.config.bind}:${options.config.port}`);
|
|
324
|
+
validated.port = options.config.port;
|
|
325
|
+
validated.bind = options.config.bind;
|
|
326
|
+
}
|
|
327
|
+
Object.assign(options.config, validated);
|
|
328
|
+
for (const key of Object.keys(secrets))
|
|
329
|
+
delete secrets[key];
|
|
330
|
+
for (const provider of validated.providers)
|
|
331
|
+
for (const key of provider.keys) {
|
|
332
|
+
const secret = options.store?.vault.get(key.vaultRef);
|
|
333
|
+
if (secret !== undefined)
|
|
334
|
+
secrets[key.vaultRef] = secret;
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
catch (error) {
|
|
338
|
+
console.error(`[khat] ignored invalid external config update: ${error?.message ?? error}`);
|
|
339
|
+
}
|
|
340
|
+
})();
|
|
341
|
+
}) : undefined;
|
|
342
|
+
// Cooldown keys revive on their own at cooldownUntil (probing them early would just re-429
|
|
343
|
+
// and extend the cooldown), so the prober only targets permanently unavailable keys.
|
|
344
|
+
const probeIntervalMs = options.probeIntervalMs ?? PROBE_INTERVAL_MS;
|
|
345
|
+
let probeInFlight = false;
|
|
346
|
+
const runProbeCycle = async () => {
|
|
347
|
+
if (probeInFlight)
|
|
348
|
+
return;
|
|
349
|
+
probeInFlight = true;
|
|
350
|
+
try {
|
|
351
|
+
for (const provider of options.config.providers) {
|
|
352
|
+
const route = options.config.routes.find((item) => item.provider === provider.id);
|
|
353
|
+
if (!route)
|
|
354
|
+
continue;
|
|
355
|
+
for (const target of provider.keys) {
|
|
356
|
+
if (states[`${provider.id}/${target.id}`]?.status !== 'unavailable')
|
|
357
|
+
continue;
|
|
358
|
+
try {
|
|
359
|
+
const secret = currentSecret(target.vaultRef);
|
|
360
|
+
if (secret === undefined)
|
|
361
|
+
continue;
|
|
362
|
+
const status = await probeKey(provider, routeUpstreamModel(route), secret);
|
|
363
|
+
if (status >= 200 && status < 300) {
|
|
364
|
+
await options.store.mutate(() => enableKey(options.store, provider.id, target.id));
|
|
365
|
+
await logger?.append({ ts: new Date().toISOString(), event: 'probe', provider: provider.id, key: `${provider.id}/${target.id}`, status, outcome: 'enabled' });
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
catch (error) {
|
|
369
|
+
console.error(`[khat] background probe failed for ${provider.id}/${target.id}: ${error?.message ?? error}`);
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
finally {
|
|
375
|
+
probeInFlight = false;
|
|
376
|
+
}
|
|
377
|
+
};
|
|
378
|
+
const prober = options.store && probeIntervalMs > 0 ? setInterval(() => { void runProbeCycle(); }, probeIntervalMs) : undefined;
|
|
379
|
+
prober?.unref();
|
|
380
|
+
server.once('close', () => { configWatcher?.close(); if (prober)
|
|
381
|
+
clearInterval(prober); });
|
|
382
|
+
return server;
|
|
383
|
+
}
|
package/dist/service.js
ADDED
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
import { execFile } from 'node:child_process';
|
|
2
|
+
import { access, mkdir, unlink, writeFile } from 'node:fs/promises';
|
|
3
|
+
import { homedir } from 'node:os';
|
|
4
|
+
import { dirname, join } from 'node:path';
|
|
5
|
+
import { fileURLToPath } from 'node:url';
|
|
6
|
+
import { promisify } from 'node:util';
|
|
7
|
+
const execFileAsync = promisify(execFile);
|
|
8
|
+
const WINDOWS_TASK_NAME = 'khat';
|
|
9
|
+
const LAUNCHD_LABEL = 'local.khat';
|
|
10
|
+
const SYSTEMD_UNIT = 'khat';
|
|
11
|
+
async function defaultRunner(command, args) {
|
|
12
|
+
try {
|
|
13
|
+
const { stdout, stderr } = await execFileAsync(command, args, { windowsHide: true, timeout: 30_000 });
|
|
14
|
+
return { code: 0, stdout: String(stdout), stderr: String(stderr) };
|
|
15
|
+
}
|
|
16
|
+
catch (error) {
|
|
17
|
+
if (error?.code === 'ENOENT')
|
|
18
|
+
return { code: null, stdout: '', stderr: `${command} is not available` };
|
|
19
|
+
return { code: typeof error?.code === 'number' ? error.code : null, stdout: String(error?.stdout ?? ''), stderr: String(error?.stderr ?? error?.message ?? 'command failed') };
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
async function fileExists(path) {
|
|
23
|
+
try {
|
|
24
|
+
await access(path);
|
|
25
|
+
return true;
|
|
26
|
+
}
|
|
27
|
+
catch {
|
|
28
|
+
return false;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
function xmlEscape(value) {
|
|
32
|
+
return value.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"');
|
|
33
|
+
}
|
|
34
|
+
function resolveOptions(options) {
|
|
35
|
+
const home = options.homeDir ?? homedir();
|
|
36
|
+
return {
|
|
37
|
+
runner: options.run ?? defaultRunner,
|
|
38
|
+
homeDir: home,
|
|
39
|
+
dataDir: options.dataDir ?? process.env.KHAT_HOME ?? join(home, '.khat'),
|
|
40
|
+
distDir: options.distDir ?? dirname(fileURLToPath(new URL('./supervisor.js', import.meta.url))),
|
|
41
|
+
execPath: options.execPath ?? process.execPath,
|
|
42
|
+
userId: options.userId ?? (process.env.USERDOMAIN && process.env.USERNAME ? `${process.env.USERDOMAIN}\\${process.env.USERNAME}` : undefined),
|
|
43
|
+
uid: options.uid ?? (typeof process.getuid === 'function' ? process.getuid() : undefined)
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
function commandFailure(result, action) {
|
|
47
|
+
return `${action}: ${(result.stderr || result.stdout).trim() || `command exited with code ${result.code}`}`;
|
|
48
|
+
}
|
|
49
|
+
export function buildTaskXml(options) {
|
|
50
|
+
const triggerUser = options.userId ? `\n <UserId>${xmlEscape(options.userId)}</UserId>` : '';
|
|
51
|
+
return `<?xml version="1.0" encoding="UTF-16"?>
|
|
52
|
+
<Task version="1.2" xmlns="http://schemas.microsoft.com/windows/2004/02/mit/task">
|
|
53
|
+
<RegistrationInfo>
|
|
54
|
+
<Description>Khat proxy supervisor; starts the khat daemon when you log on</Description>
|
|
55
|
+
</RegistrationInfo>
|
|
56
|
+
<Triggers>
|
|
57
|
+
<LogonTrigger>
|
|
58
|
+
<Enabled>true</Enabled>${triggerUser}
|
|
59
|
+
</LogonTrigger>
|
|
60
|
+
</Triggers>
|
|
61
|
+
<Principals>
|
|
62
|
+
<Principal id="Author">
|
|
63
|
+
<LogonType>InteractiveToken</LogonType>
|
|
64
|
+
<RunLevel>LeastPrivilege</RunLevel>
|
|
65
|
+
</Principal>
|
|
66
|
+
</Principals>
|
|
67
|
+
<Settings>
|
|
68
|
+
<MultipleInstancesPolicy>IgnoreNew</MultipleInstancesPolicy>
|
|
69
|
+
<DisallowStartIfOnBatteries>false</DisallowStartIfOnBatteries>
|
|
70
|
+
<StopIfGoingOnBatteries>false</StopIfGoingOnBatteries>
|
|
71
|
+
<StartWhenAvailable>true</StartWhenAvailable>
|
|
72
|
+
<RunOnlyIfIdle>false</RunOnlyIfIdle>
|
|
73
|
+
<WakeToRun>false</WakeToRun>
|
|
74
|
+
<ExecutionTimeLimit>PT0S</ExecutionTimeLimit>
|
|
75
|
+
<Priority>7</Priority>
|
|
76
|
+
</Settings>
|
|
77
|
+
<Actions Context="Author">
|
|
78
|
+
<Exec>
|
|
79
|
+
<Command>${xmlEscape(options.command)}</Command>
|
|
80
|
+
<Arguments>${xmlEscape(options.arguments)}</Arguments>
|
|
81
|
+
</Exec>
|
|
82
|
+
</Actions>
|
|
83
|
+
</Task>`;
|
|
84
|
+
}
|
|
85
|
+
export function buildLaunchdPlist(options) {
|
|
86
|
+
const programArgs = options.programArgs.map((arg) => ` <string>${xmlEscape(arg)}</string>`).join('\n');
|
|
87
|
+
const env = Object.entries(options.env).map(([key, value]) => ` <key>${xmlEscape(key)}</key>\n <string>${xmlEscape(value)}</string>`).join('\n');
|
|
88
|
+
return `<?xml version="1.0" encoding="UTF-8"?>
|
|
89
|
+
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
90
|
+
<plist version="1.0">
|
|
91
|
+
<dict>
|
|
92
|
+
<key>Label</key>
|
|
93
|
+
<string>${xmlEscape(options.label)}</string>
|
|
94
|
+
<key>ProgramArguments</key>
|
|
95
|
+
<array>
|
|
96
|
+
<string>${xmlEscape(options.program)}</string>
|
|
97
|
+
${programArgs}
|
|
98
|
+
</array>
|
|
99
|
+
<key>WorkingDirectory</key>
|
|
100
|
+
<string>${xmlEscape(options.workDir)}</string>
|
|
101
|
+
<key>EnvironmentVariables</key>
|
|
102
|
+
<dict>
|
|
103
|
+
${env}
|
|
104
|
+
</dict>
|
|
105
|
+
<key>RunAtLoad</key>
|
|
106
|
+
<true/>
|
|
107
|
+
<key>StandardOutPath</key>
|
|
108
|
+
<string>${xmlEscape(options.logPath)}</string>
|
|
109
|
+
<key>StandardErrorPath</key>
|
|
110
|
+
<string>${xmlEscape(options.logPath)}</string>
|
|
111
|
+
</dict>
|
|
112
|
+
</plist>`;
|
|
113
|
+
}
|
|
114
|
+
export function buildSystemdUnit(options) {
|
|
115
|
+
const execStart = options.execStart.map((part) => `"${part}"`).join(' ');
|
|
116
|
+
const env = Object.entries(options.env).map(([key, value]) => `Environment="${key}=${value}"`).join('\n');
|
|
117
|
+
return `[Unit]
|
|
118
|
+
Description=Khat proxy supervisor
|
|
119
|
+
|
|
120
|
+
[Service]
|
|
121
|
+
ExecStart=${execStart}
|
|
122
|
+
WorkingDirectory="${options.workDir}"
|
|
123
|
+
${env}
|
|
124
|
+
Restart=no
|
|
125
|
+
|
|
126
|
+
[Install]
|
|
127
|
+
WantedBy=default.target
|
|
128
|
+
`;
|
|
129
|
+
}
|
|
130
|
+
// Windows: a scheduled task with an interactive token, not an NSSM service —
|
|
131
|
+
// the vault master key is protected with user-scope DPAPI (ADR 0004), which
|
|
132
|
+
// LocalSystem cannot decrypt, and NSSM cannot run as the current user without
|
|
133
|
+
// that user's password.
|
|
134
|
+
function windowsManager(options) {
|
|
135
|
+
const { runner, dataDir, distDir, execPath, userId } = resolveOptions(options);
|
|
136
|
+
const supervisorPath = join(distDir, 'supervisor.js');
|
|
137
|
+
async function install({ startNow = true } = {}) {
|
|
138
|
+
await mkdir(dataDir, { recursive: true });
|
|
139
|
+
const xmlPath = join(dataDir, 'service-task.xml');
|
|
140
|
+
// The task XML cannot set environment variables, so a cmd wrapper pins
|
|
141
|
+
// KHAT_HOME; the quoted `set` form avoids trailing-space capture.
|
|
142
|
+
const arguments_ = `/c set "KHAT_HOME=${dataDir}" && "${execPath}" "${supervisorPath}"`;
|
|
143
|
+
const xml = buildTaskXml({ userId, command: 'cmd.exe', arguments: arguments_ });
|
|
144
|
+
await writeFile(xmlPath, Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from(xml, 'utf16le')]));
|
|
145
|
+
const created = await runner('schtasks', ['/Create', '/F', '/TN', WINDOWS_TASK_NAME, '/XML', xmlPath]);
|
|
146
|
+
if (created.code !== 0)
|
|
147
|
+
throw new Error(`${commandFailure(created, 'could not register the scheduled task')} (definition kept at ${xmlPath})`);
|
|
148
|
+
await unlink(xmlPath).catch(() => undefined);
|
|
149
|
+
if (!startNow)
|
|
150
|
+
return `scheduled task '${WINDOWS_TASK_NAME}' registered; it auto-starts the daemon at your next logon`;
|
|
151
|
+
const started = await runner('schtasks', ['/Run', '/TN', WINDOWS_TASK_NAME]);
|
|
152
|
+
if (started.code !== 0)
|
|
153
|
+
throw new Error(commandFailure(started, 'task registered but could not be started'));
|
|
154
|
+
return `scheduled task '${WINDOWS_TASK_NAME}' registered and running; it auto-starts at logon`;
|
|
155
|
+
}
|
|
156
|
+
async function uninstall() {
|
|
157
|
+
const query = await runner('schtasks', ['/Query', '/TN', WINDOWS_TASK_NAME]);
|
|
158
|
+
if (query.code !== 0)
|
|
159
|
+
return 'khat service is not installed';
|
|
160
|
+
const removed = await runner('schtasks', ['/Delete', '/F', '/TN', WINDOWS_TASK_NAME]);
|
|
161
|
+
if (removed.code !== 0)
|
|
162
|
+
throw new Error(commandFailure(removed, 'could not remove the scheduled task'));
|
|
163
|
+
return `scheduled task '${WINDOWS_TASK_NAME}' removed`;
|
|
164
|
+
}
|
|
165
|
+
async function status() {
|
|
166
|
+
const query = await runner('schtasks', ['/Query', '/TN', WINDOWS_TASK_NAME]);
|
|
167
|
+
if (query.code !== 0)
|
|
168
|
+
return { installed: false, enabled: null, running: null, detail: '' };
|
|
169
|
+
// schtasks /Query status text is localized; PowerShell's StateEnum is not.
|
|
170
|
+
const state = await runner('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', `(Get-ScheduledTask -TaskName '${WINDOWS_TASK_NAME}' -ErrorAction SilentlyContinue).State`]);
|
|
171
|
+
const value = Number.parseInt(state.stdout.trim(), 10);
|
|
172
|
+
if (!Number.isInteger(value))
|
|
173
|
+
return { installed: true, enabled: null, running: null, detail: 'task state could not be read' };
|
|
174
|
+
return {
|
|
175
|
+
installed: true,
|
|
176
|
+
enabled: value !== 1,
|
|
177
|
+
running: value === 4,
|
|
178
|
+
detail: value === 4 ? 'task running' : value === 3 ? 'task ready (not running)' : value === 1 ? 'task disabled' : `task state ${value}`
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
return { install, uninstall, status };
|
|
182
|
+
}
|
|
183
|
+
// macOS: launchd user agent. RunAtLoad starts it at login; KeepAlive stays off
|
|
184
|
+
// so the supervisor's own crash circuit breaker is the only restart authority.
|
|
185
|
+
function darwinManager(options) {
|
|
186
|
+
const { runner, homeDir, dataDir, distDir, execPath, uid } = resolveOptions(options);
|
|
187
|
+
const supervisorPath = join(distDir, 'supervisor.js');
|
|
188
|
+
const plistPath = join(homeDir, 'Library', 'LaunchAgents', `${LAUNCHD_LABEL}.plist`);
|
|
189
|
+
const target = `gui/${uid}/${LAUNCHD_LABEL}`;
|
|
190
|
+
async function install({ startNow = true } = {}) {
|
|
191
|
+
if (uid === undefined)
|
|
192
|
+
throw new Error('could not determine the user id for launchctl');
|
|
193
|
+
await mkdir(dirname(plistPath), { recursive: true });
|
|
194
|
+
await mkdir(dataDir, { recursive: true });
|
|
195
|
+
const plist = buildLaunchdPlist({
|
|
196
|
+
label: LAUNCHD_LABEL,
|
|
197
|
+
program: execPath,
|
|
198
|
+
programArgs: [supervisorPath],
|
|
199
|
+
workDir: distDir,
|
|
200
|
+
env: { KHAT_HOME: dataDir },
|
|
201
|
+
logPath: join(dataDir, 'service.log')
|
|
202
|
+
});
|
|
203
|
+
await writeFile(plistPath, plist, 'utf8');
|
|
204
|
+
if (!startNow)
|
|
205
|
+
return `launch agent ${LAUNCHD_LABEL} installed; it auto-starts the daemon at your next logon`;
|
|
206
|
+
await runner('launchctl', ['bootout', target]);
|
|
207
|
+
const boot = await runner('launchctl', ['bootstrap', `gui/${uid}`, plistPath]);
|
|
208
|
+
if (boot.code !== 0)
|
|
209
|
+
throw new Error(commandFailure(boot, 'could not load the launch agent'));
|
|
210
|
+
return `launch agent ${LAUNCHD_LABEL} installed and running; it auto-starts at logon`;
|
|
211
|
+
}
|
|
212
|
+
async function uninstall() {
|
|
213
|
+
if (!(await fileExists(plistPath)))
|
|
214
|
+
return 'khat service is not installed';
|
|
215
|
+
await runner('launchctl', ['bootout', target]);
|
|
216
|
+
await unlink(plistPath).catch(() => undefined);
|
|
217
|
+
return `launch agent ${LAUNCHD_LABEL} removed`;
|
|
218
|
+
}
|
|
219
|
+
async function status() {
|
|
220
|
+
if (!(await fileExists(plistPath)))
|
|
221
|
+
return { installed: false, enabled: null, running: null, detail: '' };
|
|
222
|
+
const printed = await runner('launchctl', ['print', target]);
|
|
223
|
+
if (printed.code !== 0)
|
|
224
|
+
return { installed: true, enabled: true, running: false, detail: 'not loaded; starts at next logon' };
|
|
225
|
+
const state = printed.stdout.match(/state\s*=\s*(\S+)/)?.[1] ?? 'unknown';
|
|
226
|
+
return { installed: true, enabled: true, running: state === 'running', detail: state === 'running' ? 'agent running' : `agent state: ${state}` };
|
|
227
|
+
}
|
|
228
|
+
return { install, uninstall, status };
|
|
229
|
+
}
|
|
230
|
+
// Linux: systemd user unit. Restart=no — the supervisor owns crash backoff;
|
|
231
|
+
// the unit only has to launch it once per login/boot.
|
|
232
|
+
function linuxManager(options) {
|
|
233
|
+
const { runner, homeDir, dataDir, distDir, execPath } = resolveOptions(options);
|
|
234
|
+
const supervisorPath = join(distDir, 'supervisor.js');
|
|
235
|
+
const unitPath = join(homeDir, '.config', 'systemd', 'user', `${SYSTEMD_UNIT}.service`);
|
|
236
|
+
async function install({ startNow = true } = {}) {
|
|
237
|
+
await mkdir(dirname(unitPath), { recursive: true });
|
|
238
|
+
const unit = buildSystemdUnit({ execStart: [execPath, supervisorPath], workDir: distDir, env: { KHAT_HOME: dataDir } });
|
|
239
|
+
await writeFile(unitPath, unit, 'utf8');
|
|
240
|
+
const reloaded = await runner('systemctl', ['--user', 'daemon-reload']);
|
|
241
|
+
if (reloaded.code !== 0)
|
|
242
|
+
throw new Error(commandFailure(reloaded, 'could not reload the systemd user units'));
|
|
243
|
+
const enable = await runner('systemctl', ['--user', ...(startNow ? ['enable', '--now'] : ['enable']), SYSTEMD_UNIT]);
|
|
244
|
+
if (enable.code !== 0)
|
|
245
|
+
throw new Error(commandFailure(enable, 'could not enable the service'));
|
|
246
|
+
// Without lingering, user units only run while the user is logged in.
|
|
247
|
+
const linger = await runner('loginctl', ['enable-linger']);
|
|
248
|
+
const lingerNote = linger.code === 0 ? '' : ' (warning: could not enable lingering, so the service starts at login rather than at boot)';
|
|
249
|
+
return `systemd user service ${SYSTEMD_UNIT} installed${startNow ? ' and running' : ''}; it auto-starts at login${lingerNote}`;
|
|
250
|
+
}
|
|
251
|
+
async function uninstall() {
|
|
252
|
+
if (!(await fileExists(unitPath)))
|
|
253
|
+
return 'khat service is not installed';
|
|
254
|
+
await runner('systemctl', ['--user', 'disable', '--now', SYSTEMD_UNIT]);
|
|
255
|
+
await unlink(unitPath).catch(() => undefined);
|
|
256
|
+
await runner('systemctl', ['--user', 'daemon-reload']);
|
|
257
|
+
return `systemd user service ${SYSTEMD_UNIT} removed`;
|
|
258
|
+
}
|
|
259
|
+
async function status() {
|
|
260
|
+
if (!(await fileExists(unitPath)))
|
|
261
|
+
return { installed: false, enabled: null, running: null, detail: '' };
|
|
262
|
+
const enabledResult = await runner('systemctl', ['--user', 'is-enabled', SYSTEMD_UNIT]);
|
|
263
|
+
const activeResult = await runner('systemctl', ['--user', 'is-active', SYSTEMD_UNIT]);
|
|
264
|
+
const enabledText = enabledResult.stdout.trim();
|
|
265
|
+
const activeText = activeResult.stdout.trim();
|
|
266
|
+
return {
|
|
267
|
+
installed: true,
|
|
268
|
+
enabled: enabledText === 'enabled' ? true : enabledText === 'disabled' ? false : null,
|
|
269
|
+
running: activeText === 'active',
|
|
270
|
+
detail: activeText === 'active' ? 'unit active' : `unit ${activeText || 'unknown'}`
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
return { install, uninstall, status };
|
|
274
|
+
}
|
|
275
|
+
export function createServiceManager(platform, options = {}) {
|
|
276
|
+
if (platform === 'win32')
|
|
277
|
+
return windowsManager(options);
|
|
278
|
+
if (platform === 'darwin')
|
|
279
|
+
return darwinManager(options);
|
|
280
|
+
if (platform === 'linux')
|
|
281
|
+
return linuxManager(options);
|
|
282
|
+
throw new Error(`unsupported platform '${platform}': khat service requires Windows Task Scheduler, macOS launchd, or Linux systemd`);
|
|
283
|
+
}
|