llmspy 0.0.0-stage → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +369 -2
- package/llms.txt +189 -0
- package/package.json +30 -5
- package/proxy/dist/capture.js +230 -0
- package/proxy/dist/cli.js +42 -0
- package/proxy/dist/command.js +50 -0
- package/proxy/dist/conversation.js +102 -0
- package/proxy/dist/envfile.js +22 -0
- package/proxy/dist/failed.js +11 -0
- package/proxy/dist/guard.js +45 -0
- package/proxy/dist/index.js +95 -0
- package/proxy/dist/parsers/anthropic.js +197 -0
- package/proxy/dist/parsers/error.js +17 -0
- package/proxy/dist/parsers/index.js +98 -0
- package/proxy/dist/parsers/json.js +31 -0
- package/proxy/dist/parsers/openai-chat.js +222 -0
- package/proxy/dist/parsers/openai-common.js +18 -0
- package/proxy/dist/parsers/openai-responses.js +279 -0
- package/proxy/dist/parsers/sse.js +68 -0
- package/proxy/dist/parsers/types.js +1 -0
- package/proxy/dist/pidfile.js +57 -0
- package/proxy/dist/proxy.js +353 -0
- package/proxy/dist/query.js +111 -0
- package/proxy/dist/static.js +61 -0
- package/proxy/dist/stats.js +71 -0
- package/proxy/dist/stop.js +68 -0
- package/proxy/dist/store.js +201 -0
- package/proxy/dist/summary.js +50 -0
- package/proxy/dist/types.js +1 -0
- package/proxy/dist/upstreams.js +85 -0
- package/proxy/dist/version.js +14 -0
- package/proxy/package.json +6 -0
- package/ui/dist/assets/index-B9XIilZJ.js +24 -0
- package/ui/dist/assets/index-moM-ONoJ.css +1 -0
- package/ui/dist/index.html +21 -0
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
import { toLlmError } from './error.js';
|
|
2
|
+
import { asArray, asNumber, asRecord, asString, isRecord, parseJsonOrKeep, } from './json.js';
|
|
3
|
+
import { partsText, toRequest } from './openai-common.js';
|
|
4
|
+
// OpenAI Responses API: POST /v1/responses. Input and output are lists of
|
|
5
|
+
// items (messages, reasoning, tool calls, tool outputs), not messages. With
|
|
6
|
+
// `previous_response_id` the earlier conversation is kept by OpenAI and only
|
|
7
|
+
// the new input is in the request.
|
|
8
|
+
/** Message content is a string or an array of parts. */
|
|
9
|
+
function contentParts(content) {
|
|
10
|
+
if (typeof content === 'string')
|
|
11
|
+
return [{ type: 'text', text: content }];
|
|
12
|
+
return asArray(content).map((part) => {
|
|
13
|
+
const p = asRecord(part);
|
|
14
|
+
switch (p.type) {
|
|
15
|
+
case 'input_text':
|
|
16
|
+
case 'output_text':
|
|
17
|
+
return { type: 'text', text: asString(p.text) ?? '' };
|
|
18
|
+
case 'refusal':
|
|
19
|
+
return { type: 'text', text: asString(p.refusal) ?? '' };
|
|
20
|
+
case 'input_image':
|
|
21
|
+
return { type: 'image' };
|
|
22
|
+
default:
|
|
23
|
+
return { type: 'other', raw: part };
|
|
24
|
+
}
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
/** Content as text; non-text parts become `[type]`. */
|
|
28
|
+
function contentText(content) {
|
|
29
|
+
return partsText(contentParts(content));
|
|
30
|
+
}
|
|
31
|
+
/** A tool output is a string, an array of content parts, or an object. */
|
|
32
|
+
function outputText(output) {
|
|
33
|
+
if (output === undefined || typeof output === 'string')
|
|
34
|
+
return output ?? '';
|
|
35
|
+
if (Array.isArray(output))
|
|
36
|
+
return contentText(output);
|
|
37
|
+
return JSON.stringify(output);
|
|
38
|
+
}
|
|
39
|
+
/** The parts of an item the model produced: a message, reasoning or a tool call. */
|
|
40
|
+
function outputParts(raw) {
|
|
41
|
+
const item = asRecord(raw);
|
|
42
|
+
switch (item.type) {
|
|
43
|
+
case 'message':
|
|
44
|
+
return contentParts(item.content);
|
|
45
|
+
case 'reasoning': {
|
|
46
|
+
// Reasoning is often encrypted only; then the part is empty but shows it happened.
|
|
47
|
+
const text = [...asArray(item.summary), ...asArray(item.content)]
|
|
48
|
+
.map((part) => asString(asRecord(part).text))
|
|
49
|
+
.filter((text) => text !== undefined)
|
|
50
|
+
.join('\n\n');
|
|
51
|
+
return [{ type: 'thinking', text }];
|
|
52
|
+
}
|
|
53
|
+
case 'function_call': {
|
|
54
|
+
const args = item.arguments;
|
|
55
|
+
return [
|
|
56
|
+
{
|
|
57
|
+
type: 'tool_call',
|
|
58
|
+
id: asString(item.call_id) ?? '',
|
|
59
|
+
name: asString(item.name) ?? '',
|
|
60
|
+
input: typeof args === 'string' ? parseJsonOrKeep(args) : args,
|
|
61
|
+
},
|
|
62
|
+
];
|
|
63
|
+
}
|
|
64
|
+
case 'custom_tool_call':
|
|
65
|
+
return [
|
|
66
|
+
{
|
|
67
|
+
type: 'tool_call',
|
|
68
|
+
id: asString(item.call_id) ?? '',
|
|
69
|
+
name: asString(item.name) ?? '',
|
|
70
|
+
input: item.input,
|
|
71
|
+
},
|
|
72
|
+
];
|
|
73
|
+
default:
|
|
74
|
+
// Built-in tool calls (web search, file search, MCP and others).
|
|
75
|
+
return [{ type: 'other', raw }];
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
/** Appends to the last message when it is the assistant's, so one turn is one message. */
|
|
79
|
+
function pushAssistant(messages, parts) {
|
|
80
|
+
const last = messages.at(-1);
|
|
81
|
+
if (last?.role === 'assistant')
|
|
82
|
+
last.parts.push(...parts);
|
|
83
|
+
else
|
|
84
|
+
messages.push({ role: 'assistant', parts });
|
|
85
|
+
}
|
|
86
|
+
function toUsage(usage) {
|
|
87
|
+
if (!isRecord(usage))
|
|
88
|
+
return undefined;
|
|
89
|
+
return {
|
|
90
|
+
input: asNumber(usage.input_tokens),
|
|
91
|
+
output: asNumber(usage.output_tokens),
|
|
92
|
+
cacheRead: asNumber(asRecord(usage.input_tokens_details).cached_tokens),
|
|
93
|
+
reasoning: asNumber(asRecord(usage.output_tokens_details).reasoning_tokens),
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
/** Function and custom tools have a name; built-in tools are named by their MCP server or type. */
|
|
97
|
+
function toTool(tool) {
|
|
98
|
+
const t = asRecord(tool);
|
|
99
|
+
return {
|
|
100
|
+
name: asString(t.name) ??
|
|
101
|
+
asString(t.server_label) ??
|
|
102
|
+
asString(t.type) ??
|
|
103
|
+
'',
|
|
104
|
+
description: asString(t.description),
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
function parseRequest(body) {
|
|
108
|
+
const b = asRecord(body);
|
|
109
|
+
const system = [];
|
|
110
|
+
const messages = [];
|
|
111
|
+
const instructions = asString(b.instructions);
|
|
112
|
+
if (instructions)
|
|
113
|
+
system.push(instructions);
|
|
114
|
+
const input = typeof b.input === 'string' ? [{ content: b.input }] : b.input;
|
|
115
|
+
for (const raw of asArray(input)) {
|
|
116
|
+
const item = asRecord(raw);
|
|
117
|
+
// Messages may leave out `type`.
|
|
118
|
+
const type = asString(item.type) ?? 'message';
|
|
119
|
+
if (type === 'message') {
|
|
120
|
+
switch (item.role) {
|
|
121
|
+
case 'system':
|
|
122
|
+
case 'developer':
|
|
123
|
+
system.push(contentText(item.content));
|
|
124
|
+
break;
|
|
125
|
+
case 'assistant':
|
|
126
|
+
pushAssistant(messages, contentParts(item.content));
|
|
127
|
+
break;
|
|
128
|
+
default:
|
|
129
|
+
messages.push({
|
|
130
|
+
role: 'user',
|
|
131
|
+
parts: contentParts(item.content),
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
else if (type.endsWith('_output')) {
|
|
136
|
+
messages.push({
|
|
137
|
+
role: 'tool',
|
|
138
|
+
parts: [
|
|
139
|
+
{
|
|
140
|
+
type: 'tool_result',
|
|
141
|
+
id: asString(item.call_id) ?? '',
|
|
142
|
+
content: outputText(item.output),
|
|
143
|
+
},
|
|
144
|
+
],
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
else {
|
|
148
|
+
pushAssistant(messages, outputParts(item));
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
const tools = asArray(b.tools).map(toTool);
|
|
152
|
+
return toRequest(b, system, messages, tools);
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* There is no finish reason: `status` is `completed`, `incomplete` or
|
|
156
|
+
* `failed`, and an incomplete response says why (`max_output_tokens`). A
|
|
157
|
+
* failed response says what went wrong in `error`.
|
|
158
|
+
*/
|
|
159
|
+
function parseResponse(body) {
|
|
160
|
+
const b = asRecord(body);
|
|
161
|
+
const error = toLlmError(b.error);
|
|
162
|
+
return {
|
|
163
|
+
model: asString(b.model),
|
|
164
|
+
output: {
|
|
165
|
+
role: 'assistant',
|
|
166
|
+
parts: asArray(b.output).flatMap(outputParts),
|
|
167
|
+
},
|
|
168
|
+
stopReason: asString(asRecord(b.incomplete_details).reason) ??
|
|
169
|
+
asString(b.status),
|
|
170
|
+
usage: toUsage(b.usage),
|
|
171
|
+
...(error && { error }),
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
/** An `error` event has `code` and `message` at its top level, or in an `error` object. */
|
|
175
|
+
function errorEvent(event) {
|
|
176
|
+
const fields = isRecord(event.error)
|
|
177
|
+
? event.error
|
|
178
|
+
: { code: event.code, message: event.message };
|
|
179
|
+
return toLlmError(fields) ?? { message: '' };
|
|
180
|
+
}
|
|
181
|
+
/** Delta events, and the field of their item they add to. */
|
|
182
|
+
const DELTAS = {
|
|
183
|
+
'response.output_text.delta': (item, text) => {
|
|
184
|
+
item.content = [{ type: 'output_text', text }];
|
|
185
|
+
},
|
|
186
|
+
'response.refusal.delta': (item, text) => {
|
|
187
|
+
item.content = [{ type: 'refusal', refusal: text }];
|
|
188
|
+
},
|
|
189
|
+
'response.function_call_arguments.delta': (item, text) => {
|
|
190
|
+
item.arguments = text;
|
|
191
|
+
},
|
|
192
|
+
'response.custom_tool_call_input.delta': (item, text) => {
|
|
193
|
+
item.input = text;
|
|
194
|
+
},
|
|
195
|
+
'response.reasoning_summary_text.delta': (item, text) => {
|
|
196
|
+
item.summary = [{ type: 'summary_text', text }];
|
|
197
|
+
},
|
|
198
|
+
'response.reasoning_text.delta': (item, text) => {
|
|
199
|
+
item.content = [{ type: 'reasoning_text', text }];
|
|
200
|
+
},
|
|
201
|
+
};
|
|
202
|
+
const CLOSING_EVENTS = new Set([
|
|
203
|
+
'response.completed',
|
|
204
|
+
'response.incomplete',
|
|
205
|
+
'response.failed',
|
|
206
|
+
]);
|
|
207
|
+
/** Keeps an output item, or adds a delta piece to the ones for its item. */
|
|
208
|
+
function addItemEvent(state, type, event) {
|
|
209
|
+
const index = asNumber(event.output_index);
|
|
210
|
+
if (index === undefined)
|
|
211
|
+
return;
|
|
212
|
+
if (type === 'response.output_item.added' ||
|
|
213
|
+
type === 'response.output_item.done') {
|
|
214
|
+
state.items[index] = { ...asRecord(event.item) };
|
|
215
|
+
if (type === 'response.output_item.done')
|
|
216
|
+
state.finished.add(index);
|
|
217
|
+
}
|
|
218
|
+
else if (Object.hasOwn(DELTAS, type)) {
|
|
219
|
+
const key = `${index} ${type}`;
|
|
220
|
+
const delta = state.deltas.get(key) ?? { index, type, text: '' };
|
|
221
|
+
delta.text += asString(event.delta) ?? '';
|
|
222
|
+
state.deltas.set(key, delta);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
/** The items, with the deltas written into those not finished. */
|
|
226
|
+
function rebuiltItems(state) {
|
|
227
|
+
for (const { index, type, text } of state.deltas.values()) {
|
|
228
|
+
const item = state.items[index];
|
|
229
|
+
if (item && !state.finished.has(index))
|
|
230
|
+
DELTAS[type]?.(item, text);
|
|
231
|
+
}
|
|
232
|
+
return state.items;
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* The closing event (`response.completed`, `.incomplete` or `.failed`)
|
|
236
|
+
* carries the whole response, so that is read. A stream cut short before it
|
|
237
|
+
* is rebuilt from the output items and the deltas of those not finished.
|
|
238
|
+
* An `error` event is kept unless the closing event names its own error.
|
|
239
|
+
*/
|
|
240
|
+
function parseStream(events) {
|
|
241
|
+
if (events.length === 0)
|
|
242
|
+
return {};
|
|
243
|
+
let model;
|
|
244
|
+
let error;
|
|
245
|
+
const state = {
|
|
246
|
+
items: [],
|
|
247
|
+
finished: new Set(),
|
|
248
|
+
deltas: new Map(),
|
|
249
|
+
};
|
|
250
|
+
for (const raw of events) {
|
|
251
|
+
const event = asRecord(raw);
|
|
252
|
+
const type = asString(event.type) ?? '';
|
|
253
|
+
if (CLOSING_EVENTS.has(type)) {
|
|
254
|
+
const response = parseResponse(event.response);
|
|
255
|
+
if (error && !response.error)
|
|
256
|
+
response.error = error;
|
|
257
|
+
return response;
|
|
258
|
+
}
|
|
259
|
+
if (type === 'error') {
|
|
260
|
+
error = errorEvent(event);
|
|
261
|
+
continue;
|
|
262
|
+
}
|
|
263
|
+
model = asString(asRecord(event.response).model) ?? model;
|
|
264
|
+
addItemEvent(state, type, event);
|
|
265
|
+
}
|
|
266
|
+
return {
|
|
267
|
+
model,
|
|
268
|
+
output: {
|
|
269
|
+
role: 'assistant',
|
|
270
|
+
parts: rebuiltItems(state).flatMap(outputParts),
|
|
271
|
+
},
|
|
272
|
+
...(error && { error }),
|
|
273
|
+
};
|
|
274
|
+
}
|
|
275
|
+
export const openaiResponses = {
|
|
276
|
+
parseRequest,
|
|
277
|
+
parseResponse,
|
|
278
|
+
parseStream,
|
|
279
|
+
};
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/** Whether a `content-type` header says the body is Server-Sent Events. */
|
|
2
|
+
export function isEventStream(contentType) {
|
|
3
|
+
return String(contentType ?? '')
|
|
4
|
+
.toLowerCase()
|
|
5
|
+
.startsWith('text/event-stream');
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* Splits a Server-Sent Events body into events, following the WHATWG rules:
|
|
9
|
+
* `data:` lines join with newlines, comments are skipped, one space after the
|
|
10
|
+
* colon is dropped. An event without its closing blank line (a stream cut
|
|
11
|
+
* short) is dropped, as a browser would.
|
|
12
|
+
*/
|
|
13
|
+
export function parseSse(text) {
|
|
14
|
+
const events = [];
|
|
15
|
+
const lines = text.split(/\r\n|\r|\n/);
|
|
16
|
+
// The last piece follows the last line break, so it is empty or a line cut short.
|
|
17
|
+
lines.pop();
|
|
18
|
+
let event = '';
|
|
19
|
+
let data = [];
|
|
20
|
+
for (const line of lines) {
|
|
21
|
+
if (line === '') {
|
|
22
|
+
if (data.length > 0) {
|
|
23
|
+
events.push({
|
|
24
|
+
event: event || 'message',
|
|
25
|
+
data: data.join('\n'),
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
event = '';
|
|
29
|
+
data = [];
|
|
30
|
+
continue;
|
|
31
|
+
}
|
|
32
|
+
const field = splitField(line);
|
|
33
|
+
if (field?.name === 'event')
|
|
34
|
+
event = field.value;
|
|
35
|
+
else if (field?.name === 'data')
|
|
36
|
+
data.push(field.value);
|
|
37
|
+
}
|
|
38
|
+
return events;
|
|
39
|
+
}
|
|
40
|
+
/** A field line as name and value; a comment line is undefined. */
|
|
41
|
+
function splitField(line) {
|
|
42
|
+
if (line.startsWith(':'))
|
|
43
|
+
return undefined;
|
|
44
|
+
const colon = line.indexOf(':');
|
|
45
|
+
if (colon === -1)
|
|
46
|
+
return { name: line, value: '' };
|
|
47
|
+
const value = line.slice(colon + 1);
|
|
48
|
+
return {
|
|
49
|
+
name: line.slice(0, colon),
|
|
50
|
+
value: value.startsWith(' ') ? value.slice(1) : value,
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* The `data` of each event, parsed as JSON. Events whose data is not JSON,
|
|
55
|
+
* such as OpenAI's closing `[DONE]`, are skipped.
|
|
56
|
+
*/
|
|
57
|
+
export function parseSseJson(text) {
|
|
58
|
+
const result = [];
|
|
59
|
+
for (const { data } of parseSse(text)) {
|
|
60
|
+
try {
|
|
61
|
+
result.push(JSON.parse(data));
|
|
62
|
+
}
|
|
63
|
+
catch {
|
|
64
|
+
// Not JSON: nothing to read from it.
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
return result;
|
|
68
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { homedir } from 'node:os';
|
|
3
|
+
import * as path from 'node:path';
|
|
4
|
+
/** Where a running LLMSpy says who it is, for `llmspy stop` (stop.ts). */
|
|
5
|
+
export const PID_FILE = path.join(homedir(), '.llmspy', 'llmspy.pid');
|
|
6
|
+
/**
|
|
7
|
+
* Records this process and the URL it answers on, and removes the record when
|
|
8
|
+
* the process exits. A failure to write is only a warning: the proxy works
|
|
9
|
+
* without it, `llmspy stop` just can't find it.
|
|
10
|
+
*/
|
|
11
|
+
export function writePidFile(url, file = PID_FILE) {
|
|
12
|
+
try {
|
|
13
|
+
mkdirSync(path.dirname(file), { recursive: true });
|
|
14
|
+
const record = { pid: process.pid, url };
|
|
15
|
+
writeFileSync(file, JSON.stringify(record));
|
|
16
|
+
}
|
|
17
|
+
catch (error) {
|
|
18
|
+
console.warn(`Could not write ${file}: ${error.message}`);
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
21
|
+
process.on('exit', () => removePidFile(file));
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* The record in `file`: `'none'` when there is no file, `'invalid'` when it
|
|
25
|
+
* is not a record this module wrote. Any other read error is thrown.
|
|
26
|
+
*/
|
|
27
|
+
export function readPidFile(file = PID_FILE) {
|
|
28
|
+
let text;
|
|
29
|
+
try {
|
|
30
|
+
text = readFileSync(file, 'utf8');
|
|
31
|
+
}
|
|
32
|
+
catch (error) {
|
|
33
|
+
if (error.code === 'ENOENT')
|
|
34
|
+
return 'none';
|
|
35
|
+
throw error;
|
|
36
|
+
}
|
|
37
|
+
try {
|
|
38
|
+
const { pid, url } = JSON.parse(text) ?? {};
|
|
39
|
+
return typeof pid === 'number' && typeof url === 'string'
|
|
40
|
+
? { pid, url }
|
|
41
|
+
: 'invalid';
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
return 'invalid';
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
/** Removes the file only if it is still this process's, so a newer LLMSpy's record is left alone. */
|
|
48
|
+
export function removePidFile(file = PID_FILE) {
|
|
49
|
+
try {
|
|
50
|
+
const record = readPidFile(file);
|
|
51
|
+
if (typeof record === 'object' && record.pid === process.pid)
|
|
52
|
+
rmSync(file);
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
// Not ours to read or remove.
|
|
56
|
+
}
|
|
57
|
+
}
|