@juspay/neurolink 12.12.4 → 12.12.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -2
- package/dist/auth/tokenStore.d.ts +9 -1
- package/dist/auth/tokenStore.js +29 -0
- package/dist/browser/neurolink.min.js +404 -403
- package/dist/cli/commands/proxy.js +65 -80
- package/dist/core/baseProvider.d.ts +10 -0
- package/dist/core/baseProvider.js +12 -2
- package/dist/middleware/builtin/guardrails.js +23 -13
- package/dist/middleware/utils/guardrailsUtils.d.ts +3 -15
- package/dist/middleware/utils/guardrailsUtils.js +14 -7
- package/dist/middleware/wrapLanguageModel.d.ts +3 -4
- package/dist/middleware/wrapLanguageModel.js +3 -4
- package/dist/providers/openaiChatCompletionsBase.js +214 -24
- package/dist/proxy/claudeFormat.d.ts +2 -2
- package/dist/proxy/claudeFormat.js +2 -2
- package/dist/proxy/codexFallback.d.ts +7 -1
- package/dist/proxy/codexFallback.js +181 -0
- package/dist/proxy/rollingProxyServer.js +10 -4
- package/dist/proxy/rollingWorkerProcess.js +8 -2
- package/dist/proxy/rollingWorkerProtocol.d.ts +2 -0
- package/dist/proxy/rollingWorkerProtocol.js +2 -0
- package/dist/proxy/rollingWorkerSupervisor.d.ts +5 -0
- package/dist/proxy/rollingWorkerSupervisor.js +75 -14
- package/dist/server/routes/claudeProxyRoutes.js +236 -61
- package/dist/server/routes/codexProxyRoutes.js +33 -6
- package/dist/types/cli.d.ts +2 -1
- package/dist/types/codex.d.ts +5 -0
- package/dist/types/proxy.d.ts +10 -2
- package/package.json +2 -1
|
@@ -66,6 +66,73 @@ const yieldsSchemaValidObject = (text, schema) => {
|
|
|
66
66
|
const coerced = coerceJsonToSchema(text, schema);
|
|
67
67
|
return coerced !== null && schemaAccepts(schema, coerced.structuredData);
|
|
68
68
|
};
|
|
69
|
+
// Pull one native chunk at a time and forward cancellation to its iterator.
|
|
70
|
+
const chunksToV3Stream = (source, completion, cancel) => {
|
|
71
|
+
const iterator = source[Symbol.asyncIterator]();
|
|
72
|
+
return new ReadableStream({
|
|
73
|
+
async pull(controller) {
|
|
74
|
+
try {
|
|
75
|
+
const next = await iterator.next();
|
|
76
|
+
if (next.done) {
|
|
77
|
+
controller.enqueue(await completion);
|
|
78
|
+
controller.close();
|
|
79
|
+
}
|
|
80
|
+
else if (next.value.reasoning) {
|
|
81
|
+
controller.enqueue({
|
|
82
|
+
type: "reasoning-delta",
|
|
83
|
+
delta: next.value.reasoning,
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
else {
|
|
87
|
+
controller.enqueue({ type: "text-delta", delta: next.value.content });
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
catch (error) {
|
|
91
|
+
controller.error(error);
|
|
92
|
+
}
|
|
93
|
+
},
|
|
94
|
+
async cancel() {
|
|
95
|
+
cancel();
|
|
96
|
+
await iterator.return?.();
|
|
97
|
+
},
|
|
98
|
+
});
|
|
99
|
+
};
|
|
100
|
+
async function* v3StreamToChunks(stream, onFinish) {
|
|
101
|
+
const reader = stream.getReader();
|
|
102
|
+
let done = false;
|
|
103
|
+
try {
|
|
104
|
+
while (true) {
|
|
105
|
+
const next = await reader.read();
|
|
106
|
+
if (next.done) {
|
|
107
|
+
done = true;
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
const part = next.value;
|
|
111
|
+
if (part.type === "text-delta") {
|
|
112
|
+
yield { content: part.delta };
|
|
113
|
+
}
|
|
114
|
+
else if (part.type === "reasoning-delta") {
|
|
115
|
+
yield { content: "", reasoning: part.delta };
|
|
116
|
+
}
|
|
117
|
+
else if (part.type === "finish") {
|
|
118
|
+
onFinish(part);
|
|
119
|
+
}
|
|
120
|
+
else if (part.type === "error") {
|
|
121
|
+
throw part.error;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
finally {
|
|
126
|
+
try {
|
|
127
|
+
if (!done) {
|
|
128
|
+
await reader.cancel();
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
finally {
|
|
132
|
+
reader.releaseLock();
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
69
136
|
export class OpenAIChatCompletionsProvider extends BaseProvider {
|
|
70
137
|
config;
|
|
71
138
|
resolvedModel;
|
|
@@ -946,7 +1013,10 @@ export class OpenAIChatCompletionsProvider extends BaseProvider {
|
|
|
946
1013
|
let wireNameMaps;
|
|
947
1014
|
let openAITools;
|
|
948
1015
|
let openAIToolChoice;
|
|
949
|
-
|
|
1016
|
+
// The prompt is kept in its pre-wire shape. Model middleware transforms
|
|
1017
|
+
// `params.prompt`, and the conversion to the chat-completions wire format
|
|
1018
|
+
// has to happen AFTER that or the transform would be discarded.
|
|
1019
|
+
let promptMessages;
|
|
950
1020
|
try {
|
|
951
1021
|
modelId = await this.resolveModelName();
|
|
952
1022
|
const shouldUseTools = !options.disableTools && this.supportsTools();
|
|
@@ -962,8 +1032,7 @@ export class OpenAIChatCompletionsProvider extends BaseProvider {
|
|
|
962
1032
|
? buildToolsForOpenAI(toolsRecord, wireNameMaps?.toWire)
|
|
963
1033
|
: undefined;
|
|
964
1034
|
openAIToolChoice = mapNeuroLinkToolChoice(resolveToolChoice(options, toolsRecord, shouldUseTools), wireNameMaps?.toWire);
|
|
965
|
-
|
|
966
|
-
conversation = messageBuilderToOpenAI(initialMessages, wireNameMaps?.toWire);
|
|
1035
|
+
promptMessages = (await this.buildMessagesForStream(options));
|
|
967
1036
|
}
|
|
968
1037
|
catch (setupErr) {
|
|
969
1038
|
timeoutController?.cleanup();
|
|
@@ -979,26 +1048,136 @@ export class OpenAIChatCompletionsProvider extends BaseProvider {
|
|
|
979
1048
|
const channel = createStreamChannel();
|
|
980
1049
|
// Per-provider lifecycle hook (e.g. OTel span wrap for LiteLLM).
|
|
981
1050
|
const lifecycle = this.onStreamStart(modelId);
|
|
982
|
-
|
|
983
|
-
|
|
1051
|
+
// Model middleware on the streaming path.
|
|
1052
|
+
//
|
|
1053
|
+
// The base model below is not `buildDelegatingModel()`'s — that one's
|
|
1054
|
+
// `doGenerate` is a single wire call and its `doStream` is a stub. This
|
|
1055
|
+
// one's `doStream` starts the real multi-step stream loop, which is what
|
|
1056
|
+
// "produce the stream for this request" means here. Wrapping it gives the
|
|
1057
|
+
// streaming path the contract the generate path has always had:
|
|
1058
|
+
// `transformParams` can rewrite the prompt before a byte is sent, and
|
|
1059
|
+
// `wrapStream` can observe, filter, or replace the stream outright.
|
|
1060
|
+
//
|
|
1061
|
+
// Honoured on the way back in: `prompt`, `maxOutputTokens`, `temperature`
|
|
1062
|
+
// and `topP`. `tools` is offered read-only — a middleware that rewrites it
|
|
1063
|
+
// gets a WARN rather than a silent drop, because re-deriving the wire tool
|
|
1064
|
+
// list here would diverge from `buildToolsForOpenAI`.
|
|
1065
|
+
const v3Tools = openAITools?.map((t) => ({
|
|
1066
|
+
type: "function",
|
|
1067
|
+
name: t.function.name,
|
|
1068
|
+
description: t.function.description,
|
|
1069
|
+
inputSchema: t.function.parameters,
|
|
1070
|
+
}));
|
|
1071
|
+
const v3Params = {
|
|
1072
|
+
prompt: promptMessages,
|
|
1073
|
+
...(v3Tools ? { tools: v3Tools } : {}),
|
|
1074
|
+
...(options.maxTokens !== undefined
|
|
1075
|
+
? { maxOutputTokens: options.maxTokens }
|
|
1076
|
+
: {}),
|
|
1077
|
+
...(options.temperature !== undefined
|
|
1078
|
+
? { temperature: options.temperature }
|
|
1079
|
+
: {}),
|
|
1080
|
+
...(options.topP !== undefined ? { topP: options.topP } : {}),
|
|
1081
|
+
};
|
|
1082
|
+
let loopPromise;
|
|
1083
|
+
const providerNameForLoop = this.providerName;
|
|
1084
|
+
const streamBaseModel = {
|
|
1085
|
+
specificationVersion: "v3",
|
|
1086
|
+
provider: providerNameForLoop,
|
|
984
1087
|
modelId,
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1088
|
+
supportedUrls: {},
|
|
1089
|
+
doGenerate: async (params) => {
|
|
1090
|
+
const model = await this.getAISDKModel();
|
|
1091
|
+
if (typeof model === "string") {
|
|
1092
|
+
throw new Error("Native model handle required");
|
|
1093
|
+
}
|
|
1094
|
+
return model.doGenerate(params);
|
|
1095
|
+
},
|
|
1096
|
+
doStream: async (params) => {
|
|
1097
|
+
if (params?.tools !== undefined && params.tools !== v3Tools) {
|
|
1098
|
+
logger.warn(`${providerNameForLoop}: middleware rewrote 'tools' on the streaming path; tool rewrites are not applied to the wire request yet — the original tool list was sent.`);
|
|
1099
|
+
}
|
|
1100
|
+
const transformedPrompt = Array.isArray(params?.prompt)
|
|
1101
|
+
? params.prompt
|
|
1102
|
+
: promptMessages;
|
|
1103
|
+
const conversation = messageBuilderToOpenAI(transformedPrompt, wireNameMaps?.toWire);
|
|
1104
|
+
const sampled = {
|
|
1105
|
+
...options,
|
|
1106
|
+
...(typeof params?.maxOutputTokens === "number"
|
|
1107
|
+
? { maxTokens: params.maxOutputTokens }
|
|
1108
|
+
: {}),
|
|
1109
|
+
...(typeof params?.temperature === "number"
|
|
1110
|
+
? { temperature: params.temperature }
|
|
1111
|
+
: {}),
|
|
1112
|
+
...(typeof params?.topP === "number" ? { topP: params.topP } : {}),
|
|
1113
|
+
};
|
|
1114
|
+
loopPromise = this.runStreamLoop({
|
|
1115
|
+
maxSteps,
|
|
1116
|
+
modelId,
|
|
1117
|
+
url,
|
|
1118
|
+
fetchImpl,
|
|
1119
|
+
abortSignal,
|
|
1120
|
+
options: sampled,
|
|
1121
|
+
conversation,
|
|
1122
|
+
openAITools,
|
|
1123
|
+
openAIToolChoice,
|
|
1124
|
+
toolsRecord,
|
|
1125
|
+
toolNameFromWire: wireNameMaps?.fromWire,
|
|
1126
|
+
emitter,
|
|
1127
|
+
toolsUsed,
|
|
1128
|
+
toolExecutionSummaries,
|
|
1129
|
+
pushChunk: channel.push,
|
|
1130
|
+
closeChannel: channel.close,
|
|
1131
|
+
resolveUsage,
|
|
1132
|
+
resolveFinish,
|
|
1133
|
+
});
|
|
1134
|
+
const completion = loopPromise.then(() => Promise.all([usagePromise, finishPromise]).then(([usage, reason]) => ({
|
|
1135
|
+
type: "finish",
|
|
1136
|
+
finishReason: { unified: reason },
|
|
1137
|
+
usage: {
|
|
1138
|
+
inputTokens: {
|
|
1139
|
+
total: usage.promptTokens,
|
|
1140
|
+
cacheRead: usage.cacheReadTokens,
|
|
1141
|
+
},
|
|
1142
|
+
outputTokens: { total: usage.completionTokens },
|
|
1143
|
+
},
|
|
1144
|
+
})));
|
|
1145
|
+
// The producer can reject before the consumer pulls its terminal event.
|
|
1146
|
+
void completion.catch(() => undefined);
|
|
1147
|
+
return {
|
|
1148
|
+
stream: chunksToV3Stream(channel.iterable, completion, () => consumerAbortController.abort()),
|
|
1149
|
+
};
|
|
1150
|
+
},
|
|
1151
|
+
};
|
|
1152
|
+
// A middleware chain that blocks (guardrails' precall path) returns its own
|
|
1153
|
+
// stream without calling `doStream`, so the loop may never start. Every
|
|
1154
|
+
// later reader of `loopPromise` has to tolerate that.
|
|
1155
|
+
let chunkSource;
|
|
1156
|
+
try {
|
|
1157
|
+
const wrappedStreamModel = await this.applyMiddlewareToModel(streamBaseModel, options);
|
|
1158
|
+
if (typeof wrappedStreamModel === "string") {
|
|
1159
|
+
throw new Error("Native stream model handle required");
|
|
1160
|
+
}
|
|
1161
|
+
const { stream } = await wrappedStreamModel.doStream(v3Params);
|
|
1162
|
+
chunkSource = v3StreamToChunks(stream, (part) => {
|
|
1163
|
+
if (!loopPromise) {
|
|
1164
|
+
const input = part.usage.inputTokens.total ?? 0;
|
|
1165
|
+
const output = part.usage.outputTokens.total ?? 0;
|
|
1166
|
+
resolveUsage({
|
|
1167
|
+
promptTokens: input,
|
|
1168
|
+
completionTokens: output,
|
|
1169
|
+
totalTokens: input + output,
|
|
1170
|
+
});
|
|
1171
|
+
resolveFinish(part.finishReason.unified);
|
|
1172
|
+
}
|
|
1173
|
+
});
|
|
1174
|
+
}
|
|
1175
|
+
catch (error) {
|
|
1176
|
+
consumerAbortController.abort();
|
|
1177
|
+
channel.close();
|
|
1178
|
+
timeoutController?.cleanup();
|
|
1179
|
+
throw error;
|
|
1180
|
+
}
|
|
1002
1181
|
// Closure-scoped capture: the runStreamLoop's catch block stashes the
|
|
1003
1182
|
// underlying provider error here so we can pass it through to
|
|
1004
1183
|
// buildNoOutputSentinel for richer telemetry (matches the pattern in
|
|
@@ -1025,7 +1204,7 @@ export class OpenAIChatCompletionsProvider extends BaseProvider {
|
|
|
1025
1204
|
const transformedStream = async function* () {
|
|
1026
1205
|
let contentYielded = 0;
|
|
1027
1206
|
try {
|
|
1028
|
-
for await (const chunk of
|
|
1207
|
+
for await (const chunk of chunkSource) {
|
|
1029
1208
|
if ("content" in chunk &&
|
|
1030
1209
|
typeof chunk.content === "string" &&
|
|
1031
1210
|
chunk.content.length > 0) {
|
|
@@ -1034,6 +1213,8 @@ export class OpenAIChatCompletionsProvider extends BaseProvider {
|
|
|
1034
1213
|
yield chunk;
|
|
1035
1214
|
}
|
|
1036
1215
|
// Surface any error that the loop threw after we drained the channel.
|
|
1216
|
+
// `loopPromise` is undefined when a middleware blocked the request
|
|
1217
|
+
// before `doStream` ran, in which case there is no loop to surface.
|
|
1037
1218
|
await loopPromise;
|
|
1038
1219
|
// No-output path: stream completed normally but yielded zero text.
|
|
1039
1220
|
// Build an enriched sentinel + stamp the active OTel span so
|
|
@@ -1062,6 +1243,15 @@ export class OpenAIChatCompletionsProvider extends BaseProvider {
|
|
|
1062
1243
|
throw streamError;
|
|
1063
1244
|
}
|
|
1064
1245
|
finally {
|
|
1246
|
+
if (!loopPromise) {
|
|
1247
|
+
resolveUsage({
|
|
1248
|
+
promptTokens: 0,
|
|
1249
|
+
completionTokens: 0,
|
|
1250
|
+
totalTokens: 0,
|
|
1251
|
+
});
|
|
1252
|
+
resolveFinish("stop");
|
|
1253
|
+
}
|
|
1254
|
+
timeoutController?.cleanup();
|
|
1065
1255
|
if (!consumerAbortController.signal.aborted) {
|
|
1066
1256
|
consumerAbortController.abort();
|
|
1067
1257
|
}
|
|
@@ -1101,7 +1291,7 @@ export class OpenAIChatCompletionsProvider extends BaseProvider {
|
|
|
1101
1291
|
}))),
|
|
1102
1292
|
});
|
|
1103
1293
|
loopPromise
|
|
1104
|
-
|
|
1294
|
+
?.finally(() => timeoutController?.cleanup())
|
|
1105
1295
|
.catch((error) => {
|
|
1106
1296
|
captureProviderError(error);
|
|
1107
1297
|
});
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ClaudeErrorResponse, ClaudeRequest, ClaudeResponse, ContentBlockType, InternalResult, ParsedClaudeRequest, StreamLifecycleState } from "../types/index.js";
|
|
1
|
+
import type { ClaudeErrorResponse, ClaudeRequest, ClaudeResponse, ContentBlockType, InternalResult, ParsedClaudeRequest, SSEMessageDelta, StreamLifecycleState } from "../types/index.js";
|
|
2
2
|
/** Generate a unique message id in the Claude format. */
|
|
3
3
|
export declare function generateMessageId(): string;
|
|
4
4
|
/** Generate a Claude-format tool use ID (`toolu_` + 24 random chars). */
|
|
@@ -121,7 +121,7 @@ export declare class ClaudeStreamSerializer {
|
|
|
121
121
|
/**
|
|
122
122
|
* Finalize the stream: content_block_stop, message_delta, message_stop.
|
|
123
123
|
*/
|
|
124
|
-
finish(outputTokens?: number, finishReason?: string): Generator<string>;
|
|
124
|
+
finish(outputTokens?: number, finishReason?: string, finalUsage?: Partial<SSEMessageDelta["usage"]>): Generator<string>;
|
|
125
125
|
/**
|
|
126
126
|
* Emit an error event. Transitions to terminal ERROR state.
|
|
127
127
|
*/
|
|
@@ -582,7 +582,7 @@ export class ClaudeStreamSerializer {
|
|
|
582
582
|
/**
|
|
583
583
|
* Finalize the stream: content_block_stop, message_delta, message_stop.
|
|
584
584
|
*/
|
|
585
|
-
*finish(outputTokens, finishReason) {
|
|
585
|
+
*finish(outputTokens, finishReason, finalUsage) {
|
|
586
586
|
// If we never started (empty response), start first
|
|
587
587
|
if (this.state === "idle") {
|
|
588
588
|
yield* this.ensureMessageStarted();
|
|
@@ -603,7 +603,7 @@ export class ClaudeStreamSerializer {
|
|
|
603
603
|
stop_reason: mapStopReason(resolvedFinishReason),
|
|
604
604
|
stop_sequence: null,
|
|
605
605
|
},
|
|
606
|
-
usage: { output_tokens: this.outputTokens },
|
|
606
|
+
usage: { ...finalUsage, output_tokens: this.outputTokens },
|
|
607
607
|
};
|
|
608
608
|
yield formatSSE("message_delta", messageDelta);
|
|
609
609
|
// message_stop
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* in the native Codex proxy handler so fallback traffic follows the same pool
|
|
7
7
|
* rules as a native Codex request.
|
|
8
8
|
*/
|
|
9
|
-
import type { ClaudeRequest, CodexFallbackResult, CodexReasoningEffort, CodexResponsesRequest } from "../types/index.js";
|
|
9
|
+
import type { ClaudeRequest, CodexFallbackResult, CodexFallbackStream, CodexReasoningEffort, CodexResponsesRequest } from "../types/index.js";
|
|
10
10
|
export declare class CodexFallbackResponseError extends Error {
|
|
11
11
|
readonly status: number;
|
|
12
12
|
readonly responseBody: string;
|
|
@@ -24,3 +24,9 @@ export declare function convertClaudeRequestToCodex(body: ClaudeRequest, model:
|
|
|
24
24
|
export declare function parseCodexFallbackSSE(sse: string): CodexFallbackResult;
|
|
25
25
|
/** Consume and validate a native Codex response before producing Claude output. */
|
|
26
26
|
export declare function consumeCodexFallbackResponse(response: Response): Promise<CodexFallbackResult>;
|
|
27
|
+
/**
|
|
28
|
+
* Translate a Codex stream as events arrive. A completed tool call is emitted
|
|
29
|
+
* once its arguments validate; text does not wait for response.completed.
|
|
30
|
+
* The caller owns error framing and must never retry after emitting output.
|
|
31
|
+
*/
|
|
32
|
+
export declare function createCodexFallbackStream(response: Response, model: string): Promise<CodexFallbackStream>;
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
* in the native Codex proxy handler so fallback traffic follows the same pool
|
|
7
7
|
* rules as a native Codex request.
|
|
8
8
|
*/
|
|
9
|
+
import { ClaudeStreamSerializer, generateToolUseId } from "./claudeFormat.js";
|
|
9
10
|
import { extractCodexUsage } from "./codexUsage.js";
|
|
10
11
|
export class CodexFallbackResponseError extends Error {
|
|
11
12
|
status;
|
|
@@ -372,3 +373,183 @@ export async function consumeCodexFallbackResponse(response) {
|
|
|
372
373
|
}
|
|
373
374
|
return parseCodexFallbackSSE(await response.text());
|
|
374
375
|
}
|
|
376
|
+
/**
|
|
377
|
+
* Translate a Codex stream as events arrive. A completed tool call is emitted
|
|
378
|
+
* once its arguments validate; text does not wait for response.completed.
|
|
379
|
+
* The caller owns error framing and must never retry after emitting output.
|
|
380
|
+
*/
|
|
381
|
+
export async function createCodexFallbackStream(response, model) {
|
|
382
|
+
if (!response.ok) {
|
|
383
|
+
throw new CodexFallbackResponseError(response.status, await response.text().catch(() => ""));
|
|
384
|
+
}
|
|
385
|
+
if (!response.body ||
|
|
386
|
+
!(response.headers.get("content-type") ?? "")
|
|
387
|
+
.toLowerCase()
|
|
388
|
+
.includes("text/event-stream")) {
|
|
389
|
+
await response.body?.cancel().catch(() => undefined);
|
|
390
|
+
throw new Error("Codex fallback returned a non-SSE or empty response");
|
|
391
|
+
}
|
|
392
|
+
const reader = response.body.getReader();
|
|
393
|
+
let cancellation;
|
|
394
|
+
const cancel = (reason) => {
|
|
395
|
+
cancellation ??= reader
|
|
396
|
+
.cancel(reason)
|
|
397
|
+
.catch(() => undefined)
|
|
398
|
+
.finally(() => reader.releaseLock());
|
|
399
|
+
return cancellation;
|
|
400
|
+
};
|
|
401
|
+
async function* frames() {
|
|
402
|
+
const serializer = new ClaudeStreamSerializer(model);
|
|
403
|
+
const decoder = new TextDecoder();
|
|
404
|
+
const toolCalls = new Map();
|
|
405
|
+
const emittedTools = new Set();
|
|
406
|
+
const emittedTextItems = new Set();
|
|
407
|
+
const textParts = [];
|
|
408
|
+
let textLength = 0;
|
|
409
|
+
let toolChars = 0;
|
|
410
|
+
let carry = "";
|
|
411
|
+
let searchFrom = 0;
|
|
412
|
+
let completed = false;
|
|
413
|
+
let usage;
|
|
414
|
+
const maxChars = 16 * 1024 * 1024;
|
|
415
|
+
function* text(value, index) {
|
|
416
|
+
if (!value) {
|
|
417
|
+
return;
|
|
418
|
+
}
|
|
419
|
+
textLength += value.length;
|
|
420
|
+
if (textLength > maxChars) {
|
|
421
|
+
throw new Error("Codex fallback output exceeded the stream limit");
|
|
422
|
+
}
|
|
423
|
+
textParts.push(value);
|
|
424
|
+
emittedTextItems.add(index);
|
|
425
|
+
yield* serializer.pushDelta(value);
|
|
426
|
+
}
|
|
427
|
+
function* item(value, index) {
|
|
428
|
+
if (!isRecord(value)) {
|
|
429
|
+
throw new Error("Codex fallback output item is malformed");
|
|
430
|
+
}
|
|
431
|
+
if (value.type === "function_call") {
|
|
432
|
+
const id = asNonEmptyString(value.call_id);
|
|
433
|
+
if (!id || !emittedTools.has(id)) {
|
|
434
|
+
toolChars += JSON.stringify(value).length;
|
|
435
|
+
if (toolChars > maxChars || toolCalls.size >= 4096) {
|
|
436
|
+
throw new Error("Codex fallback tools exceeded the stream limit");
|
|
437
|
+
}
|
|
438
|
+
addFunctionCall(value, toolCalls);
|
|
439
|
+
const call = id ? toolCalls.get(id) : undefined;
|
|
440
|
+
if (id && call) {
|
|
441
|
+
emittedTools.add(id);
|
|
442
|
+
yield* serializer.pushToolUse(generateToolUseId(), call.toolName, call.args);
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
if (!emittedTextItems.has(index)) {
|
|
447
|
+
yield* text(outputTextFromItem(value), index);
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
function* event(frame) {
|
|
451
|
+
for (const { event: eventName, payload } of parseSSEPayloads(frame)) {
|
|
452
|
+
const type = asNonEmptyString(payload.type) ?? eventName;
|
|
453
|
+
if (!type) {
|
|
454
|
+
throw new Error("Codex fallback stream event is missing a type");
|
|
455
|
+
}
|
|
456
|
+
if (completed) {
|
|
457
|
+
throw new Error("Codex fallback stream emitted events after completion");
|
|
458
|
+
}
|
|
459
|
+
if (type === "error" ||
|
|
460
|
+
type === "response.failed" ||
|
|
461
|
+
type === "response.incomplete") {
|
|
462
|
+
throw new Error(`Codex fallback stream terminated with ${type}`);
|
|
463
|
+
}
|
|
464
|
+
const index = typeof payload.output_index === "number" ? payload.output_index : 0;
|
|
465
|
+
if (type === "response.output_text.delta") {
|
|
466
|
+
if (typeof payload.delta !== "string") {
|
|
467
|
+
throw new Error("Codex fallback text delta is malformed");
|
|
468
|
+
}
|
|
469
|
+
yield* text(payload.delta, index);
|
|
470
|
+
}
|
|
471
|
+
else if (type === "response.output_item.done") {
|
|
472
|
+
if (!isRecord(payload.item)) {
|
|
473
|
+
throw new Error("Codex fallback output item is malformed");
|
|
474
|
+
}
|
|
475
|
+
yield* item(payload.item, index);
|
|
476
|
+
}
|
|
477
|
+
else if (type === "response.completed") {
|
|
478
|
+
if (responseStatus(payload) !== "completed") {
|
|
479
|
+
throw new Error("Codex fallback response did not complete");
|
|
480
|
+
}
|
|
481
|
+
completed = true;
|
|
482
|
+
const parsedUsage = extractCodexUsage(payload);
|
|
483
|
+
if (parsedUsage) {
|
|
484
|
+
usage = {
|
|
485
|
+
input: parsedUsage.inputTokens,
|
|
486
|
+
output: parsedUsage.outputTokens,
|
|
487
|
+
total: parsedUsage.inputTokens + parsedUsage.outputTokens,
|
|
488
|
+
cacheReadTokens: parsedUsage.cacheReadTokens,
|
|
489
|
+
cacheCreationTokens: parsedUsage.cacheCreationTokens,
|
|
490
|
+
};
|
|
491
|
+
}
|
|
492
|
+
const responseBody = payload.response;
|
|
493
|
+
if (isRecord(responseBody) && Array.isArray(responseBody.output)) {
|
|
494
|
+
for (const [i, output] of responseBody.output.entries()) {
|
|
495
|
+
yield* item(output, i);
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
try {
|
|
502
|
+
yield* serializer.start();
|
|
503
|
+
while (true) {
|
|
504
|
+
const chunk = await reader.read();
|
|
505
|
+
carry += decoder.decode(chunk.value, { stream: !chunk.done });
|
|
506
|
+
// Search only new bytes plus the boundary overlap. Long tool payloads
|
|
507
|
+
// split over many chunks must not rescan their accumulated prefix.
|
|
508
|
+
const boundary = /\r?\n\r?\n/g;
|
|
509
|
+
boundary.lastIndex = searchFrom;
|
|
510
|
+
let match;
|
|
511
|
+
while ((match = boundary.exec(carry)) !== null) {
|
|
512
|
+
if (match.index > maxChars) {
|
|
513
|
+
throw new Error("Codex fallback event exceeded the stream limit");
|
|
514
|
+
}
|
|
515
|
+
yield* event(carry.slice(0, match.index));
|
|
516
|
+
carry = carry.slice(match.index + match[0].length);
|
|
517
|
+
boundary.lastIndex = 0;
|
|
518
|
+
}
|
|
519
|
+
if (carry.length > maxChars) {
|
|
520
|
+
throw new Error("Codex fallback event exceeded the stream limit");
|
|
521
|
+
}
|
|
522
|
+
searchFrom = Math.max(0, carry.length - 3);
|
|
523
|
+
if (chunk.done) {
|
|
524
|
+
break;
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
if (carry.trim()) {
|
|
528
|
+
yield* event(carry);
|
|
529
|
+
}
|
|
530
|
+
if (!completed) {
|
|
531
|
+
throw new Error("Codex fallback stream ended before response.completed");
|
|
532
|
+
}
|
|
533
|
+
if (textLength === 0 && toolCalls.size === 0) {
|
|
534
|
+
throw new Error("Codex fallback returned no content or tool calls");
|
|
535
|
+
}
|
|
536
|
+
const finishReason = toolCalls.size > 0 ? "tool_use" : "end_turn";
|
|
537
|
+
yield* serializer.finish(usage?.output, finishReason, {
|
|
538
|
+
input_tokens: usage?.input,
|
|
539
|
+
cache_read_input_tokens: usage?.cacheReadTokens,
|
|
540
|
+
cache_creation_input_tokens: usage?.cacheCreationTokens,
|
|
541
|
+
});
|
|
542
|
+
return {
|
|
543
|
+
text: textParts.join(""),
|
|
544
|
+
toolCalls: [...toolCalls.values()],
|
|
545
|
+
finishReason,
|
|
546
|
+
...(usage ? { usage } : {}),
|
|
547
|
+
};
|
|
548
|
+
}
|
|
549
|
+
finally {
|
|
550
|
+
await cancel();
|
|
551
|
+
reader.releaseLock();
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
return { frames: frames(), cancel };
|
|
555
|
+
}
|
|
@@ -13,6 +13,7 @@ export async function startRollingProxyServer(options) {
|
|
|
13
13
|
let requestedReplacementTimer;
|
|
14
14
|
let requestedReplacementSchedule = 0;
|
|
15
15
|
let requestedReplacementPending = false;
|
|
16
|
+
let requestedReplacementReason = "environment";
|
|
16
17
|
let replacementQueueTail = null;
|
|
17
18
|
const recoveryDelayMs = Math.max(1, options.recoveryDelayMs ?? DEFAULT_RECOVERY_DELAY_MS);
|
|
18
19
|
const maxRecoveryDelayMs = Math.max(recoveryDelayMs, options.maxRecoveryDelayMs ?? DEFAULT_MAX_RECOVERY_DELAY_MS);
|
|
@@ -70,11 +71,14 @@ export async function startRollingProxyServer(options) {
|
|
|
70
71
|
scheduleRecovery();
|
|
71
72
|
}
|
|
72
73
|
};
|
|
73
|
-
function scheduleRequestedReplacement() {
|
|
74
|
+
function scheduleRequestedReplacement(request) {
|
|
74
75
|
if (closing) {
|
|
75
76
|
return;
|
|
76
77
|
}
|
|
77
78
|
requestedReplacementPending = true;
|
|
79
|
+
if (request) {
|
|
80
|
+
requestedReplacementReason = request.reason;
|
|
81
|
+
}
|
|
78
82
|
if (requestedReplacementTimer || replacementQueueTail) {
|
|
79
83
|
return;
|
|
80
84
|
}
|
|
@@ -89,15 +93,16 @@ export async function startRollingProxyServer(options) {
|
|
|
89
93
|
}
|
|
90
94
|
requestedReplacementPending = false;
|
|
91
95
|
const replacementVersion = desiredVersion;
|
|
96
|
+
const replacementReason = requestedReplacementReason;
|
|
92
97
|
void queueReplacement(async () => {
|
|
93
98
|
if (closing || !supervisor.snapshot().active) {
|
|
94
99
|
return;
|
|
95
100
|
}
|
|
96
|
-
options.log?.(`[proxy-supervisor] preparing same-version worker replacement version=${replacementVersion} reason
|
|
101
|
+
options.log?.(`[proxy-supervisor] preparing same-version worker replacement version=${replacementVersion} reason=${replacementReason}`);
|
|
97
102
|
await supervisor.replace(replacementVersion);
|
|
98
|
-
options.log?.(`[proxy-supervisor] same-version worker replacement complete version=${replacementVersion} reason
|
|
103
|
+
options.log?.(`[proxy-supervisor] same-version worker replacement complete version=${replacementVersion} reason=${replacementReason}`);
|
|
99
104
|
}).catch((error) => {
|
|
100
|
-
options.log?.(`[proxy-supervisor] same-version worker replacement failed version=${replacementVersion} reason
|
|
105
|
+
options.log?.(`[proxy-supervisor] same-version worker replacement failed version=${replacementVersion} reason=${replacementReason}: ${error instanceof Error ? error.message : String(error)}`);
|
|
101
106
|
});
|
|
102
107
|
}, 50);
|
|
103
108
|
requestedReplacementTimer.unref?.();
|
|
@@ -106,6 +111,7 @@ export async function startRollingProxyServer(options) {
|
|
|
106
111
|
spawnWorker: options.spawnWorker,
|
|
107
112
|
readyTimeoutMs: options.readyTimeoutMs,
|
|
108
113
|
socketQueueLimit: options.socketQueueLimit,
|
|
114
|
+
maxPendingTransfers: options.maxPendingTransfers,
|
|
109
115
|
socketQueueTimeoutMs: options.socketQueueTimeoutMs,
|
|
110
116
|
shutdownTimeoutMs: options.shutdownTimeoutMs,
|
|
111
117
|
onStateChange: stateChanged,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
2
|
import { ErrorFactory } from "../utils/errorHandling.js";
|
|
3
|
-
import { isProxyWorkerStatusMessage, PROXY_SOCKET_WORKER_ENV, } from "./rollingWorkerProtocol.js";
|
|
3
|
+
import { isProxyWorkerStatusMessage, PROXY_SOCKET_WORKER_ENV, PROXY_SOCKET_OFFER_TIMEOUT, } from "./rollingWorkerProtocol.js";
|
|
4
4
|
export function spawnProxySocketWorker(options) {
|
|
5
5
|
const socketAckTimeoutMs = Math.max(1, options.socketAckTimeoutMs ?? 30_000);
|
|
6
6
|
let nextSocketId = 0;
|
|
@@ -134,7 +134,13 @@ export function spawnProxySocketWorker(options) {
|
|
|
134
134
|
}
|
|
135
135
|
const socketId = `${generation}:${++nextSocketId}`;
|
|
136
136
|
const timeout = setTimeout(() => {
|
|
137
|
-
|
|
137
|
+
const error = new Error(`proxy worker ${childPid} did not accept socket within ${socketAckTimeoutMs}ms`);
|
|
138
|
+
if (!pendingSockets.get(socketId)?.accepted) {
|
|
139
|
+
// No commit was sent. The cancel message settles this offer without
|
|
140
|
+
// terminating unrelated requests already owned by the worker.
|
|
141
|
+
error.code = PROXY_SOCKET_OFFER_TIMEOUT;
|
|
142
|
+
}
|
|
143
|
+
settleSocket(socketId, error);
|
|
138
144
|
}, socketAckTimeoutMs);
|
|
139
145
|
timeout.unref?.();
|
|
140
146
|
pendingSockets.set(socketId, {
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import type { ProxyWorkerControlMessage, ProxyWorkerStatusMessage } from "../types/index.js";
|
|
2
2
|
export declare const PROXY_SOCKET_WORKER_ENV = "NEUROLINK_PROXY_SOCKET_WORKER";
|
|
3
|
+
/** The worker has not been sent a commit and cannot have served this socket. */
|
|
4
|
+
export declare const PROXY_SOCKET_OFFER_TIMEOUT = "PROXY_SOCKET_OFFER_TIMEOUT";
|
|
3
5
|
export declare const PROXY_ROLLING_SUPERVISOR_ENV = "NEUROLINK_PROXY_ROLLING_SUPERVISOR";
|
|
4
6
|
export declare function isProxyWorkerControlMessage(value: unknown): value is ProxyWorkerControlMessage;
|
|
5
7
|
export declare function isProxyWorkerStatusMessage(value: unknown): value is ProxyWorkerStatusMessage;
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
export const PROXY_SOCKET_WORKER_ENV = "NEUROLINK_PROXY_SOCKET_WORKER";
|
|
2
|
+
/** The worker has not been sent a commit and cannot have served this socket. */
|
|
3
|
+
export const PROXY_SOCKET_OFFER_TIMEOUT = "PROXY_SOCKET_OFFER_TIMEOUT";
|
|
2
4
|
export const PROXY_ROLLING_SUPERVISOR_ENV = "NEUROLINK_PROXY_ROLLING_SUPERVISOR";
|
|
3
5
|
export function isProxyWorkerControlMessage(value) {
|
|
4
6
|
if (!value || typeof value !== "object") {
|
|
@@ -11,6 +11,10 @@ export declare class RollingWorkerSupervisor {
|
|
|
11
11
|
private candidate;
|
|
12
12
|
private readonly draining;
|
|
13
13
|
private readonly queuedSockets;
|
|
14
|
+
private flushingSockets;
|
|
15
|
+
private consecutiveOfferTimeouts;
|
|
16
|
+
private lastStallReplacementAt;
|
|
17
|
+
private transferStateTimer;
|
|
14
18
|
private replacement;
|
|
15
19
|
private rejectedSockets;
|
|
16
20
|
private failedTransfers;
|
|
@@ -40,5 +44,6 @@ export declare class RollingWorkerSupervisor {
|
|
|
40
44
|
private extractLifecycleFailureDetails;
|
|
41
45
|
private recordFailure;
|
|
42
46
|
private recordEvent;
|
|
47
|
+
private scheduleTransferState;
|
|
43
48
|
private publishState;
|
|
44
49
|
}
|