@qvac/sdk 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +337 -0
- package/README.md +58 -294
- package/dist/client/api/cancel.d.ts +9 -0
- package/dist/client/api/cancel.js +9 -0
- package/dist/client/api/heartbeat.d.ts +30 -0
- package/dist/client/api/heartbeat.js +38 -0
- package/dist/client/api/index.d.ts +1 -1
- package/dist/client/api/index.js +1 -1
- package/dist/examples/embed-p2p.d.ts +2 -0
- package/dist/examples/llamacpp-p2p.d.ts +2 -0
- package/dist/examples/seed-p2p.d.ts +2 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models/registry/models.d.ts +64 -64
- package/dist/models/registry/models.js +108 -108
- package/dist/schemas/cancel.d.ts +12 -0
- package/dist/schemas/cancel.js +2 -0
- package/dist/schemas/common.d.ts +39 -2
- package/dist/schemas/common.js +4 -3
- package/dist/schemas/delegate.d.ts +8 -0
- package/dist/schemas/delegate.js +5 -2
- package/dist/schemas/heartbeat.d.ts +17 -0
- package/dist/schemas/heartbeat.js +10 -0
- package/dist/schemas/index.d.ts +1 -1
- package/dist/schemas/index.js +1 -1
- package/dist/schemas/load-model.d.ts +113 -0
- package/dist/server/bare/delegate-rpc-client.js +42 -11
- package/dist/server/bare/plugins/nmtcpp-translation/plugin.js +1 -4
- package/dist/server/bare/registry/model-registry.d.ts +2 -0
- package/dist/server/bare/registry/model-registry.js +2 -1
- package/dist/server/rpc/delegate-transport.js +65 -44
- package/dist/server/rpc/delegate-utils.d.ts +2 -0
- package/dist/server/rpc/delegate-utils.js +5 -0
- package/dist/server/rpc/handler-registry.js +26 -3
- package/dist/server/rpc/handler-utils.js +43 -1
- package/dist/server/rpc/handlers/cancel-delegated.d.ts +4 -0
- package/dist/server/rpc/handlers/cancel-delegated.js +71 -0
- package/dist/server/rpc/handlers/completion-stream-delegated.js +2 -2
- package/dist/server/rpc/handlers/heartbeat-delegated.d.ts +4 -0
- package/dist/server/rpc/handlers/heartbeat-delegated.js +32 -0
- package/dist/server/rpc/handlers/heartbeat.d.ts +3 -0
- package/dist/server/rpc/handlers/heartbeat.js +3 -0
- package/dist/server/rpc/handlers/index.d.ts +2 -2
- package/dist/server/rpc/handlers/index.js +2 -2
- package/dist/server/rpc/handlers/load-model/registry.js +25 -41
- package/dist/server/rpc/handlers/load-model-delegated.js +6 -4
- package/dist/server/rpc/handlers/unload-model-delegated.js +3 -3
- package/package.json +10 -3
- package/dist/client/api/ping.d.ts +0 -11
- package/dist/client/api/ping.js +0 -19
- package/dist/examples/embed-hyperdrive.d.ts +0 -2
- package/dist/examples/llamacpp-hyperdrive.d.ts +0 -2
- package/dist/examples/seed-hyperdrive.d.ts +0 -2
- package/dist/schemas/ping.d.ts +0 -11
- package/dist/schemas/ping.js +0 -8
- package/dist/server/rpc/handlers/ping.d.ts +0 -3
- package/dist/server/rpc/handlers/ping.js +0 -3
- /package/dist/examples/{embed-hyperdrive.js → embed-p2p.js} +0 -0
- /package/dist/examples/{llamacpp-hyperdrive.js → llamacpp-p2p.js} +0 -0
- /package/dist/examples/{seed-hyperdrive.js → seed-p2p.js} +0 -0
|
@@ -12,6 +12,26 @@ const activeRPCs = new Map();
|
|
|
12
12
|
const activeConnections = new Map();
|
|
13
13
|
// Track whether the global connection handler has been registered
|
|
14
14
|
let connectionHandlerRegistered = false;
|
|
15
|
+
const HEALTH_CHECK_TIMEOUT_MS = 1500;
|
|
16
|
+
import { getNextCommandId } from "../rpc/delegate-utils.js";
|
|
17
|
+
function isHeartbeatResponse(payload) {
|
|
18
|
+
return (typeof payload === "object" &&
|
|
19
|
+
payload !== null &&
|
|
20
|
+
payload["type"] === "heartbeat");
|
|
21
|
+
}
|
|
22
|
+
async function isRPCConnectionHealthy(rpc, timeout = HEALTH_CHECK_TIMEOUT_MS) {
|
|
23
|
+
try {
|
|
24
|
+
const req = rpc.request(getNextCommandId());
|
|
25
|
+
req.send(JSON.stringify({ type: "heartbeat" }), "utf-8");
|
|
26
|
+
const response = await withTimeout(req.reply("utf-8"), timeout);
|
|
27
|
+
const payload = JSON.parse(response?.toString() || "{}");
|
|
28
|
+
return isHeartbeatResponse(payload);
|
|
29
|
+
}
|
|
30
|
+
catch (error) {
|
|
31
|
+
logger.debug("RPC health check failed", { error });
|
|
32
|
+
return false;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
15
35
|
/**
|
|
16
36
|
* Register the swarm "connection" handler once (not per getRPC call).
|
|
17
37
|
* This fixes the event listener leak where each getRPC() call added
|
|
@@ -68,11 +88,28 @@ async function closeConnection(publicKey) {
|
|
|
68
88
|
}
|
|
69
89
|
}
|
|
70
90
|
// Establish RPC connection to a peer if not already connected
|
|
71
|
-
async function ensureRPCConnection(topic, publicKey, timeout) {
|
|
91
|
+
async function ensureRPCConnection(topic, publicKey, timeout, healthCheckTimeout) {
|
|
92
|
+
const healthCheckCap = healthCheckTimeout ?? HEALTH_CHECK_TIMEOUT_MS;
|
|
93
|
+
const operationStart = nowMs();
|
|
94
|
+
const getRemainingTimeout = () => {
|
|
95
|
+
if (timeout === undefined) {
|
|
96
|
+
return undefined;
|
|
97
|
+
}
|
|
98
|
+
return Math.max(timeout - (nowMs() - operationStart), 0);
|
|
99
|
+
};
|
|
72
100
|
// Check if we already have an RPC instance for this peer
|
|
73
101
|
const existingRpc = activeRPCs.get(publicKey);
|
|
74
102
|
if (existingRpc) {
|
|
75
|
-
|
|
103
|
+
const remainingTimeout = getRemainingTimeout();
|
|
104
|
+
const probeTimeout = remainingTimeout === undefined
|
|
105
|
+
? healthCheckCap
|
|
106
|
+
: Math.min(remainingTimeout / 2, healthCheckCap);
|
|
107
|
+
const isHealthy = await isRPCConnectionHealthy(existingRpc, probeTimeout);
|
|
108
|
+
if (isHealthy) {
|
|
109
|
+
return existingRpc;
|
|
110
|
+
}
|
|
111
|
+
logger.info(`🧹 Cached RPC failed health check for peer ${publicKey}, reconnecting`);
|
|
112
|
+
cleanupStaleConnection(publicKey);
|
|
76
113
|
}
|
|
77
114
|
const swarm = getSwarm();
|
|
78
115
|
const connectionStart = nowMs();
|
|
@@ -132,7 +169,7 @@ async function ensureRPCConnection(topic, publicKey, timeout) {
|
|
|
132
169
|
logger.debug(`⏳ Peer not connected after flush, waiting for connection event...`);
|
|
133
170
|
});
|
|
134
171
|
});
|
|
135
|
-
const rpc = await withTimeout(connectionPromise,
|
|
172
|
+
const rpc = await withTimeout(connectionPromise, getRemainingTimeout());
|
|
136
173
|
const connectionDuration = nowMs() - connectionStart;
|
|
137
174
|
cacheDelegationConnectionTime(publicKey, connectionDuration);
|
|
138
175
|
return rpc;
|
|
@@ -142,13 +179,7 @@ async function ensureRPCConnection(topic, publicKey, timeout) {
|
|
|
142
179
|
swarm.removeListener("connection", onConnection);
|
|
143
180
|
// Remove stale connection so next attempt creates a fresh one
|
|
144
181
|
// instead of reusing a dead RPC
|
|
145
|
-
|
|
146
|
-
activeRPCs.delete(publicKey);
|
|
147
|
-
const conn = activeConnections.get(publicKey);
|
|
148
|
-
if (conn) {
|
|
149
|
-
conn.destroy();
|
|
150
|
-
activeConnections.delete(publicKey);
|
|
151
|
-
}
|
|
182
|
+
cleanupStaleConnection(publicKey);
|
|
152
183
|
logger.error("Failed to establish RPC connection:", error);
|
|
153
184
|
throw new DelegateConnectionFailedError(`RPC connection failed: ${error instanceof Error ? error.message : String(error)}`, error);
|
|
154
185
|
}
|
|
@@ -162,7 +193,7 @@ export async function getRPC(topic, publicKey, options = {}) {
|
|
|
162
193
|
if (options.forceNewConnection) {
|
|
163
194
|
await closeConnection(publicKey);
|
|
164
195
|
}
|
|
165
|
-
return await ensureRPCConnection(topic, publicKey, options.timeout);
|
|
196
|
+
return await ensureRPCConnection(topic, publicKey, options.timeout, options.healthCheckTimeout);
|
|
166
197
|
}
|
|
167
198
|
/**
|
|
168
199
|
* Remove a stale RPC connection for a peer.
|
|
@@ -4,7 +4,7 @@ import { definePlugin, defineHandler, translateRequestSchema, translateResponseS
|
|
|
4
4
|
import { createStreamLogger, registerAddonLogger } from "../../../../logging/index.js";
|
|
5
5
|
import { parseModelPath } from "../../../utils/index.js";
|
|
6
6
|
import FilesystemDL from "@qvac/dl-filesystem";
|
|
7
|
-
import {
|
|
7
|
+
import { ModelLoadFailedError } from "../../../../utils/errors-server.js";
|
|
8
8
|
import { asLoader } from "../../utils/loader-adapter.js";
|
|
9
9
|
import { translate } from "../../ops/translate.js";
|
|
10
10
|
const BERGAMOT_CJK_LANG_PAIRS = ["enja", "enko", "enzh"];
|
|
@@ -90,9 +90,6 @@ function createNmtModel(modelId, modelPath, nmtConfig, srcVocabPath, dstVocabPat
|
|
|
90
90
|
}),
|
|
91
91
|
}),
|
|
92
92
|
};
|
|
93
|
-
if (config.modelType === TranslationNmtcpp.ModelTypes.IndicTrans) {
|
|
94
|
-
throw new TranslationFailedError("IndicTrans models are not supported with current NMT addon version.");
|
|
95
|
-
}
|
|
96
93
|
const model = new TranslationNmtcpp(args, config);
|
|
97
94
|
return { model, loader };
|
|
98
95
|
}
|
|
@@ -12,6 +12,7 @@ interface DelegateOptions {
|
|
|
12
12
|
topic: string;
|
|
13
13
|
providerPublicKey: string;
|
|
14
14
|
timeout?: number | undefined;
|
|
15
|
+
healthCheckTimeout?: number | undefined;
|
|
15
16
|
}
|
|
16
17
|
interface LocalOptions {
|
|
17
18
|
model: AnyModel;
|
|
@@ -39,6 +40,7 @@ export declare function registerModel(id: string, options: {
|
|
|
39
40
|
topic: string;
|
|
40
41
|
providerPublicKey: string;
|
|
41
42
|
timeout?: number;
|
|
43
|
+
healthCheckTimeout?: number;
|
|
42
44
|
}): void;
|
|
43
45
|
export declare function getModelEntry(id: string): ModelEntry | null;
|
|
44
46
|
export declare function getModel(id: string): AnyModel;
|
|
@@ -9,7 +9,7 @@ export function registerModel(id, options) {
|
|
|
9
9
|
}
|
|
10
10
|
const isDelegated = "topic" in options && "providerPublicKey" in options;
|
|
11
11
|
if (isDelegated) {
|
|
12
|
-
const { topic, providerPublicKey, timeout } = options;
|
|
12
|
+
const { topic, providerPublicKey, timeout, healthCheckTimeout } = options;
|
|
13
13
|
modelRegistry.set(id, {
|
|
14
14
|
id,
|
|
15
15
|
isDelegated: true,
|
|
@@ -17,6 +17,7 @@ export function registerModel(id, options) {
|
|
|
17
17
|
topic,
|
|
18
18
|
providerPublicKey,
|
|
19
19
|
timeout,
|
|
20
|
+
healthCheckTimeout,
|
|
20
21
|
},
|
|
21
22
|
});
|
|
22
23
|
logger.info(`Delegated model registered: ${id} -> topic: ${topic}, provider: ${providerPublicKey}, timeout: ${timeout}ms`);
|
|
@@ -9,18 +9,25 @@ import { nowMs, extractProfilingMeta, recordFailure, generateId, } from "../../p
|
|
|
9
9
|
import { withTimeout, withTimeoutStream } from "../../utils/withTimeout.js";
|
|
10
10
|
import { getServerLogger } from "../../logging/index.js";
|
|
11
11
|
import { DelegateProviderError } from "../../utils/errors-server.js";
|
|
12
|
+
import { cleanupStaleConnection } from "../bare/delegate-rpc-client.js";
|
|
12
13
|
import { shouldProfileDelegation, createDelegationTimings, createDelegationStreamTimings, recordDelegationEvents, recordDelegationStreamEvents, buildDelegationStreamBreakdown, flushServerConnectionEvent, consumeBreakdownConnectionTime, } from "./profiling/delegation-profiler.js";
|
|
13
14
|
const logger = getServerLogger();
|
|
14
|
-
|
|
15
|
-
function getNextCommandId() {
|
|
16
|
-
commandCounter = (commandCounter + 1) % Number.MAX_SAFE_INTEGER;
|
|
17
|
-
return commandCounter;
|
|
18
|
-
}
|
|
15
|
+
import { getNextCommandId } from "./delegate-utils.js";
|
|
19
16
|
function checkAndThrowError(response) {
|
|
20
17
|
if (response.type === "error") {
|
|
21
18
|
throw new DelegateProviderError(response.message || "Unknown provider error", response.code);
|
|
22
19
|
}
|
|
23
20
|
}
|
|
21
|
+
function isConnectionError(error) {
|
|
22
|
+
return !(error instanceof DelegateProviderError);
|
|
23
|
+
}
|
|
24
|
+
function cleanupDelegationPeer(options, error) {
|
|
25
|
+
if (!options?.peerKey)
|
|
26
|
+
return;
|
|
27
|
+
if (error !== undefined && !isConnectionError(error))
|
|
28
|
+
return;
|
|
29
|
+
cleanupStaleConnection(options.peerKey);
|
|
30
|
+
}
|
|
24
31
|
export async function send(request, rpc, options) {
|
|
25
32
|
const { profilingMeta } = options ?? {};
|
|
26
33
|
const shouldProfile = shouldProfileDelegation(request.type, profilingMeta);
|
|
@@ -30,20 +37,26 @@ export async function send(request, rpc, options) {
|
|
|
30
37
|
return sendProfiled(request, rpc, options, profilingMeta);
|
|
31
38
|
}
|
|
32
39
|
async function sendBase(request, rpc, options, profilingMeta) {
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
40
|
+
try {
|
|
41
|
+
const parsedRequest = requestSchema.parse(request);
|
|
42
|
+
const req = rpc.request(getNextCommandId());
|
|
43
|
+
logger.debug("[delegate-transport] Sending:", { type: request.type });
|
|
44
|
+
// Propagate per-call disable signal to delegated provider
|
|
45
|
+
const finalRequest = profilingMeta?.enabled === false
|
|
46
|
+
? { ...parsedRequest, [PROFILING_KEY]: { enabled: false } }
|
|
47
|
+
: parsedRequest;
|
|
48
|
+
const payload = JSON.stringify(finalRequest);
|
|
49
|
+
req.send(payload, "utf-8");
|
|
50
|
+
const response = await withTimeout(req.reply("utf-8"), options?.timeout);
|
|
51
|
+
const resPayload = responseSchema.parse(JSON.parse(response?.toString() || "{}"));
|
|
52
|
+
logger.debug("[delegate-transport] Response:", { type: resPayload.type });
|
|
53
|
+
checkAndThrowError(resPayload);
|
|
54
|
+
return resPayload;
|
|
55
|
+
}
|
|
56
|
+
catch (error) {
|
|
57
|
+
cleanupDelegationPeer(options, error);
|
|
58
|
+
throw error;
|
|
59
|
+
}
|
|
47
60
|
}
|
|
48
61
|
async function sendProfiled(request, rpc, options, profilingMeta) {
|
|
49
62
|
const profileId = profilingMeta?.id ?? generateId();
|
|
@@ -93,6 +106,7 @@ async function sendProfiled(request, rpc, options, profilingMeta) {
|
|
|
93
106
|
return resPayload;
|
|
94
107
|
}
|
|
95
108
|
catch (error) {
|
|
109
|
+
cleanupDelegationPeer(options, error);
|
|
96
110
|
const base = {
|
|
97
111
|
ts: nowMs(),
|
|
98
112
|
op: timings.requestType,
|
|
@@ -113,35 +127,41 @@ export async function* stream(request, rpc, options = {}) {
|
|
|
113
127
|
yield* streamProfiled(request, rpc, options, profilingMeta);
|
|
114
128
|
}
|
|
115
129
|
async function* streamBase(request, rpc, options = {}, profilingMeta) {
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
130
|
+
try {
|
|
131
|
+
const parsedRequest = requestSchema.parse(request);
|
|
132
|
+
const req = rpc.request(getNextCommandId());
|
|
133
|
+
logger.debug("[delegate-transport] Streaming:", { type: request.type });
|
|
134
|
+
// Propagate per-call disable signal to delegated provider
|
|
135
|
+
const finalRequest = profilingMeta?.enabled === false
|
|
136
|
+
? { ...parsedRequest, [PROFILING_KEY]: { enabled: false } }
|
|
137
|
+
: parsedRequest;
|
|
138
|
+
req.send(JSON.stringify(finalRequest), "utf-8");
|
|
139
|
+
const responseStream = req.createResponseStream({ encoding: "utf-8" });
|
|
140
|
+
let buffer = "";
|
|
141
|
+
async function* processStream() {
|
|
142
|
+
for await (const chunk of responseStream) {
|
|
143
|
+
yield chunk;
|
|
144
|
+
}
|
|
129
145
|
}
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
146
|
+
const streamWithTimeout = withTimeoutStream(processStream(), options?.timeout);
|
|
147
|
+
for await (const chunk of streamWithTimeout) {
|
|
148
|
+
buffer += chunk.toString();
|
|
149
|
+
// Process complete lines (newline-delimited JSON)
|
|
150
|
+
const lines = buffer.split("\n");
|
|
151
|
+
buffer = lines.pop() || ""; // Keep incomplete line in buffer
|
|
152
|
+
for (const line of lines) {
|
|
153
|
+
if (line.trim()) {
|
|
154
|
+
const response = responseSchema.parse(JSON.parse(line));
|
|
155
|
+
checkAndThrowError(response);
|
|
156
|
+
yield response;
|
|
157
|
+
}
|
|
142
158
|
}
|
|
143
159
|
}
|
|
144
160
|
}
|
|
161
|
+
catch (error) {
|
|
162
|
+
cleanupDelegationPeer(options, error);
|
|
163
|
+
throw error;
|
|
164
|
+
}
|
|
145
165
|
}
|
|
146
166
|
async function* streamProfiled(request, rpc, options = {}, profilingMeta) {
|
|
147
167
|
const profileId = profilingMeta?.id ?? generateId();
|
|
@@ -203,6 +223,7 @@ async function* streamProfiled(request, rpc, options = {}, profilingMeta) {
|
|
|
203
223
|
recordDelegationStreamEvents(timings, lastServerMeta);
|
|
204
224
|
}
|
|
205
225
|
catch (error) {
|
|
226
|
+
cleanupDelegationPeer(options, error);
|
|
206
227
|
const base = {
|
|
207
228
|
ts: nowMs(),
|
|
208
229
|
op: timings.requestType,
|
|
@@ -19,7 +19,9 @@ import { handleDeleteCache } from "./handlers/delete-cache.js";
|
|
|
19
19
|
import { handleTextToSpeech } from "./handlers/text-to-speech.js";
|
|
20
20
|
import { handleGetModelInfo } from "./handlers/get-model-info.js";
|
|
21
21
|
import { handleOCRStream } from "./handlers/ocr-stream.js";
|
|
22
|
-
import {
|
|
22
|
+
import { handleHeartbeat } from "./handlers/heartbeat.js";
|
|
23
|
+
import { handleHeartbeatDelegated } from "./handlers/heartbeat-delegated.js";
|
|
24
|
+
import { handleCancelDelegated } from "./handlers/cancel-delegated.js";
|
|
23
25
|
import { handlePluginInvoke, handlePluginInvokeStream, } from "./handlers/plugin-invoke.js";
|
|
24
26
|
import { handleModelRegistryList, handleModelRegistrySearch, handleModelRegistryGetModel, } from "./handlers/registry.js";
|
|
25
27
|
function ragSupportsProgress(request) {
|
|
@@ -33,9 +35,25 @@ function isModelDelegated(request) {
|
|
|
33
35
|
const entry = getModelEntry(request.modelId);
|
|
34
36
|
return entry?.isDelegated ?? false;
|
|
35
37
|
}
|
|
38
|
+
function isCancelDelegated(request) {
|
|
39
|
+
if (request.type !== "cancel")
|
|
40
|
+
return false;
|
|
41
|
+
if (request.operation === "inference") {
|
|
42
|
+
return isModelDelegated(request);
|
|
43
|
+
}
|
|
44
|
+
if (request.operation === "downloadAsset") {
|
|
45
|
+
return !!request.delegate;
|
|
46
|
+
}
|
|
47
|
+
return false;
|
|
48
|
+
}
|
|
36
49
|
export const registry = {
|
|
37
50
|
// Simple Reply handlers
|
|
38
|
-
|
|
51
|
+
heartbeat: {
|
|
52
|
+
type: "reply",
|
|
53
|
+
handler: handleHeartbeat,
|
|
54
|
+
delegatedHandler: handleHeartbeatDelegated,
|
|
55
|
+
isDelegated: (r) => r.type === "heartbeat" && !!r.delegate,
|
|
56
|
+
},
|
|
39
57
|
unloadModel: {
|
|
40
58
|
type: "reply",
|
|
41
59
|
handler: handleUnloadModel,
|
|
@@ -43,7 +61,12 @@ export const registry = {
|
|
|
43
61
|
isDelegated: isModelDelegated,
|
|
44
62
|
},
|
|
45
63
|
embed: { type: "reply", handler: handleEmbed },
|
|
46
|
-
cancel: {
|
|
64
|
+
cancel: {
|
|
65
|
+
type: "reply",
|
|
66
|
+
handler: cancelHandler,
|
|
67
|
+
delegatedHandler: handleCancelDelegated,
|
|
68
|
+
isDelegated: isCancelDelegated,
|
|
69
|
+
},
|
|
47
70
|
provide: { type: "reply", handler: provideHandler },
|
|
48
71
|
stopProvide: { type: "reply", handler: stopProvideHandler },
|
|
49
72
|
deleteCache: { type: "reply", handler: handleDeleteCache },
|
|
@@ -3,6 +3,7 @@ import { sendErrorResponse, sendStreamErrorResponse, } from "../error-handlers.j
|
|
|
3
3
|
import { setSDKConfig } from "../bare/registry/config-registry.js";
|
|
4
4
|
import { setRuntimeContext } from "../bare/registry/runtime-context-registry.js";
|
|
5
5
|
import {} from "./profiling/index.js";
|
|
6
|
+
import { nowMs } from "../../profiling/clock.js";
|
|
6
7
|
function getProfilingMetaFromRequest(request) {
|
|
7
8
|
if (PROFILING_KEY in request) {
|
|
8
9
|
return request[PROFILING_KEY];
|
|
@@ -56,12 +57,37 @@ async function executeStreamHandler(req, request, handler, profiler, isDelegated
|
|
|
56
57
|
sendStreamErrorResponse(stream, error, profiler);
|
|
57
58
|
}
|
|
58
59
|
}
|
|
60
|
+
const PROGRESS_THROTTLE_MS = 150;
|
|
59
61
|
async function executeProgressHandler(req, request, handler, profiler, isDelegated) {
|
|
60
62
|
const stream = req.createResponseStream();
|
|
61
63
|
profiler.startHandler();
|
|
62
|
-
|
|
64
|
+
let lastProgressWrite = 0;
|
|
65
|
+
let pendingUpdate = null;
|
|
66
|
+
let flushTimer = null;
|
|
67
|
+
const writeProgress = (update) => {
|
|
63
68
|
stream.write(profiler.serialize(update, false) + "\n", "utf-8");
|
|
64
69
|
};
|
|
70
|
+
const progressCallback = (update) => {
|
|
71
|
+
const now = nowMs();
|
|
72
|
+
if (now - lastProgressWrite >= PROGRESS_THROTTLE_MS) {
|
|
73
|
+
lastProgressWrite = now;
|
|
74
|
+
pendingUpdate = null;
|
|
75
|
+
writeProgress(update);
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
pendingUpdate = update;
|
|
79
|
+
if (!flushTimer) {
|
|
80
|
+
flushTimer = setTimeout(() => {
|
|
81
|
+
flushTimer = null;
|
|
82
|
+
if (pendingUpdate) {
|
|
83
|
+
lastProgressWrite = nowMs();
|
|
84
|
+
writeProgress(pendingUpdate);
|
|
85
|
+
pendingUpdate = null;
|
|
86
|
+
}
|
|
87
|
+
}, PROGRESS_THROTTLE_MS - (now - lastProgressWrite));
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
};
|
|
65
91
|
try {
|
|
66
92
|
let response;
|
|
67
93
|
if (isDelegated) {
|
|
@@ -75,11 +101,27 @@ async function executeProgressHandler(req, request, handler, profiler, isDelegat
|
|
|
75
101
|
else {
|
|
76
102
|
response = await handler(request, progressCallback);
|
|
77
103
|
}
|
|
104
|
+
if (flushTimer) {
|
|
105
|
+
clearTimeout(flushTimer);
|
|
106
|
+
flushTimer = null;
|
|
107
|
+
}
|
|
108
|
+
if (pendingUpdate) {
|
|
109
|
+
writeProgress(pendingUpdate);
|
|
110
|
+
pendingUpdate = null;
|
|
111
|
+
}
|
|
78
112
|
profiler.endHandler();
|
|
79
113
|
stream.write(profiler.serialize(response, true) + "\n", "utf-8");
|
|
80
114
|
stream.end();
|
|
81
115
|
}
|
|
82
116
|
catch (error) {
|
|
117
|
+
if (flushTimer) {
|
|
118
|
+
clearTimeout(flushTimer);
|
|
119
|
+
flushTimer = null;
|
|
120
|
+
}
|
|
121
|
+
if (pendingUpdate) {
|
|
122
|
+
writeProgress(pendingUpdate);
|
|
123
|
+
pendingUpdate = null;
|
|
124
|
+
}
|
|
83
125
|
profiler.endHandler();
|
|
84
126
|
sendStreamErrorResponse(stream, error, profiler);
|
|
85
127
|
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { CancelRequest, CancelResponse } from "../../../schemas/cancel";
|
|
2
|
+
import type { DelegatedHandlerOptions } from "../profiling/index";
|
|
3
|
+
export declare function handleCancelDelegated(request: CancelRequest, options?: DelegatedHandlerOptions): Promise<CancelResponse>;
|
|
4
|
+
//# sourceMappingURL=cancel-delegated.d.ts.map
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { getServerLogger } from "../../../logging/index.js";
|
|
2
|
+
import { getModelEntry } from "../../bare/registry/model-registry.js";
|
|
3
|
+
import { getRPC } from "../../bare/delegate-rpc-client.js";
|
|
4
|
+
import { send } from "../delegate-transport.js";
|
|
5
|
+
const logger = getServerLogger();
|
|
6
|
+
function resolveDelegationTarget(request) {
|
|
7
|
+
if (request.operation === "inference") {
|
|
8
|
+
const entry = getModelEntry(request.modelId);
|
|
9
|
+
if (!entry?.isDelegated || !entry.delegated) {
|
|
10
|
+
return null;
|
|
11
|
+
}
|
|
12
|
+
const target = {
|
|
13
|
+
topic: entry.delegated.topic,
|
|
14
|
+
providerPublicKey: entry.delegated.providerPublicKey,
|
|
15
|
+
};
|
|
16
|
+
if (entry.delegated.timeout !== undefined) {
|
|
17
|
+
target.timeout = entry.delegated.timeout;
|
|
18
|
+
}
|
|
19
|
+
return target;
|
|
20
|
+
}
|
|
21
|
+
if (request.operation === "downloadAsset" && request.delegate) {
|
|
22
|
+
const target = {
|
|
23
|
+
topic: request.delegate.topic,
|
|
24
|
+
providerPublicKey: request.delegate.providerPublicKey,
|
|
25
|
+
};
|
|
26
|
+
if (request.delegate.timeout !== undefined) {
|
|
27
|
+
target.timeout = request.delegate.timeout;
|
|
28
|
+
}
|
|
29
|
+
return target;
|
|
30
|
+
}
|
|
31
|
+
return null;
|
|
32
|
+
}
|
|
33
|
+
function toProviderCancelRequest(request) {
|
|
34
|
+
if (request.operation !== "downloadAsset") {
|
|
35
|
+
return request;
|
|
36
|
+
}
|
|
37
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
38
|
+
const { delegate: _delegate, ...providerRequest } = request;
|
|
39
|
+
return providerRequest;
|
|
40
|
+
}
|
|
41
|
+
export async function handleCancelDelegated(request, options) {
|
|
42
|
+
const target = resolveDelegationTarget(request);
|
|
43
|
+
if (!target) {
|
|
44
|
+
logger.warn(`Delegated cancel skipped (no delegation target): operation=${request.operation}`);
|
|
45
|
+
return { type: "cancel", success: true };
|
|
46
|
+
}
|
|
47
|
+
try {
|
|
48
|
+
const rpc = await getRPC(target.topic, target.providerPublicKey, {
|
|
49
|
+
timeout: target.timeout,
|
|
50
|
+
});
|
|
51
|
+
const delegateOpts = {
|
|
52
|
+
peerKey: target.providerPublicKey,
|
|
53
|
+
};
|
|
54
|
+
if (target.timeout !== undefined) {
|
|
55
|
+
delegateOpts.timeout = target.timeout;
|
|
56
|
+
}
|
|
57
|
+
if (options?.profilingMeta) {
|
|
58
|
+
delegateOpts.profilingMeta = options.profilingMeta;
|
|
59
|
+
}
|
|
60
|
+
await send(toProviderCancelRequest(request), rpc, delegateOpts);
|
|
61
|
+
return { type: "cancel", success: true };
|
|
62
|
+
}
|
|
63
|
+
catch (error) {
|
|
64
|
+
logger.error("Error during delegated cancellation:", error);
|
|
65
|
+
return {
|
|
66
|
+
type: "cancel",
|
|
67
|
+
success: false,
|
|
68
|
+
error: error instanceof Error ? error.message : "Unknown error",
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
}
|
|
@@ -11,11 +11,11 @@ export async function* handleCompletionStreamDelegated(request, options) {
|
|
|
11
11
|
if (!entry?.isDelegated || !entry.delegated) {
|
|
12
12
|
throw new ModelIsDelegatedError(request.modelId);
|
|
13
13
|
}
|
|
14
|
-
const { topic, providerPublicKey, timeout } = entry.delegated;
|
|
14
|
+
const { topic, providerPublicKey, timeout, healthCheckTimeout } = entry.delegated;
|
|
15
15
|
try {
|
|
16
16
|
logger.debug(`📤 Sending delegated completionStream request to provider: ${providerPublicKey}${timeout ? `, timeout: ${timeout}ms` : ""}`);
|
|
17
17
|
// Create RPC instance for this HyperSwarm peer
|
|
18
|
-
const rpc = await getRPC(topic, providerPublicKey, { timeout });
|
|
18
|
+
const rpc = await getRPC(topic, providerPublicKey, { timeout, healthCheckTimeout });
|
|
19
19
|
// Build delegate options with profiling metadata
|
|
20
20
|
const delegateOpts = { peerKey: providerPublicKey };
|
|
21
21
|
if (profilingMeta) {
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { HeartbeatRequest, HeartbeatResponse } from "../../../schemas/index";
|
|
2
|
+
import type { DelegatedHandlerOptions } from "../profiling/index";
|
|
3
|
+
export declare function handleHeartbeatDelegated(request: HeartbeatRequest, options?: DelegatedHandlerOptions): Promise<HeartbeatResponse>;
|
|
4
|
+
//# sourceMappingURL=heartbeat-delegated.d.ts.map
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { getServerLogger } from "../../../logging/index.js";
|
|
2
|
+
import { getRPC } from "../../bare/delegate-rpc-client.js";
|
|
3
|
+
import { send } from "../delegate-transport.js";
|
|
4
|
+
import { DelegateConnectionFailedError } from "../../../utils/errors-server.js";
|
|
5
|
+
const logger = getServerLogger();
|
|
6
|
+
export async function handleHeartbeatDelegated(request, options) {
|
|
7
|
+
const { delegate } = request;
|
|
8
|
+
if (!delegate) {
|
|
9
|
+
throw new DelegateConnectionFailedError("Delegated heartbeat handler called without delegate info");
|
|
10
|
+
}
|
|
11
|
+
const { topic, providerPublicKey, timeout } = delegate;
|
|
12
|
+
try {
|
|
13
|
+
const rpc = await getRPC(topic, providerPublicKey, { timeout });
|
|
14
|
+
const delegateOpts = {
|
|
15
|
+
peerKey: providerPublicKey,
|
|
16
|
+
};
|
|
17
|
+
if (timeout !== undefined) {
|
|
18
|
+
delegateOpts.timeout = timeout;
|
|
19
|
+
}
|
|
20
|
+
if (options?.profilingMeta) {
|
|
21
|
+
delegateOpts.profilingMeta = options.profilingMeta;
|
|
22
|
+
}
|
|
23
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
24
|
+
const { delegate: _delegate, ...providerRequest } = request;
|
|
25
|
+
const response = await send(providerRequest, rpc, delegateOpts);
|
|
26
|
+
return response;
|
|
27
|
+
}
|
|
28
|
+
catch (error) {
|
|
29
|
+
logger.error("Error during delegated heartbeat:", error);
|
|
30
|
+
throw error;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
@@ -14,11 +14,11 @@ import { handleDeleteCache } from "./delete-cache";
|
|
|
14
14
|
import { handleTextToSpeech } from "./text-to-speech";
|
|
15
15
|
import { handleGetModelInfo } from "./get-model-info";
|
|
16
16
|
import { handleOCRStream } from "./ocr-stream";
|
|
17
|
-
import {
|
|
17
|
+
import { handleHeartbeat } from "./heartbeat";
|
|
18
18
|
import { handlePluginInvoke, handlePluginInvokeStream } from "./plugin-invoke";
|
|
19
19
|
import { handleModelRegistryList, handleModelRegistrySearch, handleModelRegistryGetModel } from "./registry";
|
|
20
20
|
export declare const handlers: {
|
|
21
|
-
|
|
21
|
+
heartbeat: typeof handleHeartbeat;
|
|
22
22
|
completionStream: typeof handleCompletionStream;
|
|
23
23
|
downloadAsset: typeof handleDownloadAsset;
|
|
24
24
|
deleteCache: typeof handleDeleteCache;
|
|
@@ -14,11 +14,11 @@ import { handleDeleteCache } from "./delete-cache.js";
|
|
|
14
14
|
import { handleTextToSpeech } from "./text-to-speech.js";
|
|
15
15
|
import { handleGetModelInfo } from "./get-model-info.js";
|
|
16
16
|
import { handleOCRStream } from "./ocr-stream.js";
|
|
17
|
-
import {
|
|
17
|
+
import { handleHeartbeat } from "./heartbeat.js";
|
|
18
18
|
import { handlePluginInvoke, handlePluginInvokeStream } from "./plugin-invoke.js";
|
|
19
19
|
import { handleModelRegistryList, handleModelRegistrySearch, handleModelRegistryGetModel, } from "./registry.js";
|
|
20
20
|
export const handlers = {
|
|
21
|
-
|
|
21
|
+
heartbeat: handleHeartbeat,
|
|
22
22
|
completionStream: handleCompletionStream,
|
|
23
23
|
downloadAsset: handleDownloadAsset,
|
|
24
24
|
deleteCache: handleDeleteCache,
|