@qvac/sdk 0.7.1 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (257) hide show
  1. package/NOTICE +73 -10
  2. package/README.md +4 -4
  3. package/bare-imports.json +9 -0
  4. package/dist/client/api/cancel.d.ts +9 -0
  5. package/dist/client/api/cancel.js +9 -0
  6. package/dist/client/api/completion-stream.d.ts +2 -1
  7. package/dist/client/api/completion-stream.js +2 -1
  8. package/dist/client/api/download-asset.d.ts +3 -2
  9. package/dist/client/api/download-asset.js +4 -3
  10. package/dist/client/api/embed.d.ts +5 -2
  11. package/dist/client/api/embed.js +2 -2
  12. package/dist/client/api/get-model-info.d.ts +1 -1
  13. package/dist/client/api/heartbeat.d.ts +30 -0
  14. package/dist/client/api/heartbeat.js +38 -0
  15. package/dist/client/api/index.d.ts +1 -1
  16. package/dist/client/api/index.js +1 -1
  17. package/dist/client/api/invoke-plugin.d.ts +3 -2
  18. package/dist/client/api/invoke-plugin.js +4 -4
  19. package/dist/client/api/load-model.d.ts +12 -10
  20. package/dist/client/api/load-model.js +3 -3
  21. package/dist/client/api/rag.d.ts +10 -10
  22. package/dist/client/api/rag.js +21 -21
  23. package/dist/client/api/text-to-speech.d.ts +2 -2
  24. package/dist/client/api/text-to-speech.js +3 -3
  25. package/dist/client/api/transcribe.d.ts +5 -3
  26. package/dist/client/api/transcribe.js +6 -4
  27. package/dist/client/api/translate.d.ts +2 -2
  28. package/dist/client/api/translate.js +3 -3
  29. package/dist/client/rpc/node-rpc-client.d.ts +2 -7
  30. package/dist/client/rpc/node-rpc-client.js +2 -39
  31. package/dist/client/rpc/profiling/index.d.ts +2 -0
  32. package/dist/client/rpc/profiling/index.js +1 -0
  33. package/dist/client/rpc/profiling/profiler.d.ts +27 -0
  34. package/dist/client/rpc/profiling/profiler.js +139 -0
  35. package/dist/client/rpc/rpc-client.d.ts +2 -2
  36. package/dist/client/rpc/rpc-client.js +174 -12
  37. package/dist/examples/delegated-inference/consumer-profiled.d.ts +2 -0
  38. package/dist/examples/delegated-inference/consumer-profiled.js +67 -0
  39. package/dist/examples/embed-p2p.d.ts +2 -0
  40. package/dist/examples/llamacpp-multimodal.js +1 -1
  41. package/dist/examples/llamacpp-p2p.d.ts +2 -0
  42. package/dist/examples/parallel-download.js +45 -13
  43. package/dist/examples/profiling/basic.d.ts +2 -0
  44. package/dist/examples/profiling/basic.js +71 -0
  45. package/dist/examples/profiling/per-call.d.ts +2 -0
  46. package/dist/examples/profiling/per-call.js +30 -0
  47. package/dist/examples/seed-p2p.d.ts +2 -0
  48. package/dist/examples/transcription/parakeet-ctc-filesystem.d.ts +2 -0
  49. package/dist/examples/transcription/parakeet-ctc-filesystem.js +40 -0
  50. package/dist/examples/transcription/parakeet-microphone-record.d.ts +2 -0
  51. package/dist/examples/transcription/parakeet-microphone-record.js +95 -0
  52. package/dist/examples/transcription/parakeet-sortformer.d.ts +2 -0
  53. package/dist/examples/transcription/parakeet-sortformer.js +115 -0
  54. package/dist/examples/transcription/parakeet-tdt-filesystem.d.ts +2 -0
  55. package/dist/examples/transcription/parakeet-tdt-filesystem.js +44 -0
  56. package/dist/examples/translation/translation-bergamot-pivot.d.ts +2 -0
  57. package/dist/examples/translation/translation-bergamot-pivot.js +73 -0
  58. package/dist/examples/translation/translation-llm-afriquegemma.d.ts +2 -0
  59. package/dist/examples/translation/translation-llm-afriquegemma.js +39 -0
  60. package/dist/index.d.ts +4 -2
  61. package/dist/index.js +3 -1
  62. package/dist/logging/index.d.ts +1 -1
  63. package/dist/logging/index.js +1 -1
  64. package/dist/logging/namespaces.d.ts +3 -9
  65. package/dist/logging/namespaces.js +2 -8
  66. package/dist/logging/registry.js +15 -1
  67. package/dist/models/registry/models.d.ts +1671 -310
  68. package/dist/models/registry/models.js +2116 -710
  69. package/dist/models/update-models/codegen.js +2 -2
  70. package/dist/models/update-models/history.d.ts +11 -0
  71. package/dist/models/update-models/history.js +24 -5
  72. package/dist/models/update-models/index.js +17 -16
  73. package/dist/models/update-models/naming.js +56 -0
  74. package/dist/profiling/aggregator.d.ts +15 -0
  75. package/dist/profiling/aggregator.js +100 -0
  76. package/dist/profiling/clock.d.ts +12 -0
  77. package/dist/profiling/clock.js +51 -0
  78. package/dist/profiling/controller.d.ts +33 -0
  79. package/dist/profiling/controller.js +123 -0
  80. package/dist/profiling/envelope.d.ts +10 -0
  81. package/dist/profiling/envelope.js +36 -0
  82. package/dist/profiling/events.d.ts +20 -0
  83. package/dist/profiling/events.js +37 -0
  84. package/dist/profiling/exporters.d.ts +10 -0
  85. package/dist/profiling/exporters.js +236 -0
  86. package/dist/profiling/index.d.ts +36 -0
  87. package/dist/profiling/index.js +31 -0
  88. package/dist/profiling/ring-buffer.d.ts +19 -0
  89. package/dist/profiling/ring-buffer.js +83 -0
  90. package/dist/profiling/types.d.ts +47 -0
  91. package/dist/profiling/types.js +1 -0
  92. package/dist/schemas/cancel.d.ts +12 -0
  93. package/dist/schemas/cancel.js +2 -0
  94. package/dist/schemas/common.d.ts +1927 -135
  95. package/dist/schemas/common.js +6 -3
  96. package/dist/schemas/completion-stream.d.ts +31 -0
  97. package/dist/schemas/completion-stream.js +13 -0
  98. package/dist/schemas/delegate.d.ts +8 -0
  99. package/dist/schemas/delegate.js +5 -2
  100. package/dist/schemas/download-asset.d.ts +14 -3
  101. package/dist/schemas/download-asset.js +3 -1
  102. package/dist/schemas/engine-addon-map.js +9 -6
  103. package/dist/schemas/get-model-info.d.ts +2 -0
  104. package/dist/schemas/get-model-info.js +1 -0
  105. package/dist/schemas/heartbeat.d.ts +17 -0
  106. package/dist/schemas/heartbeat.js +10 -0
  107. package/dist/schemas/index.d.ts +3 -2
  108. package/dist/schemas/index.js +3 -2
  109. package/dist/schemas/llamacpp-config.d.ts +140 -14
  110. package/dist/schemas/llamacpp-config.js +11 -7
  111. package/dist/schemas/load-model.d.ts +9762 -2515
  112. package/dist/schemas/load-model.js +142 -182
  113. package/dist/schemas/model-src-utils.d.ts +10 -2
  114. package/dist/schemas/model-types.d.ts +19 -1
  115. package/dist/schemas/model-types.js +11 -0
  116. package/dist/schemas/ocr.d.ts +37 -0
  117. package/dist/schemas/ocr.js +5 -0
  118. package/dist/schemas/plugin.d.ts +51 -5
  119. package/dist/schemas/plugin.js +26 -0
  120. package/dist/schemas/profiling.d.ts +105 -0
  121. package/dist/schemas/profiling.js +66 -0
  122. package/dist/schemas/registry.d.ts +18 -0
  123. package/dist/schemas/registry.js +2 -0
  124. package/dist/schemas/reload-config.d.ts +126 -6
  125. package/dist/schemas/reload-config.js +1 -1
  126. package/dist/schemas/sdk-config.d.ts +2280 -24
  127. package/dist/schemas/sdk-config.js +6 -1
  128. package/dist/schemas/sdk-errors-client.d.ts +1 -0
  129. package/dist/schemas/sdk-errors-client.js +7 -0
  130. package/dist/schemas/sdk-errors-server.d.ts +3 -0
  131. package/dist/schemas/sdk-errors-server.js +15 -0
  132. package/dist/schemas/text-to-speech.d.ts +90 -68
  133. package/dist/schemas/text-to-speech.js +16 -33
  134. package/dist/schemas/transcription-config.d.ts +383 -0
  135. package/dist/schemas/{whispercpp-config.js → transcription-config.js} +29 -1
  136. package/dist/schemas/translation-config.d.ts +463 -8
  137. package/dist/schemas/translation-config.js +35 -2
  138. package/dist/server/bare/delegate-rpc-client.js +52 -11
  139. package/dist/server/bare/ops/load-model.d.ts +7 -1
  140. package/dist/server/bare/ops/load-model.js +41 -42
  141. package/dist/server/bare/ops/transcribe.d.ts +3 -0
  142. package/dist/server/bare/ops/transcribe.js +75 -0
  143. package/dist/server/bare/ops/translate.d.ts +2 -0
  144. package/dist/server/bare/ops/translate.js +12 -4
  145. package/dist/server/bare/plugins/index.d.ts +1 -0
  146. package/dist/server/bare/plugins/index.js +1 -0
  147. package/dist/server/bare/plugins/llamacpp-completion/ops/completion-stream.d.ts +2 -1
  148. package/dist/server/bare/plugins/llamacpp-completion/ops/completion-stream.js +8 -6
  149. package/dist/server/bare/plugins/llamacpp-completion/plugin.d.ts +108 -3
  150. package/dist/server/bare/plugins/llamacpp-completion/plugin.js +19 -6
  151. package/dist/server/bare/plugins/llamacpp-embedding/plugin.d.ts +32 -2
  152. package/dist/server/bare/plugins/llamacpp-embedding/plugin.js +31 -17
  153. package/dist/server/bare/plugins/nmtcpp-translation/plugin.d.ts +325 -3
  154. package/dist/server/bare/plugins/nmtcpp-translation/plugin.js +117 -13
  155. package/dist/server/bare/plugins/onnx-ocr/plugin.d.ts +50 -2
  156. package/dist/server/bare/plugins/onnx-ocr/plugin.js +46 -5
  157. package/dist/server/bare/plugins/onnx-tts/plugin.d.ts +351 -4
  158. package/dist/server/bare/plugins/onnx-tts/plugin.js +81 -61
  159. package/dist/server/bare/plugins/parakeet-transcription/plugin.d.ts +317 -0
  160. package/dist/server/bare/plugins/parakeet-transcription/plugin.js +154 -0
  161. package/dist/server/bare/plugins/whispercpp-transcription/plugin.d.ts +209 -3
  162. package/dist/server/bare/plugins/whispercpp-transcription/plugin.js +21 -9
  163. package/dist/server/bare/rag-hyperdb/rag-workspace-manager.js +2 -2
  164. package/dist/server/bare/registry/model-config-utils.js +3 -1
  165. package/dist/server/bare/registry/model-registry.d.ts +2 -0
  166. package/dist/server/bare/registry/model-registry.js +2 -1
  167. package/dist/server/bare/utils/audio-input.d.ts +10 -0
  168. package/dist/server/bare/utils/audio-input.js +33 -0
  169. package/dist/server/error-handlers.d.ts +3 -2
  170. package/dist/server/error-handlers.js +11 -8
  171. package/dist/server/plugins/registry.js +5 -1
  172. package/dist/server/rpc/delegate-transport.d.ts +11 -3
  173. package/dist/server/rpc/delegate-transport.js +214 -38
  174. package/dist/server/rpc/delegate-utils.d.ts +2 -0
  175. package/dist/server/rpc/delegate-utils.js +5 -0
  176. package/dist/server/rpc/handle-request.js +43 -4
  177. package/dist/server/rpc/handler-registry.js +33 -4
  178. package/dist/server/rpc/handler-utils.d.ts +5 -4
  179. package/dist/server/rpc/handler-utils.js +114 -31
  180. package/dist/server/rpc/handlers/cancel-delegated.d.ts +4 -0
  181. package/dist/server/rpc/handlers/cancel-delegated.js +71 -0
  182. package/dist/server/rpc/handlers/completion-stream-delegated.d.ts +3 -1
  183. package/dist/server/rpc/handlers/completion-stream-delegated.js +13 -4
  184. package/dist/server/rpc/handlers/download-asset.js +55 -3
  185. package/dist/server/rpc/handlers/heartbeat-delegated.d.ts +4 -0
  186. package/dist/server/rpc/handlers/heartbeat-delegated.js +32 -0
  187. package/dist/server/rpc/handlers/heartbeat.d.ts +3 -0
  188. package/dist/server/rpc/handlers/heartbeat.js +3 -0
  189. package/dist/server/rpc/handlers/index.d.ts +2 -2
  190. package/dist/server/rpc/handlers/index.js +2 -2
  191. package/dist/server/rpc/handlers/load-model/download-stats.d.ts +6 -0
  192. package/dist/server/rpc/handlers/load-model/download-stats.js +87 -0
  193. package/dist/server/rpc/handlers/load-model/handler.js +108 -123
  194. package/dist/server/rpc/handlers/load-model/http.d.ts +2 -1
  195. package/dist/server/rpc/handlers/load-model/http.js +21 -5
  196. package/dist/server/rpc/handlers/load-model/hyperdrive.d.ts +2 -1
  197. package/dist/server/rpc/handlers/load-model/hyperdrive.js +19 -14
  198. package/dist/server/rpc/handlers/load-model/registry.d.ts +2 -1
  199. package/dist/server/rpc/handlers/load-model/registry.js +113 -136
  200. package/dist/server/rpc/handlers/load-model/resolve.d.ts +2 -0
  201. package/dist/server/rpc/handlers/load-model/resolve.js +55 -22
  202. package/dist/server/rpc/handlers/load-model/types.d.ts +29 -0
  203. package/dist/server/rpc/handlers/load-model/types.js +1 -0
  204. package/dist/server/rpc/handlers/load-model-delegated.d.ts +5 -1
  205. package/dist/server/rpc/handlers/load-model-delegated.js +41 -11
  206. package/dist/server/rpc/handlers/plugin-dispatch.js +15 -10
  207. package/dist/server/rpc/handlers/plugin-invoke.js +56 -51
  208. package/dist/server/rpc/handlers/rag.js +24 -0
  209. package/dist/server/rpc/handlers/unload-model-delegated.d.ts +3 -0
  210. package/dist/server/rpc/handlers/unload-model-delegated.js +32 -0
  211. package/dist/server/rpc/profiling/context.d.ts +18 -0
  212. package/dist/server/rpc/profiling/context.js +38 -0
  213. package/dist/server/rpc/profiling/delegation-profiler.d.ts +32 -0
  214. package/dist/server/rpc/profiling/delegation-profiler.js +154 -0
  215. package/dist/server/rpc/profiling/index.d.ts +5 -0
  216. package/dist/server/rpc/profiling/index.js +4 -0
  217. package/dist/server/rpc/profiling/operation-metrics.d.ts +25 -0
  218. package/dist/server/rpc/profiling/operation-metrics.js +199 -0
  219. package/dist/server/rpc/profiling/operation-wrappers.d.ts +18 -0
  220. package/dist/server/rpc/profiling/operation-wrappers.js +124 -0
  221. package/dist/server/rpc/profiling/profiler.d.ts +13 -0
  222. package/dist/server/rpc/profiling/profiler.js +97 -0
  223. package/dist/server/utils/cache.d.ts +2 -1
  224. package/dist/server/utils/cache.js +5 -1
  225. package/dist/server/utils/checksum.d.ts +2 -0
  226. package/dist/server/utils/checksum.js +7 -0
  227. package/dist/server/utils/formatting.d.ts +6 -0
  228. package/dist/server/utils/formatting.js +35 -0
  229. package/dist/server/worker.js +2 -1
  230. package/dist/utils/errors-client.d.ts +3 -0
  231. package/dist/utils/errors-client.js +6 -0
  232. package/dist/utils/errors-server.d.ts +9 -0
  233. package/dist/utils/errors-server.js +15 -0
  234. package/dist/utils/global-singleton.d.ts +2 -0
  235. package/dist/utils/global-singleton.js +10 -0
  236. package/package.json +22 -18
  237. package/dist/client/api/ping.d.ts +0 -11
  238. package/dist/client/api/ping.js +0 -19
  239. package/dist/examples/embed-hyperdrive.d.ts +0 -2
  240. package/dist/examples/llamacpp-hyperdrive.d.ts +0 -2
  241. package/dist/examples/seed-hyperdrive.d.ts +0 -2
  242. package/dist/schemas/ping.d.ts +0 -11
  243. package/dist/schemas/ping.js +0 -8
  244. package/dist/schemas/whispercpp-config.d.ts +0 -70
  245. package/dist/server/bare/plugins/whispercpp-transcription/ops/transcribe-stream.d.ts +0 -3
  246. package/dist/server/bare/plugins/whispercpp-transcription/ops/transcribe-stream.js +0 -81
  247. package/dist/server/rpc/handlers/ping.d.ts +0 -3
  248. package/dist/server/rpc/handlers/ping.js +0 -3
  249. /package/dist/examples/{embed-hyperdrive.js → embed-p2p.js} +0 -0
  250. /package/dist/examples/{llamacpp-hyperdrive.js → llamacpp-p2p.js} +0 -0
  251. /package/dist/examples/{seed-hyperdrive.js → seed-p2p.js} +0 -0
  252. /package/dist/examples/{whispercpp-filesystem.d.ts → transcription/whispercpp-filesystem.d.ts} +0 -0
  253. /package/dist/examples/{whispercpp-filesystem.js → transcription/whispercpp-filesystem.js} +0 -0
  254. /package/dist/examples/{whispercpp-microphone-record.d.ts → transcription/whispercpp-microphone-record.d.ts} +0 -0
  255. /package/dist/examples/{whispercpp-microphone-record.js → transcription/whispercpp-microphone-record.js} +0 -0
  256. /package/dist/examples/{whispercpp-prompt.d.ts → transcription/whispercpp-prompt.d.ts} +0 -0
  257. /package/dist/examples/{whispercpp-prompt.js → transcription/whispercpp-prompt.js} +0 -0
@@ -4,57 +4,233 @@
4
4
  * This module provides `send` and `stream` functions for server-side
5
5
  * delegation to remote peers (via HyperSwarm RPC).
6
6
  */
7
- import { requestSchema, responseSchema, } from "../../schemas/index.js";
7
+ import { requestSchema, responseSchema, PROFILING_KEY, DELEGATION_BREAKDOWN_KEY, OPERATION_EVENT_KEY, } from "../../schemas/index.js";
8
+ import { nowMs, extractProfilingMeta, recordFailure, generateId, } from "../../profiling/index.js";
8
9
  import { withTimeout, withTimeoutStream } from "../../utils/withTimeout.js";
9
10
  import { getServerLogger } from "../../logging/index.js";
10
11
  import { DelegateProviderError } from "../../utils/errors-server.js";
12
+ import { cleanupStaleConnection } from "../bare/delegate-rpc-client.js";
13
+ import { shouldProfileDelegation, createDelegationTimings, createDelegationStreamTimings, recordDelegationEvents, recordDelegationStreamEvents, buildDelegationStreamBreakdown, flushServerConnectionEvent, consumeBreakdownConnectionTime, } from "./profiling/delegation-profiler.js";
11
14
  const logger = getServerLogger();
12
- let commandCounter = 0;
13
- function getNextCommandId() {
14
- commandCounter = (commandCounter + 1) % Number.MAX_SAFE_INTEGER;
15
- return commandCounter;
16
- }
15
+ import { getNextCommandId } from "./delegate-utils.js";
17
16
  function checkAndThrowError(response) {
18
17
  if (response.type === "error") {
19
18
  throw new DelegateProviderError(response.message || "Unknown provider error", response.code);
20
19
  }
21
20
  }
21
+ function isConnectionError(error) {
22
+ return !(error instanceof DelegateProviderError);
23
+ }
24
+ function cleanupDelegationPeer(options, error) {
25
+ if (!options?.peerKey)
26
+ return;
27
+ if (error !== undefined && !isConnectionError(error))
28
+ return;
29
+ cleanupStaleConnection(options.peerKey);
30
+ }
22
31
  export async function send(request, rpc, options) {
23
- const parsedRequest = requestSchema.parse(request);
24
- const req = rpc.request(getNextCommandId());
25
- logger.debug("[delegate-transport] Sending:", { type: request.type });
26
- const payload = JSON.stringify(parsedRequest);
27
- req.send(payload, "utf-8");
28
- const response = await withTimeout(req.reply("utf-8"), options?.timeout);
29
- const resPayload = responseSchema.parse(JSON.parse(response?.toString() || "{}"));
30
- logger.debug("[delegate-transport] Response:", { type: resPayload.type });
31
- checkAndThrowError(resPayload);
32
- return resPayload;
32
+ const { profilingMeta } = options ?? {};
33
+ const shouldProfile = shouldProfileDelegation(request.type, profilingMeta);
34
+ if (!shouldProfile) {
35
+ return sendBase(request, rpc, options, profilingMeta);
36
+ }
37
+ return sendProfiled(request, rpc, options, profilingMeta);
38
+ }
39
+ async function sendBase(request, rpc, options, profilingMeta) {
40
+ try {
41
+ const parsedRequest = requestSchema.parse(request);
42
+ const req = rpc.request(getNextCommandId());
43
+ logger.debug("[delegate-transport] Sending:", { type: request.type });
44
+ // Propagate per-call disable signal to delegated provider
45
+ const finalRequest = profilingMeta?.enabled === false
46
+ ? { ...parsedRequest, [PROFILING_KEY]: { enabled: false } }
47
+ : parsedRequest;
48
+ const payload = JSON.stringify(finalRequest);
49
+ req.send(payload, "utf-8");
50
+ const response = await withTimeout(req.reply("utf-8"), options?.timeout);
51
+ const resPayload = responseSchema.parse(JSON.parse(response?.toString() || "{}"));
52
+ logger.debug("[delegate-transport] Response:", { type: resPayload.type });
53
+ checkAndThrowError(resPayload);
54
+ return resPayload;
55
+ }
56
+ catch (error) {
57
+ cleanupDelegationPeer(options, error);
58
+ throw error;
59
+ }
60
+ }
61
+ async function sendProfiled(request, rpc, options, profilingMeta) {
62
+ const profileId = profilingMeta?.id ?? generateId();
63
+ const includeServerBreakdown = profilingMeta?.includeServer ?? false;
64
+ const timings = createDelegationTimings(profileId, request.type);
65
+ try {
66
+ if (options?.peerKey) {
67
+ flushServerConnectionEvent(options.peerKey);
68
+ }
69
+ const connectionMs = options?.peerKey
70
+ ? consumeBreakdownConnectionTime(options.peerKey)
71
+ : undefined;
72
+ const parsedRequest = requestSchema.parse(request);
73
+ const req = rpc.request(getNextCommandId());
74
+ logger.debug("[delegate-transport] Sending (profiled):", {
75
+ type: request.type,
76
+ });
77
+ const stringifyStart = nowMs();
78
+ const profilingEnvelope = { id: profileId };
79
+ if (includeServerBreakdown) {
80
+ profilingEnvelope["includeServer"] = true;
81
+ }
82
+ const requestWithProfiling = {
83
+ ...parsedRequest,
84
+ [PROFILING_KEY]: profilingEnvelope,
85
+ };
86
+ const payload = JSON.stringify(requestWithProfiling);
87
+ timings.requestStringifyMs = nowMs() - stringifyStart;
88
+ timings.sendStart = nowMs();
89
+ req.send(payload, "utf-8");
90
+ const response = await withTimeout(req.reply("utf-8"), options?.timeout);
91
+ timings.firstResponseAt = nowMs();
92
+ const parseStart = nowMs();
93
+ const rawPayload = JSON.parse(response?.toString() || "{}");
94
+ timings.responseJsonParseMs = nowMs() - parseStart;
95
+ const resPayload = responseSchema.parse(rawPayload);
96
+ logger.debug("[delegate-transport] Response (profiled):", {
97
+ type: resPayload.type,
98
+ });
99
+ checkAndThrowError(resPayload);
100
+ const serverMeta = extractProfilingMeta(rawPayload);
101
+ const delegationBreakdown = recordDelegationEvents(timings, serverMeta, connectionMs);
102
+ resPayload[DELEGATION_BREAKDOWN_KEY] = delegationBreakdown;
103
+ if (serverMeta?.operation) {
104
+ resPayload[OPERATION_EVENT_KEY] = serverMeta.operation;
105
+ }
106
+ return resPayload;
107
+ }
108
+ catch (error) {
109
+ cleanupDelegationPeer(options, error);
110
+ const base = {
111
+ ts: nowMs(),
112
+ op: timings.requestType,
113
+ kind: "delegation",
114
+ profileId: timings.profileId,
115
+ };
116
+ recordFailure(base, timings.requestStart, error);
117
+ throw error;
118
+ }
33
119
  }
34
120
  export async function* stream(request, rpc, options = {}) {
35
- const parsedRequest = requestSchema.parse(request);
36
- const req = rpc.request(getNextCommandId());
37
- logger.debug("[delegate-transport] Streaming:", { type: request.type });
38
- req.send(JSON.stringify(parsedRequest), "utf-8");
39
- const responseStream = req.createResponseStream({ encoding: "utf-8" });
40
- let buffer = "";
41
- async function* processStream() {
42
- for await (const chunk of responseStream) {
43
- yield chunk;
44
- }
45
- }
46
- const streamWithTimeout = withTimeoutStream(processStream(), options?.timeout);
47
- for await (const chunk of streamWithTimeout) {
48
- buffer += chunk.toString();
49
- // Process complete lines (newline-delimited JSON)
50
- const lines = buffer.split("\n");
51
- buffer = lines.pop() || ""; // Keep incomplete line in buffer
52
- for (const line of lines) {
53
- if (line.trim()) {
54
- const response = responseSchema.parse(JSON.parse(line));
55
- checkAndThrowError(response);
56
- yield response;
121
+ const { profilingMeta } = options;
122
+ const shouldProfile = shouldProfileDelegation(request.type, profilingMeta);
123
+ if (!shouldProfile) {
124
+ yield* streamBase(request, rpc, options, profilingMeta);
125
+ return;
126
+ }
127
+ yield* streamProfiled(request, rpc, options, profilingMeta);
128
+ }
129
+ async function* streamBase(request, rpc, options = {}, profilingMeta) {
130
+ try {
131
+ const parsedRequest = requestSchema.parse(request);
132
+ const req = rpc.request(getNextCommandId());
133
+ logger.debug("[delegate-transport] Streaming:", { type: request.type });
134
+ // Propagate per-call disable signal to delegated provider
135
+ const finalRequest = profilingMeta?.enabled === false
136
+ ? { ...parsedRequest, [PROFILING_KEY]: { enabled: false } }
137
+ : parsedRequest;
138
+ req.send(JSON.stringify(finalRequest), "utf-8");
139
+ const responseStream = req.createResponseStream({ encoding: "utf-8" });
140
+ let buffer = "";
141
+ async function* processStream() {
142
+ for await (const chunk of responseStream) {
143
+ yield chunk;
144
+ }
145
+ }
146
+ const streamWithTimeout = withTimeoutStream(processStream(), options?.timeout);
147
+ for await (const chunk of streamWithTimeout) {
148
+ buffer += chunk.toString();
149
+ // Process complete lines (newline-delimited JSON)
150
+ const lines = buffer.split("\n");
151
+ buffer = lines.pop() || ""; // Keep incomplete line in buffer
152
+ for (const line of lines) {
153
+ if (line.trim()) {
154
+ const response = responseSchema.parse(JSON.parse(line));
155
+ checkAndThrowError(response);
156
+ yield response;
157
+ }
57
158
  }
58
159
  }
59
160
  }
161
+ catch (error) {
162
+ cleanupDelegationPeer(options, error);
163
+ throw error;
164
+ }
165
+ }
166
+ async function* streamProfiled(request, rpc, options = {}, profilingMeta) {
167
+ const profileId = profilingMeta?.id ?? generateId();
168
+ const timings = createDelegationStreamTimings(profileId, request.type);
169
+ try {
170
+ if (options.peerKey) {
171
+ flushServerConnectionEvent(options.peerKey);
172
+ }
173
+ const parsedRequest = requestSchema.parse(request);
174
+ const req = rpc.request(getNextCommandId());
175
+ logger.debug("[delegate-transport] Streaming (profiled):", {
176
+ type: request.type,
177
+ });
178
+ const stringifyStart = nowMs();
179
+ const requestWithProfiling = {
180
+ ...parsedRequest,
181
+ [PROFILING_KEY]: { id: profileId },
182
+ };
183
+ const payload = JSON.stringify(requestWithProfiling);
184
+ timings.requestStringifyMs = nowMs() - stringifyStart;
185
+ timings.sendStart = nowMs();
186
+ req.send(payload, "utf-8");
187
+ const responseStream = req.createResponseStream({ encoding: "utf-8" });
188
+ let buffer = "";
189
+ async function* processStream() {
190
+ for await (const chunk of responseStream) {
191
+ yield chunk;
192
+ }
193
+ }
194
+ const streamWithTimeout = withTimeoutStream(processStream(), options?.timeout);
195
+ let lastServerMeta;
196
+ for await (const chunk of streamWithTimeout) {
197
+ buffer += chunk.toString();
198
+ const lines = buffer.split("\n");
199
+ buffer = lines.pop() || "";
200
+ for (const line of lines) {
201
+ if (line.trim()) {
202
+ const rawPayload = JSON.parse(line);
203
+ const response = responseSchema.parse(rawPayload);
204
+ checkAndThrowError(response);
205
+ timings.chunkCount++;
206
+ if (timings.firstChunkAt === undefined) {
207
+ timings.firstChunkAt = nowMs();
208
+ }
209
+ timings.lastChunkAt = nowMs();
210
+ const serverMeta = extractProfilingMeta(rawPayload);
211
+ if (serverMeta?.operation) {
212
+ response[OPERATION_EVENT_KEY] = serverMeta.operation;
213
+ }
214
+ if (serverMeta) {
215
+ lastServerMeta = serverMeta;
216
+ }
217
+ response[DELEGATION_BREAKDOWN_KEY] =
218
+ buildDelegationStreamBreakdown(timings);
219
+ yield response;
220
+ }
221
+ }
222
+ }
223
+ recordDelegationStreamEvents(timings, lastServerMeta);
224
+ }
225
+ catch (error) {
226
+ cleanupDelegationPeer(options, error);
227
+ const base = {
228
+ ts: nowMs(),
229
+ op: timings.requestType,
230
+ kind: "delegation",
231
+ profileId: timings.profileId,
232
+ };
233
+ recordFailure(base, timings.requestStart, error);
234
+ throw error;
235
+ }
60
236
  }
@@ -0,0 +1,2 @@
1
+ export declare function getNextCommandId(): number;
2
+ //# sourceMappingURL=delegate-utils.d.ts.map
@@ -0,0 +1,5 @@
1
+ let commandCounter = 0;
2
+ export function getNextCommandId() {
3
+ commandCounter = (commandCounter + 1) % Number.MAX_SAFE_INTEGER;
4
+ return commandCounter;
5
+ }
@@ -1,32 +1,71 @@
1
- import { requestSchema, normalizeModelType, } from "../../schemas/index.js";
1
+ import { requestSchema, normalizeModelType, PROFILING_KEY, } from "../../schemas/index.js";
2
+ import { nowMs } from "../../profiling/index.js";
2
3
  import { resolveModelConfig } from "../bare/registry/model-config-registry.js";
3
4
  import { sendErrorResponse } from "../error-handlers.js";
4
5
  import { RPCNoDataReceivedError, RPCUnknownRequestTypeError, } from "../../utils/errors-server.js";
5
6
  import { registry } from "./handler-registry.js";
6
7
  import { executeHandler, handleInitConfig, isInitConfigMessage, } from "./handler-utils.js";
8
+ import { createServerProfiler } from "./profiling/index.js";
7
9
  export async function handleRequest(req) {
10
+ let profiler;
11
+ let validationStart = 0;
8
12
  try {
9
13
  const rawData = req.data?.toString();
10
14
  if (!rawData) {
11
15
  throw new RPCNoDataReceivedError();
12
16
  }
17
+ // Timing runs unconditionally since we can't know if client
18
+ // requested profiling until after parsing.
19
+ const parseStart = nowMs();
13
20
  const jsonData = JSON.parse(rawData);
21
+ const jsonParseMs = nowMs() - parseStart;
14
22
  // Handle internal config initialization (bypasses schema)
15
23
  if (isInitConfigMessage(jsonData)) {
16
24
  handleInitConfig(req, jsonData);
17
25
  return;
18
26
  }
19
- const processedData = applyDeviceDefaultsToRequest(jsonData);
27
+ const { data: cleanData, profilingMeta } = extractProfilingMeta(jsonData);
28
+ profiler = createServerProfiler(profilingMeta);
29
+ profiler.markRequestParsed(jsonParseMs);
30
+ validationStart = nowMs();
31
+ const processedData = applyDeviceDefaultsToRequest(cleanData);
20
32
  const request = requestSchema.parse(processedData);
33
+ attachProfilingMetaToRequest(request, profilingMeta);
34
+ profiler.markRequestValidated(nowMs() - validationStart);
35
+ validationStart = 0;
21
36
  const entry = registry[request.type];
22
37
  if (!entry) {
23
38
  throw new RPCUnknownRequestTypeError(request.type);
24
39
  }
25
- await executeHandler(req, request, entry);
40
+ await executeHandler(req, request, entry, profiler);
26
41
  }
27
42
  catch (error) {
28
- sendErrorResponse(req, error);
43
+ if (profiler && validationStart > 0) {
44
+ profiler.markRequestValidated(nowMs() - validationStart);
45
+ }
46
+ sendErrorResponse(req, error, profiler);
47
+ }
48
+ }
49
+ function attachProfilingMetaToRequest(request, profilingMeta) {
50
+ if (!profilingMeta)
51
+ return;
52
+ Object.defineProperty(request, PROFILING_KEY, {
53
+ value: profilingMeta,
54
+ enumerable: false,
55
+ configurable: true,
56
+ writable: false,
57
+ });
58
+ }
59
+ function extractProfilingMeta(data) {
60
+ if (!data || typeof data !== "object" || !(PROFILING_KEY in data)) {
61
+ return { data, profilingMeta: undefined };
29
62
  }
63
+ const obj = data;
64
+ const { [PROFILING_KEY]: meta, ...rest } = obj;
65
+ return {
66
+ data: rest,
67
+ profilingMeta: meta,
68
+ };
30
69
  }
31
70
  /**
32
71
  * Apply device-specific config defaults to loadModel requests before schema parsing.
@@ -6,6 +6,7 @@ import { handleLoadModelDelegated } from "./handlers/load-model-delegated.js";
6
6
  import { handleCompletionStreamDelegated } from "./handlers/completion-stream-delegated.js";
7
7
  import { getModelEntry } from "../bare/registry/model-registry.js";
8
8
  import { handleUnloadModel } from "./handlers/unload-model.js";
9
+ import { handleUnloadModelDelegated } from "./handlers/unload-model-delegated.js";
9
10
  import { handleTranscribeStream } from "./handlers/transcribe-stream.js";
10
11
  import { handleEmbed } from "./handlers/embed.js";
11
12
  import { handleTranslate } from "./handlers/translate.js";
@@ -18,7 +19,9 @@ import { handleDeleteCache } from "./handlers/delete-cache.js";
18
19
  import { handleTextToSpeech } from "./handlers/text-to-speech.js";
19
20
  import { handleGetModelInfo } from "./handlers/get-model-info.js";
20
21
  import { handleOCRStream } from "./handlers/ocr-stream.js";
21
- import { handlePing } from "./handlers/ping.js";
22
+ import { handleHeartbeat } from "./handlers/heartbeat.js";
23
+ import { handleHeartbeatDelegated } from "./handlers/heartbeat-delegated.js";
24
+ import { handleCancelDelegated } from "./handlers/cancel-delegated.js";
22
25
  import { handlePluginInvoke, handlePluginInvokeStream, } from "./handlers/plugin-invoke.js";
23
26
  import { handleModelRegistryList, handleModelRegistrySearch, handleModelRegistryGetModel, } from "./handlers/registry.js";
24
27
  function ragSupportsProgress(request) {
@@ -32,12 +35,38 @@ function isModelDelegated(request) {
32
35
  const entry = getModelEntry(request.modelId);
33
36
  return entry?.isDelegated ?? false;
34
37
  }
38
+ function isCancelDelegated(request) {
39
+ if (request.type !== "cancel")
40
+ return false;
41
+ if (request.operation === "inference") {
42
+ return isModelDelegated(request);
43
+ }
44
+ if (request.operation === "downloadAsset") {
45
+ return !!request.delegate;
46
+ }
47
+ return false;
48
+ }
35
49
  export const registry = {
36
50
  // Simple Reply handlers
37
- ping: { type: "reply", handler: handlePing },
38
- unloadModel: { type: "reply", handler: handleUnloadModel },
51
+ heartbeat: {
52
+ type: "reply",
53
+ handler: handleHeartbeat,
54
+ delegatedHandler: handleHeartbeatDelegated,
55
+ isDelegated: (r) => r.type === "heartbeat" && !!r.delegate,
56
+ },
57
+ unloadModel: {
58
+ type: "reply",
59
+ handler: handleUnloadModel,
60
+ delegatedHandler: handleUnloadModelDelegated,
61
+ isDelegated: isModelDelegated,
62
+ },
39
63
  embed: { type: "reply", handler: handleEmbed },
40
- cancel: { type: "reply", handler: cancelHandler },
64
+ cancel: {
65
+ type: "reply",
66
+ handler: cancelHandler,
67
+ delegatedHandler: handleCancelDelegated,
68
+ isDelegated: isCancelDelegated,
69
+ },
41
70
  provide: { type: "reply", handler: provideHandler },
42
71
  stopProvide: { type: "reply", handler: stopProvideHandler },
43
72
  deleteCache: { type: "reply", handler: handleDeleteCache },
@@ -1,8 +1,9 @@
1
1
  import { type QvacConfig, type Request, type Response, type RuntimeContext } from "../../schemas/index";
2
2
  import type RPC from "bare-rpc";
3
- type ReplyHandler = (request: any) => Promise<Response> | Response;
4
- type StreamHandler = (request: any) => AsyncGenerator<Response>;
5
- type ProgressHandler = (request: any, onProgress?: (update: Response) => void) => Promise<Response>;
3
+ import { type ServerProfiler } from "./profiling/index";
4
+ type ReplyHandler = (request: any, ...args: any[]) => Promise<Response> | Response;
5
+ type StreamHandler = (request: any, ...args: any[]) => AsyncGenerator<Response>;
6
+ type ProgressHandler = (request: any, ...args: any[]) => Promise<Response>;
6
7
  export type HandlerEntry = {
7
8
  type: "reply" | "stream";
8
9
  handler: ReplyHandler | StreamHandler | ProgressHandler;
@@ -10,7 +11,7 @@ export type HandlerEntry = {
10
11
  isDelegated?: (request: Request) => boolean;
11
12
  supportsProgress?: boolean | ((request: Request) => boolean);
12
13
  };
13
- export declare function executeHandler(req: RPC.IncomingRequest, request: Request, entry: HandlerEntry): Promise<void>;
14
+ export declare function executeHandler(req: RPC.IncomingRequest, request: Request, entry: HandlerEntry, profiler: ServerProfiler): Promise<void>;
14
15
  type InitConfigMessage = {
15
16
  type: "__init_config";
16
17
  config: QvacConfig;
@@ -1,65 +1,148 @@
1
- import { responseSchema, } from "../../schemas/index.js";
1
+ import { PROFILING_KEY, } from "../../schemas/index.js";
2
2
  import { sendErrorResponse, sendStreamErrorResponse, } from "../error-handlers.js";
3
3
  import { setSDKConfig } from "../bare/registry/config-registry.js";
4
4
  import { setRuntimeContext } from "../bare/registry/runtime-context-registry.js";
5
- function writeToStream(stream, response) {
6
- stream.write(JSON.stringify(responseSchema.parse(response)) + "\n", "utf-8");
5
+ import {} from "./profiling/index.js";
6
+ import { nowMs } from "../../profiling/clock.js";
7
+ function getProfilingMetaFromRequest(request) {
8
+ if (PROFILING_KEY in request) {
9
+ return request[PROFILING_KEY];
10
+ }
11
+ return undefined;
7
12
  }
8
- async function executeReplyHandler(req, request, handler) {
13
+ async function executeReplyHandler(req, request, handler, profiler, isDelegated) {
14
+ profiler.startHandler();
9
15
  try {
10
- const response = await handler(request);
11
- req.reply(JSON.stringify(responseSchema.parse(response)), "utf-8");
16
+ let response;
17
+ if (isDelegated) {
18
+ const profilingMeta = getProfilingMetaFromRequest(request);
19
+ response = await handler(request, profilingMeta ? { profilingMeta } : undefined);
20
+ }
21
+ else {
22
+ response = await handler(request);
23
+ }
24
+ profiler.endHandler();
25
+ req.reply(profiler.serialize(response, true), "utf-8");
12
26
  }
13
27
  catch (error) {
14
- sendErrorResponse(req, error);
28
+ profiler.endHandler();
29
+ sendErrorResponse(req, error, profiler);
15
30
  }
16
31
  }
17
- async function executeStreamHandler(req, request, handler) {
32
+ async function executeStreamHandler(req, request, handler, profiler, isDelegated) {
18
33
  const stream = req.createResponseStream();
34
+ profiler.startHandler();
19
35
  try {
20
- for await (const response of handler(request)) {
21
- stream.write(JSON.stringify(responseSchema.parse(response)) + "\n", "utf-8");
36
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
37
+ let generator;
38
+ if (isDelegated) {
39
+ const profilingMeta = getProfilingMetaFromRequest(request);
40
+ generator = handler(request, profilingMeta ? { profilingMeta } : undefined);
41
+ }
42
+ else {
43
+ generator = handler(request);
44
+ }
45
+ for await (const response of generator) {
46
+ stream.write(profiler.serialize(response, false) + "\n", "utf-8");
47
+ }
48
+ profiler.endHandler();
49
+ const trailer = profiler.serialize();
50
+ if (trailer) {
51
+ stream.write(trailer + "\n", "utf-8");
22
52
  }
23
53
  stream.end();
24
54
  }
25
55
  catch (error) {
26
- sendStreamErrorResponse(stream, error);
56
+ profiler.endHandler();
57
+ sendStreamErrorResponse(stream, error, profiler);
27
58
  }
28
59
  }
29
- async function executeProgressHandler(req, request, handler) {
60
+ const PROGRESS_THROTTLE_MS = 150;
61
+ async function executeProgressHandler(req, request, handler, profiler, isDelegated) {
30
62
  const stream = req.createResponseStream();
63
+ profiler.startHandler();
64
+ let lastProgressWrite = 0;
65
+ let pendingUpdate = null;
66
+ let flushTimer = null;
67
+ const writeProgress = (update) => {
68
+ stream.write(profiler.serialize(update, false) + "\n", "utf-8");
69
+ };
70
+ const progressCallback = (update) => {
71
+ const now = nowMs();
72
+ if (now - lastProgressWrite >= PROGRESS_THROTTLE_MS) {
73
+ lastProgressWrite = now;
74
+ pendingUpdate = null;
75
+ writeProgress(update);
76
+ }
77
+ else {
78
+ pendingUpdate = update;
79
+ if (!flushTimer) {
80
+ flushTimer = setTimeout(() => {
81
+ flushTimer = null;
82
+ if (pendingUpdate) {
83
+ lastProgressWrite = nowMs();
84
+ writeProgress(pendingUpdate);
85
+ pendingUpdate = null;
86
+ }
87
+ }, PROGRESS_THROTTLE_MS - (now - lastProgressWrite));
88
+ }
89
+ }
90
+ };
31
91
  try {
32
- const response = await handler(request, (update) => writeToStream(stream, update));
33
- writeToStream(stream, response);
92
+ let response;
93
+ if (isDelegated) {
94
+ const profilingMeta = getProfilingMetaFromRequest(request);
95
+ const options = { progressCallback };
96
+ if (profilingMeta) {
97
+ options.profilingMeta = profilingMeta;
98
+ }
99
+ response = await handler(request, options);
100
+ }
101
+ else {
102
+ response = await handler(request, progressCallback);
103
+ }
104
+ if (flushTimer) {
105
+ clearTimeout(flushTimer);
106
+ flushTimer = null;
107
+ }
108
+ if (pendingUpdate) {
109
+ writeProgress(pendingUpdate);
110
+ pendingUpdate = null;
111
+ }
112
+ profiler.endHandler();
113
+ stream.write(profiler.serialize(response, true) + "\n", "utf-8");
34
114
  stream.end();
35
115
  }
36
116
  catch (error) {
37
- sendStreamErrorResponse(stream, error);
117
+ if (flushTimer) {
118
+ clearTimeout(flushTimer);
119
+ flushTimer = null;
120
+ }
121
+ if (pendingUpdate) {
122
+ writeProgress(pendingUpdate);
123
+ pendingUpdate = null;
124
+ }
125
+ profiler.endHandler();
126
+ sendStreamErrorResponse(stream, error, profiler);
38
127
  }
39
128
  }
40
129
  // Unified handler executor with delegation and progress support
41
- export async function executeHandler(req, request, entry) {
42
- const handler = entry.delegatedHandler && entry.isDelegated?.(request)
43
- ? entry.delegatedHandler
44
- : entry.handler;
130
+ export async function executeHandler(req, request, entry, profiler) {
131
+ const isDelegated = !!(entry.delegatedHandler && entry.isDelegated?.(request));
132
+ const handler = isDelegated ? entry.delegatedHandler : entry.handler;
45
133
  const wantsProgress = "withProgress" in request &&
46
134
  request.withProgress &&
47
135
  (typeof entry.supportsProgress === "function"
48
136
  ? entry.supportsProgress(request)
49
137
  : entry.supportsProgress);
50
- try {
51
- if (entry.type === "stream") {
52
- await executeStreamHandler(req, request, handler);
53
- }
54
- else if (wantsProgress) {
55
- await executeProgressHandler(req, request, handler);
56
- }
57
- else {
58
- await executeReplyHandler(req, request, handler);
59
- }
138
+ if (entry.type === "stream") {
139
+ await executeStreamHandler(req, request, handler, profiler, isDelegated);
60
140
  }
61
- catch (error) {
62
- sendErrorResponse(req, error);
141
+ else if (wantsProgress) {
142
+ await executeProgressHandler(req, request, handler, profiler, isDelegated);
143
+ }
144
+ else {
145
+ await executeReplyHandler(req, request, handler, profiler, isDelegated);
63
146
  }
64
147
  }
65
148
  export function isInitConfigMessage(data) {
@@ -0,0 +1,4 @@
1
+ import type { CancelRequest, CancelResponse } from "../../../schemas/cancel";
2
+ import type { DelegatedHandlerOptions } from "../profiling/index";
3
+ export declare function handleCancelDelegated(request: CancelRequest, options?: DelegatedHandlerOptions): Promise<CancelResponse>;
4
+ //# sourceMappingURL=cancel-delegated.d.ts.map