@juspay/neurolink 12.11.3 → 12.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/CHANGELOG.md +3 -4
  2. package/dist/browser/neurolink.min.js +533 -581
  3. package/dist/constants/enums.d.ts +13 -0
  4. package/dist/constants/enums.js +14 -0
  5. package/dist/core/baseProvider.d.ts +71 -3
  6. package/dist/core/baseProvider.js +152 -44
  7. package/dist/core/modules/GenerationHandler.d.ts +22 -24
  8. package/dist/core/modules/GenerationHandler.js +28 -463
  9. package/dist/core/nativeGenerateLoop.d.ts +35 -0
  10. package/dist/core/nativeGenerateLoop.js +261 -0
  11. package/dist/files/fileTools.d.ts +5 -5
  12. package/dist/index.d.ts +1 -0
  13. package/dist/index.js +4 -0
  14. package/dist/middleware/builtin/guardrails.d.ts +0 -5
  15. package/dist/middleware/builtin/guardrails.js +33 -5
  16. package/dist/middleware/factory.js +1 -1
  17. package/dist/middleware/wrapLanguageModel.d.ts +18 -0
  18. package/dist/middleware/wrapLanguageModel.js +53 -0
  19. package/dist/processors/media/AudioProcessor.js +46 -11
  20. package/dist/providers/amazonSagemaker.d.ts +17 -1
  21. package/dist/providers/amazonSagemaker.js +110 -0
  22. package/dist/providers/anthropic/client.d.ts +11 -0
  23. package/dist/providers/anthropic/client.js +148 -1
  24. package/dist/providers/catalog/index.generated.d.ts +1 -1
  25. package/dist/providers/catalog/index.generated.js +3 -0
  26. package/dist/providers/catalog/loader.js +1 -0
  27. package/dist/providers/catalog/mancer.json +192 -0
  28. package/dist/providers/configuredOpenAICompat.d.ts +11 -0
  29. package/dist/providers/configuredOpenAICompat.js +16 -0
  30. package/dist/providers/googleVertex/client.d.ts +0 -9
  31. package/dist/providers/googleVertex/client.js +0 -33
  32. package/dist/providers/openaiChatCompletionsBase.d.ts +21 -1
  33. package/dist/providers/openaiChatCompletionsBase.js +178 -0
  34. package/dist/providers/providerTypeUtils.d.ts +1 -2
  35. package/dist/providers/providerTypeUtils.js +5 -1
  36. package/dist/types/aiCompat.d.ts +485 -0
  37. package/dist/types/aiCompat.js +17 -0
  38. package/dist/types/conversation.d.ts +1 -1
  39. package/dist/types/generate.d.ts +52 -0
  40. package/dist/types/middleware.d.ts +3 -6
  41. package/dist/types/providerCatalog.generated.d.ts +2 -2
  42. package/dist/types/providers.d.ts +14 -1
  43. package/dist/types/tools.d.ts +2 -2
  44. package/dist/utils/generationErrors.d.ts +78 -6
  45. package/dist/utils/generationErrors.js +114 -6
  46. package/dist/utils/nativeSingleShot.d.ts +3 -0
  47. package/dist/utils/nativeSingleShot.js +83 -0
  48. package/dist/utils/tool.d.ts +30 -5
  49. package/dist/utils/tool.js +43 -5
  50. package/package.json +3 -6
  51. package/dist/utils/generation.d.ts +0 -8
  52. package/dist/utils/generation.js +0 -8
@@ -0,0 +1,261 @@
1
+ /**
2
+ * The multi-step tool loop that the ai package's `generateText` used to supply.
3
+ *
4
+ * It deliberately loops over a provider's own `doGenerate` rather than over any
5
+ * streaming machinery. That is the whole lesson of the reverted first attempt:
6
+ * `doGenerate` is where the JSON-versus-SSE wire choice lives, along with the
7
+ * 400 retry, the context-overflow refit and the provider's own structured-output
8
+ * handling. Looping around the streaming path instead silently changed
9
+ * generate() to send `stream: true` and broke ten providers against a
10
+ * non-streaming body.
11
+ *
12
+ * Every provider whose delegating model exposes a v3-shaped `doGenerate` can
13
+ * share this: the v3 result shape (`content` parts, `finishReason`, `usage`) is
14
+ * the same across Anthropic, the OpenAI-compatible family and SageMaker.
15
+ */
16
+ import { guardToolExecutor } from "./toolExecutionGuards.js";
17
+ /**
18
+ * Narrow a model handle to the delegating shape this loop drives.
19
+ * `LanguageModel` is a union that includes a bare string id, and a double
20
+ * assertion through unknown is banned by Critical Rule 14.
21
+ */
22
+ export const hasNativeDoGenerate = (value) => typeof value === "object" &&
23
+ value !== null &&
24
+ typeof value.doGenerate === "function";
25
+ const asParts = (value) => Array.isArray(value) ? value : [];
26
+ const readTotal = (value) => typeof value === "object" &&
27
+ value !== null &&
28
+ typeof value.total === "number"
29
+ ? value.total
30
+ : 0;
31
+ /**
32
+ * Parse a tool call's arguments, distinguishing "no arguments" from "the model
33
+ * emitted something that is not JSON". Silently substituting `{}` for the
34
+ * second case ran the tool with empty input and reported success, so a
35
+ * malformed call looked identical to a legitimate no-arg one.
36
+ */
37
+ const parseToolInput = (raw) => {
38
+ if (typeof raw !== "string") {
39
+ return { input: raw ?? {} };
40
+ }
41
+ if (raw.trim() === "") {
42
+ return { input: {} };
43
+ }
44
+ try {
45
+ return { input: JSON.parse(raw) };
46
+ }
47
+ catch {
48
+ return { input: {}, error: "arguments were not valid JSON" };
49
+ }
50
+ };
51
+ /**
52
+ * Validate parsed input against the tool's own schema when it exposes one.
53
+ *
54
+ * `generateText` validated tool input before dispatch; the native loop did
55
+ * not, so a call whose shape the tool rejects reached `execute` and failed
56
+ * inside user code — or worse, did not fail. A Zod schema is detected by
57
+ * `safeParse`; anything else is passed through, since a JSON Schema needs a
58
+ * validator this loop has no business carrying.
59
+ */
60
+ const validateToolInput = (tool, input) => {
61
+ const schema = tool?.inputSchema;
62
+ if (typeof schema?.safeParse !== "function") {
63
+ return undefined;
64
+ }
65
+ const result = schema.safeParse(input);
66
+ if (result.success) {
67
+ return undefined;
68
+ }
69
+ const detail = result.error instanceof Error ? result.error.message : "schema mismatch";
70
+ return `input did not match the tool's schema: ${detail}`;
71
+ };
72
+ /**
73
+ * Spell a JSON Schema into the conversation's system turn.
74
+ *
75
+ * The structured-output fallback for vendors that reject or ignore
76
+ * `response_format`. Merged into an existing trailing system message rather
77
+ * than appended as a second one: several self-hosted OpenAI-compatible stacks
78
+ * honour only the first system message, so a second would be dropped and the
79
+ * fallback would silently do nothing.
80
+ */
81
+ export const appendJsonSchemaInstruction = (conversation, schema) => {
82
+ const instruction = "When you give your final answer, respond with only a single JSON object " +
83
+ "that conforms to the following JSON Schema. No prose before or after it, " +
84
+ `and no markdown code fence. JSON Schema: ${JSON.stringify(schema)}`;
85
+ const lastSystemIndex = conversation.reduce((found, message, index) => (message.role === "system" ? index : found), -1);
86
+ if (lastSystemIndex === -1) {
87
+ return [{ role: "system", content: instruction }, ...conversation];
88
+ }
89
+ const existing = conversation[lastSystemIndex];
90
+ const content = typeof existing.content === "string"
91
+ ? `${existing.content}\n\n${instruction}`
92
+ : instruction;
93
+ return conversation.map((message, index) => index === lastSystemIndex ? { ...message, content } : message);
94
+ };
95
+ export async function runNativeGenerateLoop(args, toolExecutionSummaries) {
96
+ const toolsUsed = [];
97
+ let text = "";
98
+ let finishReason = "stop";
99
+ let rawFinishReason;
100
+ let inputTokens = 0;
101
+ let outputTokens = 0;
102
+ let cacheReadTokens = 0;
103
+ let cacheWriteTokens = 0;
104
+ let steps = 0;
105
+ // One bounded recovery re-ask per turn; see the empty-tool-calls branch.
106
+ let reasked = false;
107
+ const hasTools = Boolean(args.tools && args.tools.length > 0);
108
+ for (let step = 0; step < args.maxSteps; step++) {
109
+ steps = step + 1;
110
+ const res = await args.runStep(() => args.doGenerate({
111
+ prompt: args.conversation,
112
+ ...(args.tools && args.tools.length > 0 ? { tools: args.tools } : {}),
113
+ // The v3 call option is an OBJECT — `{ type: "none" }`. Passing the
114
+ // bare string "none" type-checks against `unknown` and is then
115
+ // dropped by every converter that switches on `choice.type`, so the
116
+ // re-ask silently went out unchanged. Caught by the stand-in asserting
117
+ // the wire body, not by any live provider.
118
+ ...(reasked
119
+ ? { toolChoice: { type: "none" } }
120
+ : args.toolChoice !== undefined
121
+ ? { toolChoice: args.toolChoice }
122
+ : {}),
123
+ ...(args.responseFormat ? { responseFormat: args.responseFormat } : {}),
124
+ ...(args.providerOptions
125
+ ? { providerOptions: args.providerOptions }
126
+ : {}),
127
+ ...(args.maxOutputTokens
128
+ ? { maxOutputTokens: args.maxOutputTokens }
129
+ : {}),
130
+ ...(args.temperature !== undefined
131
+ ? { temperature: args.temperature }
132
+ : {}),
133
+ ...(args.abortSignal ? { abortSignal: args.abortSignal } : {}),
134
+ }));
135
+ const parts = asParts(res.content);
136
+ // Each step REPLACES the text rather than appending: the final step's
137
+ // answer is the turn's answer, matching what generateText reported.
138
+ text = parts
139
+ .filter((p) => p.type === "text" && typeof p.text === "string")
140
+ .map((p) => p.text)
141
+ .join("");
142
+ const fr = res.finishReason;
143
+ if (typeof fr === "string") {
144
+ finishReason = fr;
145
+ }
146
+ else if (typeof fr === "object" && fr !== null) {
147
+ const shaped = fr;
148
+ finishReason = shaped.unified ?? finishReason;
149
+ rawFinishReason = shaped.raw ?? rawFinishReason;
150
+ }
151
+ const usage = res.usage;
152
+ inputTokens += readTotal(usage?.inputTokens);
153
+ outputTokens += readTotal(usage?.outputTokens);
154
+ const inShaped = usage?.inputTokens;
155
+ cacheReadTokens += inShaped?.cacheRead ?? 0;
156
+ cacheWriteTokens += inShaped?.cacheWrite ?? 0;
157
+ const calls = parts.filter((p) => p.type === "tool-call");
158
+ if (calls.length === 0) {
159
+ // io.net's Llama endpoint ends a tool loop on `finish_reason:
160
+ // tool_calls` carrying neither a tool call nor any text: the model's
161
+ // JSON-shaped answer trips the vendor's tool-call parser, which drops it
162
+ // and reports `content: null` with no `tool_calls`. There is nothing to
163
+ // execute, so the loop would stop and hand the caller an empty turn even
164
+ // though the tool ran. Replaying the request once with
165
+ // `toolChoice: "none"` returns the answer.
166
+ //
167
+ // Ported from GenerationHandler.recoverEmptyToolCallsFinish, which runs
168
+ // this on the ai-package path. That path is unreachable for every
169
+ // provider driven by this loop — io.net among them, since it is a
170
+ // Tier-2 catalog provider on the OpenAI-compatible base — so without
171
+ // this the recovery would simply not happen for the provider it was
172
+ // written for.
173
+ const emptyToolCallsFinish = finishReason === "tool-calls" &&
174
+ text.trim() === "" &&
175
+ hasTools &&
176
+ !reasked &&
177
+ step + 1 < args.maxSteps;
178
+ if (emptyToolCallsFinish) {
179
+ reasked = true;
180
+ args.conversation.push({ role: "assistant", content: parts });
181
+ continue;
182
+ }
183
+ break;
184
+ }
185
+ // Tool turns go back in the message-builder shape each provider's own
186
+ // conversion already round-trips: an assistant message of tool-call parts,
187
+ // then one tool message of tool-result parts.
188
+ args.conversation.push({ role: "assistant", content: parts });
189
+ const resultParts = [];
190
+ for (const call of calls) {
191
+ const name = String(call.toolName ?? "");
192
+ const id = String(call.toolCallId ?? "");
193
+ const startTime = new Date();
194
+ const parsed = parseToolInput(call.input);
195
+ const input = parsed.input;
196
+ const tool = args.toolsRecord[name];
197
+ let output;
198
+ let failure;
199
+ const rejection = parsed.error ??
200
+ (typeof tool?.execute === "function"
201
+ ? validateToolInput(tool, input)
202
+ : undefined);
203
+ if (typeof tool?.execute !== "function") {
204
+ failure = `Tool not found: ${name}`;
205
+ output = { error: failure };
206
+ }
207
+ else if (rejection) {
208
+ // An error tool-result rather than a throw: the model gets to see what
209
+ // was wrong and correct it on the next step, which is what the SDK's
210
+ // own validation did.
211
+ failure = `Tool ${name}: ${rejection}`;
212
+ output = { error: failure };
213
+ }
214
+ else {
215
+ try {
216
+ // The turn's abort signal and the per-tool cap have to reach the
217
+ // tool, or a wedged tool parks the loop inside `await execute` and
218
+ // outlives the deadline that withTurnTimeout composed. Reuses the
219
+ // guard every other native loop already applies.
220
+ const guarded = guardToolExecutor(name, tool.execute, {
221
+ ...(args.abortSignal ? { abortSignal: args.abortSignal } : {}),
222
+ ...(args.toolTimeoutMs !== undefined
223
+ ? { toolTimeoutMs: args.toolTimeoutMs }
224
+ : {}),
225
+ });
226
+ output = await guarded(input, {
227
+ toolCallId: id,
228
+ messages: [],
229
+ ...(args.abortSignal ? { abortSignal: args.abortSignal } : {}),
230
+ });
231
+ toolsUsed.push(name);
232
+ }
233
+ catch (err) {
234
+ failure = err instanceof Error ? err.message : String(err);
235
+ output = { error: failure };
236
+ }
237
+ }
238
+ toolExecutionSummaries.push({
239
+ toolCallId: id,
240
+ toolName: name,
241
+ input,
242
+ ...(failure ? { error: failure } : { output }),
243
+ startTime,
244
+ endTime: new Date(),
245
+ });
246
+ resultParts.push({ type: "tool-result", toolCallId: id, output });
247
+ }
248
+ args.conversation.push({ role: "tool", content: resultParts });
249
+ }
250
+ return {
251
+ text,
252
+ finishReason,
253
+ ...(rawFinishReason ? { rawFinishReason } : {}),
254
+ inputTokens,
255
+ outputTokens,
256
+ cacheReadTokens,
257
+ cacheWriteTokens,
258
+ toolsUsed,
259
+ steps,
260
+ };
261
+ }
@@ -18,7 +18,7 @@ import type { FileReferenceRegistry } from "./fileReferenceRegistry.js";
18
18
  * ```
19
19
  */
20
20
  export declare function createFileTools(registry: FileReferenceRegistry): {
21
- list_attached_files: import("ai").Tool<Record<string, never>, {
21
+ list_attached_files: import("../index.js").Tool<Record<string, never>, {
22
22
  success: boolean;
23
23
  message: string;
24
24
  fileCount: number;
@@ -53,7 +53,7 @@ export declare function createFileTools(registry: FileReferenceRegistry): {
53
53
  files?: undefined;
54
54
  formatted?: undefined;
55
55
  }>;
56
- read_file_section: import("ai").Tool<{
56
+ read_file_section: import("../index.js").Tool<{
57
57
  file_id: string;
58
58
  start_line: number;
59
59
  token_budget: number;
@@ -79,7 +79,7 @@ export declare function createFileTools(registry: FileReferenceRegistry): {
79
79
  guidance: string | undefined;
80
80
  error?: undefined;
81
81
  }>;
82
- search_in_file: import("ai").Tool<{
82
+ search_in_file: import("../index.js").Tool<{
83
83
  file_id: string;
84
84
  pattern: string;
85
85
  max_matches: number;
@@ -107,7 +107,7 @@ export declare function createFileTools(registry: FileReferenceRegistry): {
107
107
  }[];
108
108
  error?: undefined;
109
109
  }>;
110
- get_file_preview: import("ai").Tool<{
110
+ get_file_preview: import("../index.js").Tool<{
111
111
  file_id: string;
112
112
  }, {
113
113
  success: boolean;
@@ -140,7 +140,7 @@ export declare function createFileTools(registry: FileReferenceRegistry): {
140
140
  hasSummary: boolean;
141
141
  error?: undefined;
142
142
  }>;
143
- extract_file_content: import("ai").Tool<{
143
+ extract_file_content: import("../index.js").Tool<{
144
144
  file_id: string;
145
145
  start_time?: number | undefined;
146
146
  end_time?: number | undefined;
package/dist/index.d.ts CHANGED
@@ -82,6 +82,7 @@ export { initializeOpenTelemetry, shutdownOpenTelemetry, flushOpenTelemetry, get
82
82
  export { clearAnalyticsMetrics, createAnalyticsMiddleware, getAnalyticsMetrics, } from "./middleware/builtin/analytics.js";
83
83
  export { createLifecycleMiddleware } from "./middleware/builtin/lifecycle.js";
84
84
  export { MiddlewareFactory } from "./middleware/factory.js";
85
+ export { tool, jsonSchema, stepCountIs } from "./utils/tool.js";
85
86
  export { ExporterRegistry } from "./observability/exporterRegistry.js";
86
87
  export { NoOpExporter } from "./observability/exporters/baseExporter.js";
87
88
  export { getMetricsAggregator, MetricsAggregator, resetMetricsAggregator, } from "./observability/metricsAggregator.js";
package/dist/index.js CHANGED
@@ -165,6 +165,10 @@ runWithCurrentLangfuseContext, };
165
165
  export { clearAnalyticsMetrics, createAnalyticsMiddleware, getAnalyticsMetrics, } from "./middleware/builtin/analytics.js";
166
166
  export { createLifecycleMiddleware } from "./middleware/builtin/lifecycle.js";
167
167
  export { MiddlewareFactory } from "./middleware/factory.js";
168
+ // Tool + schema helpers. Previously a consumer reached for `tool()` and
169
+ // `jsonSchema()` from the ai package to build tools for generate({tools}).
170
+ // That package is no longer a dependency, so the equivalents ship here.
171
+ export { tool, jsonSchema, stepCountIs } from "./utils/tool.js";
168
172
  export { ExporterRegistry } from "./observability/exporterRegistry.js";
169
173
  export { NoOpExporter } from "./observability/exporters/baseExporter.js";
170
174
  // Observability modules and types
@@ -1,7 +1,2 @@
1
1
  import type { NeuroLinkMiddleware, GuardrailsMiddlewareConfig } from "../../types/index.js";
2
- /**
3
- * Create Guardrails AI middleware for content filtering and policy enforcement
4
- * @param config Configuration for the guardrails middleware
5
- * @returns NeuroLink middleware instance
6
- */
7
2
  export declare function createGuardrailsMiddleware(config?: GuardrailsMiddlewareConfig): NeuroLinkMiddleware;
@@ -1,11 +1,36 @@
1
1
  import { createBlockedResponse, createBlockedStream, applyContentFiltering, handlePrecallGuardrails, } from "../utils/guardrailsUtils.js";
2
2
  import { logger } from "../../utils/logger.js";
3
- import { generateText } from "../../utils/generation.js";
3
+ import { generateOnceNative } from "../../utils/nativeSingleShot.js";
4
4
  /**
5
5
  * Create Guardrails AI middleware for content filtering and policy enforcement
6
6
  * @param config Configuration for the guardrails middleware
7
7
  * @returns NeuroLink middleware instance
8
8
  */
9
+ /**
10
+ * Turn whatever a caller put in `filterModel` into a model handle.
11
+ *
12
+ * A handle is used as-is. A string is resolved through NeuroLink's own
13
+ * provider factory, accepting either "provider:model" or a bare model id.
14
+ * Imported lazily so the middleware module does not pull the provider factory
15
+ * into every bundle that merely registers guardrails.
16
+ */
17
+ async function resolveFilterModel(filterModel) {
18
+ if (typeof filterModel !== "string") {
19
+ return filterModel;
20
+ }
21
+ const [maybeProvider, ...rest] = filterModel.split(":");
22
+ const hasProvider = rest.length > 0;
23
+ const { ProviderFactory } = await import("../../factories/providerFactory.js");
24
+ const provider = await ProviderFactory.createProvider(hasProvider ? maybeProvider : undefined, hasProvider ? rest.join(":") : filterModel);
25
+ // `getModel()` is BaseProvider's sanctioned public handle but is not on the
26
+ // narrower `AIProvider` type, so narrow at the boundary rather than assert
27
+ // through it (Critical Rule 14).
28
+ const handle = provider;
29
+ if (typeof handle.getModel !== "function") {
30
+ throw new Error(`guardrails: provider for "${filterModel}" exposes no model handle`);
31
+ }
32
+ return handle.getModel();
33
+ }
9
34
  export function createGuardrailsMiddleware(config = {}) {
10
35
  const metadata = {
11
36
  id: "guardrails",
@@ -44,10 +69,13 @@ export function createGuardrailsMiddleware(config = {}) {
44
69
  logger.debug(`[GuardrailsMiddleware] Invoking model-based filter.`);
45
70
  try {
46
71
  const filterPrompt = `Is the following text safe? Respond with only "safe" or "unsafe".\n\nText: "${result.text}"`;
47
- const { text: filterResponse } = await generateText({
48
- model: config.modelFilter.filterModel,
49
- prompt: filterPrompt,
50
- });
72
+ // `ModelFilterConfig.filterModel` is typed `LanguageModel`, which
73
+ // admits a bare model id, and the documented examples used one.
74
+ // `generateOnceNative` needs a handle exposing doGenerate, so a
75
+ // string threw, the catch below logged it, and the turn returned
76
+ // unfiltered — a security control that silently did nothing.
77
+ const filterModel = await resolveFilterModel(config.modelFilter.filterModel);
78
+ const { text: filterResponse } = await generateOnceNative(filterModel, { prompt: filterPrompt });
51
79
  if (filterResponse.toLowerCase().trim() === "unsafe") {
52
80
  logger.warn(`[GuardrailsMiddleware] Model-based filter flagged content as unsafe.`);
53
81
  result = { ...result, text: "<REDACTED BY AI GUARDRAIL>" };
@@ -4,7 +4,7 @@ import { createGuardrailsMiddleware } from "./builtin/guardrails.js";
4
4
  import { createAutoEvaluationMiddleware } from "./builtin/autoEvaluation.js";
5
5
  import { createLifecycleMiddleware } from "./builtin/lifecycle.js";
6
6
  import { logger } from "../utils/logger.js";
7
- import { wrapLanguageModel } from "../utils/generation.js";
7
+ import { wrapLanguageModel } from "./wrapLanguageModel.js";
8
8
  /**
9
9
  * Middleware factory for creating and applying middleware chains.
10
10
  * Each factory instance manages its own registry and configuration.
@@ -0,0 +1,18 @@
1
+ /**
2
+ * Local `wrapLanguageModel`.
3
+ *
4
+ * Upstream is a reduce over the middleware array that returns a model whose
5
+ * `doGenerate` / `doStream` route through `transformParams` and the optional
6
+ * `wrapGenerate` / `wrapStream` hooks. Reproduced here so the middleware
7
+ * factory no longer needs the ai package.
8
+ *
9
+ * Worth recording: `wrapStream` does not currently run in this codebase. Every
10
+ * streaming path is native and bypasses the wrapped model entirely, so only
11
+ * `wrapGenerate` is reachable. That is a pre-existing gap, not one this
12
+ * introduced.
13
+ */
14
+ import type { LanguageModelV3, LanguageModelV3Middleware } from "../types/index.js";
15
+ export declare const wrapLanguageModel: ({ model, middleware, }: {
16
+ model: LanguageModelV3;
17
+ middleware: LanguageModelV3Middleware | LanguageModelV3Middleware[];
18
+ }) => LanguageModelV3;
@@ -0,0 +1,53 @@
1
+ /**
2
+ * Local `wrapLanguageModel`.
3
+ *
4
+ * Upstream is a reduce over the middleware array that returns a model whose
5
+ * `doGenerate` / `doStream` route through `transformParams` and the optional
6
+ * `wrapGenerate` / `wrapStream` hooks. Reproduced here so the middleware
7
+ * factory no longer needs the ai package.
8
+ *
9
+ * Worth recording: `wrapStream` does not currently run in this codebase. Every
10
+ * streaming path is native and bypasses the wrapped model entirely, so only
11
+ * `wrapGenerate` is reachable. That is a pre-existing gap, not one this
12
+ * introduced.
13
+ */
14
+ const doWrap = (model, middleware) => {
15
+ const transform = async (params, type) => middleware.transformParams
16
+ ? await middleware.transformParams({ type, params, model })
17
+ : params;
18
+ return {
19
+ specificationVersion: "v3",
20
+ provider: middleware.overrideProvider?.({ model }) ?? model.provider,
21
+ modelId: middleware.overrideModelId?.({ model }) ?? model.modelId,
22
+ supportedUrls: middleware.overrideSupportedUrls?.({ model }) ?? model.supportedUrls,
23
+ async doGenerate(params) {
24
+ const transformed = await transform(params, "generate");
25
+ const doGenerate = () => model.doGenerate(transformed);
26
+ const doStream = () => model.doStream(transformed);
27
+ return middleware.wrapGenerate
28
+ ? await middleware.wrapGenerate({
29
+ doGenerate,
30
+ doStream,
31
+ params: transformed,
32
+ model,
33
+ })
34
+ : await doGenerate();
35
+ },
36
+ async doStream(params) {
37
+ const transformed = await transform(params, "stream");
38
+ const doGenerate = () => model.doGenerate(transformed);
39
+ const doStream = () => model.doStream(transformed);
40
+ return middleware.wrapStream
41
+ ? await middleware.wrapStream({
42
+ doGenerate,
43
+ doStream,
44
+ params: transformed,
45
+ model,
46
+ })
47
+ : await doStream();
48
+ },
49
+ };
50
+ };
51
+ export const wrapLanguageModel = ({ model, middleware, }) => (Array.isArray(middleware) ? [...middleware] : [middleware])
52
+ .reverse()
53
+ .reduce((wrapped, m) => doWrap(wrapped, m), model);
@@ -335,22 +335,57 @@ export class AudioProcessor extends BaseFileProcessor {
335
335
  return skipped(`format is not one Whisper accepts (extension "${ext ?? "none"}", mimetype "${mimetype ?? "none"}"); supported: ${AUDIO_CONFIG.WHISPER_SUPPORTED_FORMATS.join(", ")}`);
336
336
  }
337
337
  try {
338
- // Dynamic imports to avoid loading these modules when transcription is not needed
339
- const [{ createOpenAI }, { experimental_transcribe }] = await Promise.all([import("@ai-sdk/openai"), import("../../utils/generation.js")]);
340
- const openai = createOpenAI({ apiKey });
341
- const model = openai.transcription("whisper-1");
338
+ // Native multipart POST to OpenAI's transcription endpoint. This used to
339
+ // go through @ai-sdk/openai's createOpenAI().transcription() plus the ai
340
+ // package's experimental_transcribe; both were dropped, and this is the
341
+ // only wire behaviour of theirs the processor ever depended on. The same
342
+ // request is already made natively by voice/providers/OpenAISTT.ts.
343
+ // Only `text` is read off the response, as before.
344
+ const baseUrl = (process.env.OPENAI_BASE_URL ?? "https://api.openai.com/v1").replace(/\/+$/, "");
345
+ const form = new FormData();
346
+ form.append("file", new Blob([new Uint8Array(buffer)], {
347
+ type: mimetype || "audio/mpeg",
348
+ }), filename);
349
+ form.append("model", "whisper-1");
350
+ form.append("response_format", "verbose_json");
342
351
  // Wrap in withTimeout — large audio files can take a while, but a
343
352
  // stalled request shouldn't block the processor forever. A TimeoutError
344
353
  // lands in the same handler as other failures below, which reports it as
345
354
  // the reason rather than discarding it.
346
- const result = await withTimeout(experimental_transcribe({
347
- model,
348
- audio: buffer,
349
- }), AUDIO_CONFIG.TRANSCRIPTION_TIMEOUT_MS, "openai-whisper", "generate");
350
- if (result.text && result.text.trim().length > 0) {
351
- logger.debug(`[AudioProcessor] Transcribed ${filename} via openai-whisper (${result.text.trim().length} chars)`);
355
+ // `withTimeout` only races the promise against a timer — it cannot
356
+ // cancel the operation. This code owns the raw fetch now, so without an
357
+ // abort the socket and its in-flight upload (up to 25MB) stay alive
358
+ // after the timeout has already resolved the caller.
359
+ const transcriptionAbort = new AbortController();
360
+ const transcriptionTimer = setTimeout(() => transcriptionAbort.abort(), AUDIO_CONFIG.TRANSCRIPTION_TIMEOUT_MS);
361
+ let response;
362
+ try {
363
+ response = await withTimeout(fetch(`${baseUrl}/audio/transcriptions`, {
364
+ method: "POST",
365
+ headers: { Authorization: `Bearer ${apiKey}` },
366
+ body: form,
367
+ signal: transcriptionAbort.signal,
368
+ }), AUDIO_CONFIG.TRANSCRIPTION_TIMEOUT_MS, "openai-whisper", "generate");
369
+ }
370
+ finally {
371
+ clearTimeout(transcriptionTimer);
372
+ }
373
+ if (!response.ok) {
374
+ const detail = await response.text().catch(() => "");
375
+ // Mirrors the old behaviour: a non-2xx used to surface as a thrown
376
+ // APICallError caught by the handler below and reported as the reason.
377
+ return skipped(`transcription request failed — HTTP ${response.status}${detail ? `: ${detail.slice(0, 200)}` : ""}`);
378
+ }
379
+ const payload = await response.json();
380
+ const rawText = typeof payload === "object" &&
381
+ payload !== null &&
382
+ typeof payload.text === "string"
383
+ ? payload.text
384
+ : "";
385
+ if (rawText.trim().length > 0) {
386
+ logger.debug(`[AudioProcessor] Transcribed ${filename} via openai-whisper (${rawText.trim().length} chars)`);
352
387
  return {
353
- transcript: result.text.trim(),
388
+ transcript: rawText.trim(),
354
389
  hasTranscript: true,
355
390
  transcriptionProvider: "openai-whisper",
356
391
  transcriptionSkippedReason: undefined,
@@ -1,7 +1,7 @@
1
1
  import type { AIProviderName } from "../constants/enums.js";
2
2
  import { BaseProvider } from "../core/baseProvider.js";
3
3
  import type { NeuroLink } from "../neurolink.js";
4
- import type { StreamOptions } from "../types/index.js";
4
+ import type { EnhancedGenerateResult, TextGenerationOptions, ValidationSchema, StreamOptions } from "../types/index.js";
5
5
  import type { LanguageModel } from "../types/index.js";
6
6
  /**
7
7
  * Amazon SageMaker Provider extending BaseProvider
@@ -20,6 +20,22 @@ export declare class AmazonSageMakerProvider extends BaseProvider {
20
20
  protected getProviderName(): AIProviderName;
21
21
  protected getDefaultModel(): string;
22
22
  protected getAISDKModel(): LanguageModel;
23
+ /**
24
+ * Native non-streaming generate.
25
+ *
26
+ * SageMaker's doGenerate makes one invokeEndpoint call and already returns
27
+ * toolCalls; no streaming is involved, so the wire hazard that reverted the
28
+ * first migration does not apply here. This supplies only the multi-step
29
+ * iteration the ai package used to.
30
+ *
31
+ * NOT EXERCISED LIVE. This machine has no SageMaker endpoint or credentials.
32
+ * The single-step shape is identical by construction — with no tool calls the
33
+ * loop breaks after exactly one doGenerate carrying the same options the ai
34
+ * loop passed. The multi-step branch is the new code and wants a real
35
+ * endpoint before it is trusted.
36
+ */
37
+ generate(optionsOrPrompt: TextGenerationOptions | string, analysisSchema?: ValidationSchema): Promise<EnhancedGenerateResult | null>;
38
+ private executeNativeGenerate;
23
39
  /**
24
40
  * Streaming was previously an `executeStream` override that unconditionally
25
41
  * threw "not yet fully implemented" — while `SageMakerLanguageModel.doStream`