@juspay/neurolink 9.63.0 → 9.64.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/CHANGELOG.md +12 -0
  2. package/dist/adapters/video/vertexVideoHandler.js +9 -2
  3. package/dist/browser/neurolink.min.js +1015 -1019
  4. package/dist/cli/factories/commandFactory.d.ts +14 -0
  5. package/dist/cli/factories/commandFactory.js +50 -25
  6. package/dist/cli/loop/optionsSchema.d.ts +1 -1
  7. package/dist/cli/loop/optionsSchema.js +12 -0
  8. package/dist/core/baseProvider.d.ts +1 -1
  9. package/dist/core/modules/MessageBuilder.js +20 -0
  10. package/dist/core/redisConversationMemoryManager.js +0 -3
  11. package/dist/factories/providerRegistry.js +5 -1
  12. package/dist/lib/adapters/video/vertexVideoHandler.js +9 -2
  13. package/dist/lib/core/baseProvider.d.ts +1 -1
  14. package/dist/lib/core/modules/MessageBuilder.js +20 -0
  15. package/dist/lib/core/redisConversationMemoryManager.js +0 -3
  16. package/dist/lib/factories/providerRegistry.js +5 -1
  17. package/dist/lib/memory/hippocampusInitializer.d.ts +2 -2
  18. package/dist/lib/memory/hippocampusInitializer.js +32 -2
  19. package/dist/lib/middleware/builtin/lifecycle.js +19 -48
  20. package/dist/lib/neurolink.js +49 -2
  21. package/dist/lib/providers/googleAiStudio.d.ts +11 -3
  22. package/dist/lib/providers/googleAiStudio.js +292 -339
  23. package/dist/lib/providers/googleNativeGemini3.d.ts +83 -1
  24. package/dist/lib/providers/googleNativeGemini3.js +208 -4
  25. package/dist/lib/providers/googleVertex.d.ts +116 -129
  26. package/dist/lib/providers/googleVertex.js +2826 -1968
  27. package/dist/lib/providers/openRouter.js +7 -3
  28. package/dist/lib/types/aliases.d.ts +14 -0
  29. package/dist/lib/types/common.d.ts +0 -3
  30. package/dist/lib/types/conversation.d.ts +10 -3
  31. package/dist/lib/types/generate.d.ts +14 -0
  32. package/dist/lib/types/index.d.ts +1 -0
  33. package/dist/lib/types/index.js +1 -0
  34. package/dist/lib/types/memory.d.ts +96 -0
  35. package/dist/lib/types/memory.js +23 -0
  36. package/dist/lib/types/providers.d.ts +140 -2
  37. package/dist/lib/types/stream.d.ts +6 -0
  38. package/dist/lib/utils/lifecycleCallbacks.d.ts +13 -0
  39. package/dist/lib/utils/lifecycleCallbacks.js +44 -0
  40. package/dist/lib/utils/messageBuilder.d.ts +10 -0
  41. package/dist/lib/utils/messageBuilder.js +40 -5
  42. package/dist/lib/utils/modelDetection.d.ts +11 -0
  43. package/dist/lib/utils/modelDetection.js +27 -0
  44. package/dist/lib/utils/providerHealth.js +7 -7
  45. package/dist/lib/utils/schemaConversion.d.ts +1 -1
  46. package/dist/lib/utils/schemaConversion.js +59 -4
  47. package/dist/lib/utils/tokenLimits.js +23 -32
  48. package/dist/memory/hippocampusInitializer.d.ts +2 -2
  49. package/dist/memory/hippocampusInitializer.js +32 -2
  50. package/dist/middleware/builtin/lifecycle.js +19 -48
  51. package/dist/neurolink.js +49 -2
  52. package/dist/providers/googleAiStudio.d.ts +11 -3
  53. package/dist/providers/googleAiStudio.js +291 -339
  54. package/dist/providers/googleNativeGemini3.d.ts +83 -1
  55. package/dist/providers/googleNativeGemini3.js +208 -4
  56. package/dist/providers/googleVertex.d.ts +116 -129
  57. package/dist/providers/googleVertex.js +2824 -1967
  58. package/dist/providers/openRouter.js +7 -3
  59. package/dist/types/aliases.d.ts +14 -0
  60. package/dist/types/common.d.ts +0 -3
  61. package/dist/types/conversation.d.ts +10 -3
  62. package/dist/types/generate.d.ts +14 -0
  63. package/dist/types/index.d.ts +1 -0
  64. package/dist/types/index.js +1 -0
  65. package/dist/types/memory.d.ts +96 -0
  66. package/dist/types/memory.js +22 -0
  67. package/dist/types/providers.d.ts +140 -2
  68. package/dist/types/stream.d.ts +6 -0
  69. package/dist/utils/lifecycleCallbacks.d.ts +13 -0
  70. package/dist/utils/lifecycleCallbacks.js +43 -0
  71. package/dist/utils/messageBuilder.d.ts +10 -0
  72. package/dist/utils/messageBuilder.js +40 -5
  73. package/dist/utils/modelDetection.d.ts +11 -0
  74. package/dist/utils/modelDetection.js +27 -0
  75. package/dist/utils/providerHealth.js +7 -7
  76. package/dist/utils/schemaConversion.d.ts +1 -1
  77. package/dist/utils/schemaConversion.js +59 -4
  78. package/dist/utils/tokenLimits.js +23 -32
  79. package/package.json +11 -4
@@ -1,22 +1,16 @@
1
- import { createGoogleGenerativeAI } from "@ai-sdk/google";
2
- import { embed, embedMany, stepCountIs, streamText, } from "ai";
1
+ import {} from "ai";
3
2
  import { ErrorCategory, ErrorSeverity, GoogleAIModels, } from "../constants/enums.js";
4
3
  import { BaseProvider } from "../core/baseProvider.js";
5
- import { DEFAULT_MAX_STEPS } from "../core/constants.js";
6
- import { streamAnalyticsCollector } from "../core/streamAnalytics.js";
7
- import { markStreamProviderEmittedGenerationEnd, } from "../neurolink.js";
8
- import { SpanStatusCode } from "@opentelemetry/api";
4
+ import { IMAGE_GENERATION_MODELS } from "../core/constants.js";
5
+ import { processUnifiedFilesArray } from "../utils/messageBuilder.js";
9
6
  import { ATTR, tracers, withClientSpan } from "../telemetry/index.js";
10
- import { AuthenticationError, NetworkError, ProviderError, RateLimitError, } from "../types/index.js";
7
+ import { AuthenticationError, InvalidModelError, NetworkError, ProviderError, RateLimitError, } from "../types/index.js";
11
8
  import { ERROR_CODES, NeuroLinkError } from "../utils/errorHandling.js";
12
9
  import { logger } from "../utils/logger.js";
13
- import { isGemini3Model } from "../utils/modelDetection.js";
14
10
  import { composeAbortSignals, createTimeoutController, TimeoutError, } from "../utils/timeout.js";
15
11
  import { estimateTokens } from "../utils/tokenEstimation.js";
16
- import { resolveToolChoice } from "../utils/toolChoice.js";
17
- import { emitToolEndFromStepFinish } from "../utils/toolEndEmitter.js";
18
- import { buildNativeConfig, buildNativeToolDeclarations, collectStreamChunks, collectStreamChunksIncremental, computeMaxSteps, createTextChannel, executeNativeToolCalls, extractTextFromParts, handleMaxStepsTermination, pushModelResponseToHistory, sanitizeToolsForGemini, } from "./googleNativeGemini3.js";
19
- import { toAnalyticsStreamResult } from "./providerTypeUtils.js";
12
+ import { buildGeminiResponseSchema, buildNativeConfig, buildNativeToolDeclarations, collectStreamChunks, collectStreamChunksIncremental, computeMaxSteps, createTextChannel, buildUserPartsWithMultimodal, executeNativeToolCalls, extractTextFromParts, handleMaxStepsTermination, prependConversationMessages, pushModelResponseToHistory, } from "./googleNativeGemini3.js";
13
+ import { createProxyFetch } from "../proxy/proxyFetch.js";
20
14
  // Google AI Live API types now imported from ../types/providerSpecific.js
21
15
  // Import proper types for multimodal message handling
22
16
  // Create Google GenAI client
@@ -34,7 +28,13 @@ async function createGoogleGenAIClient(apiKey) {
34
28
  });
35
29
  }
36
30
  const Ctor = ctor;
37
- return new Ctor({ apiKey });
31
+ // Include httpOptions with proxy fetch for corporate network support
32
+ return new Ctor({
33
+ apiKey,
34
+ httpOptions: {
35
+ fetch: createProxyFetch(),
36
+ },
37
+ });
38
38
  }
39
39
  /**
40
40
  * Google AI Studio provider implementation using BaseProvider
@@ -88,12 +88,18 @@ export class GoogleAIStudioProvider extends BaseProvider {
88
88
  return process.env.GOOGLE_AI_MODEL || GoogleAIModels.GEMINI_2_5_FLASH;
89
89
  }
90
90
  /**
91
- * 🔧 PHASE 2: Return AI SDK model instance for tool calling
91
+ * AI SDK model instance no longer used.
92
+ * All models are routed through native @google/genai SDK directly.
92
93
  */
93
94
  getAISDKModel() {
94
- const apiKey = this.getApiKey();
95
- const google = createGoogleGenerativeAI({ apiKey });
96
- return google(this.modelName);
95
+ throw new NeuroLinkError({
96
+ code: ERROR_CODES.INVALID_CONFIGURATION,
97
+ message: "GoogleAIStudioProvider no longer uses @ai-sdk/google. All models use native @google/genai SDK.",
98
+ category: ErrorCategory.CONFIGURATION,
99
+ severity: ErrorSeverity.CRITICAL,
100
+ retriable: false,
101
+ context: { provider: this.providerName, model: this.modelName },
102
+ });
97
103
  }
98
104
  formatProviderError(error) {
99
105
  if (error instanceof TimeoutError) {
@@ -103,12 +109,53 @@ export class GoogleAIStudioProvider extends BaseProvider {
103
109
  const message = typeof errorRecord?.message === "string"
104
110
  ? errorRecord.message
105
111
  : "Unknown error";
106
- if (message.includes("API_KEY_INVALID")) {
112
+ const statusCode = typeof errorRecord?.status === "number"
113
+ ? errorRecord.status
114
+ : typeof errorRecord?.statusCode === "number"
115
+ ? errorRecord.statusCode
116
+ : undefined;
117
+ // Authentication errors
118
+ if (message.includes("API_KEY_INVALID") ||
119
+ message.includes("Invalid API key") ||
120
+ statusCode === 401) {
107
121
  return new AuthenticationError("Invalid Google AI API key. Please check your GOOGLE_AI_API_KEY environment variable.", this.providerName);
108
122
  }
109
- if (message.includes("RATE_LIMIT_EXCEEDED")) {
123
+ // Rate limit errors
124
+ if (message.includes("RATE_LIMIT_EXCEEDED") ||
125
+ message.includes("rate limit") ||
126
+ message.includes("429") ||
127
+ statusCode === 429) {
110
128
  return new RateLimitError("Google AI rate limit exceeded. Please try again later.", this.providerName);
111
129
  }
130
+ // Model not found errors — gate on a 404 status when available; fall
131
+ // back to literal phrase matching only when we have no status code at
132
+ // all. Avoids misclassifying permission/validation errors that happen
133
+ // to mention model resource paths (e.g. "...models/foo permission...").
134
+ if (statusCode === 404 ||
135
+ (statusCode === undefined &&
136
+ (message.includes("model not found") ||
137
+ message.includes("Model not found")))) {
138
+ return new InvalidModelError(`Model '${this.modelName}' not found. Please check the model name and ensure it is available.`, this.providerName);
139
+ }
140
+ // Network connectivity errors
141
+ if (message.includes("ECONNRESET") ||
142
+ message.includes("ENOTFOUND") ||
143
+ message.includes("ETIMEDOUT") ||
144
+ message.includes("ECONNREFUSED") ||
145
+ message.includes("network") ||
146
+ message.includes("connection")) {
147
+ return new NetworkError(`Connection error: ${message}`, this.providerName);
148
+ }
149
+ // Server errors (5xx)
150
+ if (message.includes("500") ||
151
+ message.includes("502") ||
152
+ message.includes("503") ||
153
+ message.includes("504") ||
154
+ message.includes("server error") ||
155
+ message.includes("Internal Server Error") ||
156
+ (statusCode && statusCode >= 500 && statusCode < 600)) {
157
+ return new ProviderError(`Google AI server error: ${message}. Please try again later.`, this.providerName);
158
+ }
112
159
  return new ProviderError(`Google AI error: ${message}`, this.providerName);
113
160
  }
114
161
  /**
@@ -388,198 +435,49 @@ export class GoogleAIStudioProvider extends BaseProvider {
388
435
  }
389
436
  // executeGenerate removed - BaseProvider handles all generation with tools
390
437
  async executeStream(options, analysisSchema) {
391
- // Check if this is a Gemini 3 model with tools - use native SDK for thought_signature
392
- const gemini3CheckModelName = options.model || this.modelName;
393
- // Structured output (analysisSchema, JSON format, or schema) is incompatible with tools on Gemini.
394
- // Compute once and reuse in both the native Gemini 3 gate and the streamText fallback path.
395
- const wantsStructuredOutput = analysisSchema || options.output?.format === "json" || options.schema;
396
- // Check for tools from options AND from SDK (MCP tools)
397
- // Need to check early if we should route to native SDK
398
- const gemini3CheckShouldUseTools = !options.disableTools && this.supportsTools() && !wantsStructuredOutput;
399
- const tools = options.tools || {};
400
- const hasTools = gemini3CheckShouldUseTools && Object.keys(tools).length > 0;
401
- if (isGemini3Model(gemini3CheckModelName) && hasTools) {
402
- // Merge SDK tools into options for native SDK path
403
- let mergedOptions = {
404
- ...options,
405
- tools: tools,
406
- };
407
- // Check for tools + JSON schema conflict (Gemini limitation)
408
- const wantsJsonOutput = options.output?.format === "json" || options.schema;
409
- if (wantsJsonOutput &&
410
- mergedOptions.tools &&
411
- Object.keys(mergedOptions.tools).length > 0 &&
412
- !mergedOptions.disableTools) {
413
- logger.warn("[GoogleAIStudio] Gemini does not support tools and JSON schema output simultaneously. Disabling tools for this request.");
414
- mergedOptions = { ...mergedOptions, disableTools: true, tools: {} };
415
- }
416
- // Only route to native path if tools are still active after conflict check
417
- const hasActiveTools = !mergedOptions.disableTools &&
418
- mergedOptions.tools &&
419
- Object.keys(mergedOptions.tools).length > 0;
420
- if (hasActiveTools) {
421
- logger.info("[GoogleAIStudio] Routing Gemini 3 to native SDK for tool calling", {
422
- model: gemini3CheckModelName,
423
- totalToolCount: Object.keys(mergedOptions.tools ?? {}).length,
424
- });
425
- return this.executeNativeGemini3Stream(mergedOptions);
426
- }
427
- // Fall through to standard stream path using merged options (tools disabled for schema)
428
- options = mergedOptions;
429
- }
438
+ const modelName = options.model || this.modelName;
430
439
  // Phase 1: if audio input present, bridge to Gemini Live (Studio) using @google/genai
431
440
  if (options.input?.audio) {
432
441
  return await this.executeAudioStreamViaGeminiLive(options);
433
442
  }
434
- this.validateStreamOptions(options);
435
- const startTime = Date.now();
436
- const model = await this.getAISDKModelWithMiddleware(options);
437
- const timeout = this.getTimeout(options);
438
- const timeoutController = createTimeoutController(timeout, this.providerName, "stream");
439
- try {
440
- // Get tools consistently with generate method (include user-provided RAG tools)
441
- // wantsStructuredOutput already computed before the Gemini 3 native-routing gate
442
- if (wantsStructuredOutput &&
443
- !options.disableTools &&
444
- this.supportsTools()) {
445
- logger.warn("[GoogleAIStudio] Structured output active — disabling tools (Gemini limitation).");
446
- }
447
- const shouldUseTools = !options.disableTools && this.supportsTools() && !wantsStructuredOutput;
448
- const filteredTools = shouldUseTools
449
- ? (options.tools ?? {})
450
- : {};
451
- // Sanitize tool schemas for Gemini proto compatibility (converts anyOf/oneOf unions to string)
452
- let tools;
453
- if (Object.keys(filteredTools).length > 0) {
454
- const sanitized = sanitizeToolsForGemini(filteredTools);
455
- if (sanitized.dropped.length > 0) {
456
- logger.warn(`[GoogleAIStudio] Dropped ${sanitized.dropped.length} incompatible tool(s): ${sanitized.dropped.join(", ")}`);
457
- }
458
- tools =
459
- Object.keys(sanitized.tools).length > 0 ? sanitized.tools : undefined;
460
- }
461
- else {
462
- tools = undefined;
463
- }
464
- // Build message array from options with multimodal support
465
- // Using protected helper from BaseProvider to eliminate code duplication
466
- const messages = await this.buildMessagesForStream(options);
467
- const collectedToolCalls = [];
468
- const collectedToolResults = [];
469
- // Reviewer follow-up: capture upstream provider errors via onError
470
- // so the post-stream NoOutput sentinel carries the real cause.
471
- let capturedProviderError;
472
- const result = await streamText({
473
- model,
474
- messages: messages,
475
- temperature: options.temperature,
476
- maxOutputTokens: options.maxTokens, // No default limit - unlimited unless specified
477
- tools,
478
- stopWhen: stepCountIs(options.maxSteps || DEFAULT_MAX_STEPS),
479
- toolChoice: resolveToolChoice(options, tools, shouldUseTools),
480
- abortSignal: composeAbortSignals(options.abortSignal, timeoutController?.controller.signal),
481
- experimental_telemetry: this.telemetryHandler.getTelemetryConfig(options),
482
- experimental_repairToolCall: this.getToolCallRepairFn(options),
483
- onError: (event) => {
484
- capturedProviderError = event.error;
485
- logger.error("GoogleAiStudio: Stream error", {
486
- error: event.error instanceof Error
487
- ? event.error.message
488
- : String(event.error),
489
- });
490
- },
491
- // Gemini 3: use thinkingLevel via providerOptions
492
- // Gemini 2.5: use thinkingBudget via providerOptions
493
- ...(options.thinkingConfig?.enabled && {
494
- providerOptions: {
495
- google: {
496
- thinkingConfig: {
497
- ...(options.thinkingConfig.thinkingLevel && {
498
- thinkingLevel: options.thinkingConfig.thinkingLevel,
499
- }),
500
- ...(options.thinkingConfig.budgetTokens &&
501
- !options.thinkingConfig.thinkingLevel && {
502
- thinkingBudget: options.thinkingConfig.budgetTokens,
503
- }),
504
- includeThoughts: true,
505
- },
506
- },
507
- },
508
- }),
509
- onStepFinish: ({ toolCalls, toolResults }) => {
510
- for (const toolCall of toolCalls) {
511
- collectedToolCalls.push({
512
- toolCallId: toolCall.toolCallId,
513
- toolName: toolCall.toolName,
514
- args: toolCall.args ??
515
- toolCall.input ??
516
- toolCall
517
- .parameters ??
518
- {},
519
- });
520
- }
521
- for (const toolResult of toolResults) {
522
- const rawToolResult = toolResult;
523
- collectedToolResults.push({
524
- toolName: toolResult.toolName,
525
- status: rawToolResult.error ? "failure" : "success",
526
- output: (rawToolResult.output ??
527
- rawToolResult.result) ??
528
- undefined,
529
- error: rawToolResult.error,
530
- id: rawToolResult.toolCallId ?? toolResult.toolName,
531
- });
532
- }
533
- // Emit tool:end for each completed tool result so Pipeline B
534
- // captures telemetry for AI-SDK-driven tool calls (gap S2).
535
- emitToolEndFromStepFinish(this.neurolink?.getEventEmitter(), toolResults);
536
- this.handleToolExecutionStorage(toolCalls, toolResults, options, new Date()).catch((error) => {
537
- logger.warn("[GoogleAiStudioProvider] Failed to store tool executions", {
538
- provider: this.providerName,
539
- error: error instanceof Error ? error.message : String(error),
540
- });
541
- });
542
- },
543
- });
544
- // Defer timeout cleanup until the stream completes or errors.
545
- // Guard against NoOutputGeneratedError becoming an unhandled rejection.
546
- Promise.resolve(result.text)
547
- .catch((err) => {
548
- logger.debug("Stream text promise rejected (expected for empty streams)", {
549
- error: err instanceof Error ? err.message : String(err),
550
- });
551
- })
552
- .finally(() => timeoutController?.cleanup());
553
- // Transform string stream to content object stream using BaseProvider method
554
- const transformedStream = this.createTextStream(result, () => capturedProviderError);
555
- // Create analytics promise that resolves after stream completion
556
- const analyticsPromise = streamAnalyticsCollector.createAnalytics(this.providerName, this.modelName, toAnalyticsStreamResult(result), Date.now() - startTime, {
557
- requestId: `google-ai-stream-${Date.now()}`,
558
- streamingMode: true,
559
- });
560
- return {
561
- stream: transformedStream,
562
- provider: this.providerName,
563
- model: this.modelName,
564
- ...(shouldUseTools && {
565
- toolCalls: collectedToolCalls,
566
- toolResults: collectedToolResults,
567
- }),
568
- analytics: analyticsPromise,
569
- metadata: {
570
- startTime,
571
- streamId: `google-ai-${Date.now()}`,
572
- },
573
- };
443
+ // Structured output (analysisSchema, JSON format, or schema) is incompatible with tools on Gemini.
444
+ const wantsStructuredOutput = analysisSchema || options.output?.format === "json" || options.schema;
445
+ // Tool filter (a0269210): trust options.tools — caller (BaseProvider.stream)
446
+ // already merged MCP/built-in tools with user tools and applied any
447
+ // enabledToolNames filter. Re-attaching getAllTools() here would clobber
448
+ // that filter and re-introduce filtered-out tools.
449
+ const shouldUseTools = !options.disableTools && this.supportsTools() && !wantsStructuredOutput;
450
+ const optionTools = options.tools || {};
451
+ // Merge into options for native SDK path
452
+ let mergedOptions = {
453
+ ...options,
454
+ tools: optionTools,
455
+ };
456
+ // Check for tools + JSON schema conflict (Gemini limitation)
457
+ const wantsJsonOutput = options.output?.format === "json" || options.schema;
458
+ if (wantsJsonOutput &&
459
+ mergedOptions.tools &&
460
+ Object.keys(mergedOptions.tools).length > 0 &&
461
+ !mergedOptions.disableTools) {
462
+ logger.warn("[GoogleAIStudio] Gemini does not support tools and JSON schema output simultaneously. Disabling tools for this request.");
463
+ mergedOptions = { ...mergedOptions, disableTools: true, tools: {} };
574
464
  }
575
- catch (error) {
576
- timeoutController?.cleanup();
577
- throw this.handleProviderError(error);
465
+ const hasActiveTools = shouldUseTools &&
466
+ !mergedOptions.disableTools &&
467
+ mergedOptions.tools &&
468
+ Object.keys(mergedOptions.tools).length > 0;
469
+ if (hasActiveTools) {
470
+ logger.info("[GoogleAIStudio] Routing to native @google/genai SDK for tool calling", {
471
+ model: modelName,
472
+ totalToolCount: Object.keys(mergedOptions.tools ?? {}).length,
473
+ });
578
474
  }
475
+ // Route ALL models through native @google/genai SDK (no more @ai-sdk/google dependency)
476
+ return this.executeNativeGemini3Stream(mergedOptions);
579
477
  }
580
478
  /**
581
- * Execute stream using native @google/genai SDK for Gemini 3 models
582
- * This bypasses @ai-sdk/google to properly handle thought_signature
479
+ * Execute stream using native @google/genai SDK
480
+ * Uses @google/genai directly for all Gemini models (2.0, 2.5, 3.x)
583
481
  */
584
482
  async executeNativeGemini3Stream(options) {
585
483
  const modelName = options.model || this.modelName;
@@ -603,8 +501,23 @@ export class GoogleAIStudioProvider extends BaseProvider {
603
501
  model: modelName,
604
502
  hasTools: !!options.tools && Object.keys(options.tools).length > 0,
605
503
  });
606
- // Build contents from input
607
- const currentContents = [{ role: "user", parts: [{ text: options.input.text }] }];
504
+ // Build contents from input. Prepend prior conversation turns so
505
+ // multi-turn callers (memory, loop REPL, agent flows) actually
506
+ // carry context — the previous build started fresh from the
507
+ // current user input only, which silently dropped history.
508
+ //
509
+ // `buildUserPartsWithMultimodal` is the shared helper that also
510
+ // attaches `input.images` and `input.pdfFiles` as `inlineData`
511
+ // parts. The previous AI Studio path pushed only `{ text }` and
512
+ // silently dropped both, which is why the model legitimately
513
+ // reported "no image attached" on multimodal calls.
514
+ const currentContents = [];
515
+ prependConversationMessages(currentContents, options.conversationMessages);
516
+ const userParts = await buildUserPartsWithMultimodal(options.input, options.input.text, "[GoogleAIStudio:stream]");
517
+ currentContents.push({
518
+ role: "user",
519
+ parts: userParts,
520
+ });
608
521
  // Convert tools
609
522
  let toolsConfig;
610
523
  let executeMap = new Map();
@@ -619,7 +532,21 @@ export class GoogleAIStudioProvider extends BaseProvider {
619
532
  toolNames: toolsConfig[0].functionDeclarations.map((t) => t.name),
620
533
  });
621
534
  }
622
- const config = buildNativeConfig(options, toolsConfig);
535
+ // Native JSON / schema enforcement: when no tools are being sent
536
+ // (the AI Studio orchestrator above already force-disables tools
537
+ // whenever JSON/schema output is requested), enforce the response
538
+ // shape natively via responseMimeType / responseSchema. Without
539
+ // this, JSON output was best-effort prompting only.
540
+ const wantsNativeJson = !toolsConfig &&
541
+ (options.output?.format === "json" || !!options.schema);
542
+ const nativeResponseSchema = wantsNativeJson && options.schema
543
+ ? buildGeminiResponseSchema(options.schema)
544
+ : undefined;
545
+ const config = buildNativeConfig({
546
+ ...options,
547
+ wantsJsonOutput: wantsNativeJson,
548
+ responseSchema: nativeResponseSchema,
549
+ }, toolsConfig);
623
550
  const maxSteps = computeMaxSteps(options.maxSteps);
624
551
  // Compose abort signal from user signal + timeout
625
552
  const composedSignal = composeAbortSignals(options.abortSignal, timeoutController?.controller.signal);
@@ -742,68 +669,9 @@ export class GoogleAIStudioProvider extends BaseProvider {
742
669
  requestDuration: responseTime,
743
670
  timestamp: new Date().toISOString(),
744
671
  });
745
- // Emit generation:end so Pipeline B (Langfuse) creates a GENERATION
746
- // observation. The native @google/genai stream path bypasses the Vercel
747
- // AI SDK so experimental_telemetry is never injected; we emit manually.
748
- const nativeStreamEmitter = this.neurolink?.getEventEmitter();
749
- if (nativeStreamEmitter) {
750
- // Curator P2-4 dedup: flag the per-stream context attached
751
- // to options so the orchestration skips its own emit.
752
- markStreamProviderEmittedGenerationEnd(options);
753
- nativeStreamEmitter.emit("generation:end", {
754
- provider: this.providerName,
755
- responseTime,
756
- timestamp: Date.now(),
757
- result: {
758
- content: "",
759
- usage: {
760
- input: totalInputTokens,
761
- output: totalOutputTokens,
762
- total: totalInputTokens + totalOutputTokens,
763
- },
764
- model: modelName,
765
- provider: this.providerName,
766
- finishReason: hitStepLimitWithoutFinalAnswer
767
- ? "max_steps"
768
- : "stop",
769
- },
770
- success: true,
771
- });
772
- }
773
672
  channel.close();
774
673
  }
775
674
  catch (err) {
776
- // Propagate error to OTel span so traces show ERROR status
777
- span.recordException(err instanceof Error ? err : new Error(String(err)));
778
- span.setStatus({
779
- code: SpanStatusCode.ERROR,
780
- message: err instanceof Error ? err.message : String(err),
781
- });
782
- // Emit failure generation:end so Pipeline B records the failed stream
783
- const errorEmitter = this.neurolink?.getEventEmitter();
784
- if (errorEmitter) {
785
- // Curator P2-4 dedup: flag the per-stream context attached
786
- // to options so the orchestration skips its own emit.
787
- markStreamProviderEmittedGenerationEnd(options);
788
- errorEmitter.emit("generation:end", {
789
- provider: this.providerName,
790
- responseTime: Date.now() - startTime,
791
- timestamp: Date.now(),
792
- result: {
793
- content: "",
794
- usage: {
795
- input: totalInputTokens,
796
- output: totalOutputTokens,
797
- total: totalInputTokens + totalOutputTokens,
798
- },
799
- model: modelName,
800
- provider: this.providerName,
801
- finishReason: "error",
802
- },
803
- success: false,
804
- error: err instanceof Error ? err.message : String(err),
805
- });
806
- }
807
675
  channel.error(err);
808
676
  analyticsReject(err);
809
677
  }
@@ -858,8 +726,21 @@ export class GoogleAIStudioProvider extends BaseProvider {
858
726
  // Prefer input.text over prompt — processCSVFilesForNativeSDK enriches
859
727
  // input.text with inlined CSV data, so using prompt first would discard it.
860
728
  const promptText = options.input?.text || options.prompt || "";
861
- const currentContents = [{ role: "user", parts: [{ text: promptText }] }];
862
- // Convert tools (merge SDK tools with options.tools)
729
+ // Prepend prior conversation turns so multi-turn generate calls
730
+ // see history; otherwise the native generate path silently drops
731
+ // every turn before the current prompt.
732
+ //
733
+ // `buildUserPartsWithMultimodal` also attaches inline image / PDF
734
+ // parts. Without it the request body was text-only and the model
735
+ // legitimately reported "no image / PDF attached".
736
+ const currentContents = [];
737
+ prependConversationMessages(currentContents, options.conversationMessages);
738
+ const userParts = await buildUserPartsWithMultimodal(options.input, promptText, "[GoogleAIStudio:generate]");
739
+ currentContents.push({
740
+ role: "user",
741
+ parts: userParts,
742
+ });
743
+ // Convert tools (a0269210: trust options.tools — already merged + filtered upstream)
863
744
  let toolsConfig;
864
745
  let executeMap = new Map();
865
746
  const shouldUseTools = !options.disableTools;
@@ -875,7 +756,19 @@ export class GoogleAIStudioProvider extends BaseProvider {
875
756
  });
876
757
  }
877
758
  }
878
- const config = buildNativeConfig(options, toolsConfig);
759
+ // Native JSON / schema enforcement (generate path). Mirrors the
760
+ // stream block above; only set when no tools are being sent
761
+ // because Gemini cannot combine function calling with JSON mime.
762
+ const wantsNativeJson = !toolsConfig &&
763
+ (options.output?.format === "json" || !!options.schema);
764
+ const nativeResponseSchema = wantsNativeJson && options.schema
765
+ ? buildGeminiResponseSchema(options.schema)
766
+ : undefined;
767
+ const config = buildNativeConfig({
768
+ ...options,
769
+ wantsJsonOutput: wantsNativeJson,
770
+ responseSchema: nativeResponseSchema,
771
+ }, toolsConfig);
879
772
  const composedSignal = composeAbortSignals(options.abortSignal, timeoutController?.controller.signal);
880
773
  const maxSteps = computeMaxSteps(options.maxSteps);
881
774
  let finalText = "";
@@ -946,31 +839,11 @@ export class GoogleAIStudioProvider extends BaseProvider {
946
839
  span.setAttribute(ATTR.GEN_AI_INPUT_TOKENS, totalInputTokens);
947
840
  span.setAttribute(ATTR.GEN_AI_OUTPUT_TOKENS, totalOutputTokens);
948
841
  span.setAttribute(ATTR.GEN_AI_FINISH_REASON, step >= maxSteps ? "max_steps" : "stop");
949
- // Emit generation:end so Pipeline B (Langfuse) creates a GENERATION
950
- // observation. The native @google/genai path bypasses the Vercel AI SDK
951
- // so experimental_telemetry is never injected; we emit the event manually.
952
- const nativeGenerateEmitter = this.neurolink?.getEventEmitter();
953
- if (nativeGenerateEmitter) {
954
- nativeGenerateEmitter.emit("generation:end", {
955
- provider: this.providerName,
956
- responseTime,
957
- timestamp: Date.now(),
958
- result: {
959
- content: finalText,
960
- usage: {
961
- input: totalInputTokens,
962
- output: totalOutputTokens,
963
- total: totalInputTokens + totalOutputTokens,
964
- },
965
- model: modelName,
966
- provider: this.providerName,
967
- finishReason: step >= maxSteps ? "max_steps" : "stop",
968
- },
969
- success: true,
970
- });
971
- }
972
- // Build EnhancedGenerateResult
973
- return {
842
+ // Build EnhancedGenerateResult and route through enhanceResult so
843
+ // analytics / evaluation / tracing stay attached. The native AI
844
+ // Studio generate path bypasses BaseProvider.generate(), so
845
+ // skipping enhanceResult would silently drop those features.
846
+ const baseResult = {
974
847
  content: finalText,
975
848
  provider: this.providerName,
976
849
  model: modelName,
@@ -984,6 +857,7 @@ export class GoogleAIStudioProvider extends BaseProvider {
984
857
  toolExecutions: toolExecutions,
985
858
  enhancedWithTools: allToolCalls.length > 0,
986
859
  };
860
+ return this.enhanceResult(baseResult, options, startTime);
987
861
  }
988
862
  finally {
989
863
  timeoutController?.cleanup();
@@ -999,40 +873,116 @@ export class GoogleAIStudioProvider extends BaseProvider {
999
873
  ? { prompt: optionsOrPrompt }
1000
874
  : optionsOrPrompt;
1001
875
  const modelName = options.model || this.modelName;
1002
- // Check if we should use native SDK for Gemini 3 with tools
1003
- const shouldUseTools = !options.disableTools && this.supportsTools();
1004
- const hasTools = shouldUseTools && options.tools && Object.keys(options.tools).length > 0;
1005
- if (isGemini3Model(modelName) && hasTools) {
1006
- // Merge SDK tools into options for native SDK path
1007
- let mergedOptions = {
1008
- ...options,
1009
- tools: options.tools,
1010
- };
1011
- // Check for tools + JSON schema conflict (Gemini limitation)
1012
- const wantsJsonOutput = options.output?.format === "json" || options.schema;
1013
- if (wantsJsonOutput &&
1014
- mergedOptions.tools &&
1015
- Object.keys(mergedOptions.tools).length > 0 &&
1016
- !mergedOptions.disableTools) {
1017
- logger.warn("[GoogleAIStudio] Gemini does not support tools and JSON schema output simultaneously. Disabling tools for this request.");
1018
- mergedOptions = { ...mergedOptions, disableTools: true, tools: {} };
876
+ // Image-generation models reject function-calling. Route them to
877
+ // executeImageGeneration without merging tools. This must happen
878
+ // BEFORE getToolsForStream to avoid leaking registered (MCP / built-in)
879
+ // tools into the image API request, which trips
880
+ // "Function calling is not enabled for this model".
881
+ // startsWith (not includes) so a hypothetical text model whose ID
882
+ // contains an image-model string as a substring isn't silently routed
883
+ // to executeImageGeneration and stripped of tool support.
884
+ const isImageModel = IMAGE_GENERATION_MODELS.some((m) => modelName.toLowerCase().startsWith(m.toLowerCase()));
885
+ if (isImageModel) {
886
+ logger.info("[GoogleAIStudio] Routing image generation model to executeImageGeneration", { model: modelName });
887
+ return this.executeImageGeneration(options);
888
+ }
889
+ // Process the unified `input.files` array before routing to the
890
+ // native SDK. BaseProvider.generate() runs this preprocessing via
891
+ // buildMultimodalMessagesArray, but AI Studio's override skips it,
892
+ // which would otherwise drop text-file content (and the
893
+ // mimetype-hint contract) on the floor. Mutates options.input.text /
894
+ // options.input.images / options.input.pdfFiles in place.
895
+ if (options.input?.files && options.input.files.length > 0) {
896
+ try {
897
+ await processUnifiedFilesArray(options, 100 * 1024 * 1024, this.providerName);
1019
898
  }
1020
- // Only route to native path if tools are still active after conflict check
1021
- const hasActiveTools = !mergedOptions.disableTools &&
1022
- mergedOptions.tools &&
1023
- Object.keys(mergedOptions.tools).length > 0;
1024
- if (hasActiveTools) {
1025
- logger.info("[GoogleAIStudio] Routing Gemini 3 generate to native SDK for tool calling", {
1026
- model: modelName,
1027
- totalToolCount: Object.keys(mergedOptions.tools ?? {}).length,
1028
- });
1029
- return this.executeNativeGemini3Generate(mergedOptions);
899
+ catch (fileError) {
900
+ logger.warn(`[GoogleAIStudio] processUnifiedFilesArray threw, continuing without file content: ${fileError instanceof Error ? fileError.message : String(fileError)}`);
1030
901
  }
1031
- // Fall through to standard generate path using merged options (tools disabled for schema)
1032
- return super.generate(mergedOptions);
1033
902
  }
1034
- // Fall back to BaseProvider implementation
1035
- return super.generate(options);
903
+ // Merge registered (built-in / MCP) tools with caller-supplied tools.
904
+ // AI Studio's generate() bypasses BaseProvider.generate(), so the
905
+ // ToolsManager-driven merge that normally injects sdk.registerTool()
906
+ // entries never runs here. Without this call, registered tools never
907
+ // reach the native function-calling path.
908
+ const baseTools = !options.disableTools
909
+ ? await this.getToolsForStream(options)
910
+ : {};
911
+ let mergedOptions = {
912
+ ...options,
913
+ tools: baseTools,
914
+ };
915
+ // Check for tools + JSON schema conflict (Gemini limitation)
916
+ const wantsJsonOutput = options.output?.format === "json" || options.schema;
917
+ if (wantsJsonOutput &&
918
+ mergedOptions.tools &&
919
+ Object.keys(mergedOptions.tools).length > 0 &&
920
+ !mergedOptions.disableTools) {
921
+ logger.warn("[GoogleAIStudio] Gemini does not support tools and JSON schema output simultaneously. Disabling tools for this request.");
922
+ mergedOptions = { ...mergedOptions, disableTools: true, tools: {} };
923
+ }
924
+ const hasActiveTools = !mergedOptions.disableTools &&
925
+ mergedOptions.tools &&
926
+ Object.keys(mergedOptions.tools).length > 0;
927
+ if (hasActiveTools) {
928
+ logger.info("[GoogleAIStudio] Routing generate to native @google/genai SDK for tool calling", {
929
+ model: modelName,
930
+ totalToolCount: Object.keys(mergedOptions.tools ?? {}).length,
931
+ });
932
+ }
933
+ // Route ALL models through native @google/genai SDK (no more @ai-sdk/google dependency).
934
+ // Emit Pipeline B `generation:end` so the observability listener
935
+ // creates a `model.generation` span — AI Studio's native path bypasses
936
+ // the AI SDK + experimental_telemetry plumbing the same way Vertex's
937
+ // does, so the event has to be emitted manually.
938
+ const generateStartTime = Date.now();
939
+ const inputPrompt = mergedOptions.input?.text ||
940
+ mergedOptions.prompt ||
941
+ "";
942
+ try {
943
+ const result = await this.executeNativeGemini3Generate(mergedOptions);
944
+ this.emitPipelineBGenerationEvent(modelName, result, generateStartTime, true, undefined, inputPrompt);
945
+ return result;
946
+ }
947
+ catch (error) {
948
+ this.emitPipelineBGenerationEvent(modelName, null, generateStartTime, false, error, inputPrompt);
949
+ throw error;
950
+ }
951
+ }
952
+ /**
953
+ * Emit `generation:end` so the Pipeline B observability listener creates
954
+ * a `model.generation` span for native Google AI Studio generate calls.
955
+ * Without this hand-off the native path silently disappears from
956
+ * Pipeline B exporters (Langfuse, custom OTEL collectors).
957
+ */
958
+ emitPipelineBGenerationEvent(modelName, result, startTime, success, error, prompt) {
959
+ const emitter = this.neurolink?.getEventEmitter();
960
+ if (!emitter) {
961
+ return;
962
+ }
963
+ const usage = result?.usage && typeof result.usage === "object"
964
+ ? result.usage
965
+ : { input: 0, output: 0, total: 0 };
966
+ emitter.emit("generation:end", {
967
+ provider: this.providerName,
968
+ responseTime: Date.now() - startTime,
969
+ timestamp: Date.now(),
970
+ // The Pipeline B listener reads `data.prompt` to populate the
971
+ // `input` span attribute. Without this, the Observability Spans
972
+ // test fails with "input capture not working".
973
+ prompt: prompt || "",
974
+ result: {
975
+ content: result?.content || "",
976
+ usage,
977
+ model: modelName,
978
+ provider: this.providerName,
979
+ finishReason: success ? "stop" : "error",
980
+ },
981
+ success,
982
+ ...(error
983
+ ? { error: error instanceof Error ? error.message : String(error) }
984
+ : {}),
985
+ });
1036
986
  }
1037
987
  // ===================
1038
988
  // HELPER METHODS
@@ -1240,18 +1190,21 @@ export class GoogleAIStudioProvider extends BaseProvider {
1240
1190
  });
1241
1191
  try {
1242
1192
  const apiKey = this.getApiKey();
1243
- const google = createGoogleGenerativeAI({ apiKey });
1244
- const embeddingModel = google.textEmbeddingModel(embeddingModelName);
1245
- const result = await embed({
1246
- model: embeddingModel,
1247
- value: text,
1193
+ const client = await createGoogleGenAIClient(apiKey);
1194
+ const result = await client.models.embedContent({
1195
+ model: embeddingModelName,
1196
+ contents: [text],
1248
1197
  });
1198
+ const embedding = result.embeddings?.[0]?.values;
1199
+ if (!embedding) {
1200
+ throw new ProviderError("No embedding returned from Google AI", this.providerName);
1201
+ }
1249
1202
  logger.debug("Embedding generated successfully", {
1250
1203
  provider: this.providerName,
1251
1204
  model: embeddingModelName,
1252
- embeddingDimension: result.embedding.length,
1205
+ embeddingDimension: embedding.length,
1253
1206
  });
1254
- return result.embedding;
1207
+ return embedding;
1255
1208
  }
1256
1209
  catch (error) {
1257
1210
  logger.error("Embedding generation failed", {
@@ -1277,19 +1230,19 @@ export class GoogleAIStudioProvider extends BaseProvider {
1277
1230
  });
1278
1231
  try {
1279
1232
  const apiKey = this.getApiKey();
1280
- const google = createGoogleGenerativeAI({ apiKey });
1281
- const embeddingModel = google.textEmbeddingModel(embeddingModelName);
1282
- const result = await embedMany({
1283
- model: embeddingModel,
1284
- values: texts,
1233
+ const client = await createGoogleGenAIClient(apiKey);
1234
+ const result = await client.models.embedContent({
1235
+ model: embeddingModelName,
1236
+ contents: texts,
1285
1237
  });
1238
+ const embeddings = (result.embeddings || []).map((e) => e.values || []);
1286
1239
  logger.debug("Batch embeddings generated successfully", {
1287
1240
  provider: this.providerName,
1288
1241
  model: embeddingModelName,
1289
- count: result.embeddings.length,
1290
- embeddingDimension: result.embeddings[0]?.length,
1242
+ count: embeddings.length,
1243
+ embeddingDimension: embeddings[0]?.length,
1291
1244
  });
1292
- return result.embeddings;
1245
+ return embeddings;
1293
1246
  }
1294
1247
  catch (error) {
1295
1248
  logger.error("Batch embedding generation failed", {