@juspay/neurolink 9.63.1 → 9.64.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/CHANGELOG.md +6 -0
  2. package/dist/adapters/video/vertexVideoHandler.js +9 -2
  3. package/dist/browser/neurolink.min.js +1014 -1018
  4. package/dist/cli/factories/commandFactory.d.ts +14 -0
  5. package/dist/cli/factories/commandFactory.js +50 -25
  6. package/dist/cli/loop/optionsSchema.d.ts +1 -1
  7. package/dist/cli/loop/optionsSchema.js +12 -0
  8. package/dist/core/baseProvider.d.ts +1 -1
  9. package/dist/core/modules/MessageBuilder.js +20 -0
  10. package/dist/factories/providerRegistry.js +5 -1
  11. package/dist/lib/adapters/video/vertexVideoHandler.js +9 -2
  12. package/dist/lib/core/baseProvider.d.ts +1 -1
  13. package/dist/lib/core/modules/MessageBuilder.js +20 -0
  14. package/dist/lib/factories/providerRegistry.js +5 -1
  15. package/dist/lib/memory/hippocampusInitializer.d.ts +2 -2
  16. package/dist/lib/memory/hippocampusInitializer.js +32 -2
  17. package/dist/lib/middleware/builtin/lifecycle.js +19 -48
  18. package/dist/lib/neurolink.js +49 -2
  19. package/dist/lib/providers/googleAiStudio.d.ts +11 -3
  20. package/dist/lib/providers/googleAiStudio.js +292 -339
  21. package/dist/lib/providers/googleNativeGemini3.d.ts +83 -1
  22. package/dist/lib/providers/googleNativeGemini3.js +208 -4
  23. package/dist/lib/providers/googleVertex.d.ts +116 -129
  24. package/dist/lib/providers/googleVertex.js +2826 -1968
  25. package/dist/lib/providers/openRouter.js +7 -3
  26. package/dist/lib/types/aliases.d.ts +14 -0
  27. package/dist/lib/types/common.d.ts +0 -3
  28. package/dist/lib/types/conversation.d.ts +10 -3
  29. package/dist/lib/types/generate.d.ts +14 -0
  30. package/dist/lib/types/index.d.ts +1 -0
  31. package/dist/lib/types/index.js +1 -0
  32. package/dist/lib/types/memory.d.ts +96 -0
  33. package/dist/lib/types/memory.js +23 -0
  34. package/dist/lib/types/providers.d.ts +140 -2
  35. package/dist/lib/types/stream.d.ts +6 -0
  36. package/dist/lib/utils/lifecycleCallbacks.d.ts +13 -0
  37. package/dist/lib/utils/lifecycleCallbacks.js +44 -0
  38. package/dist/lib/utils/messageBuilder.d.ts +10 -0
  39. package/dist/lib/utils/messageBuilder.js +40 -5
  40. package/dist/lib/utils/modelDetection.d.ts +11 -0
  41. package/dist/lib/utils/modelDetection.js +27 -0
  42. package/dist/lib/utils/providerHealth.js +7 -7
  43. package/dist/lib/utils/schemaConversion.d.ts +1 -1
  44. package/dist/lib/utils/schemaConversion.js +59 -4
  45. package/dist/lib/utils/tokenLimits.js +23 -32
  46. package/dist/memory/hippocampusInitializer.d.ts +2 -2
  47. package/dist/memory/hippocampusInitializer.js +32 -2
  48. package/dist/middleware/builtin/lifecycle.js +19 -48
  49. package/dist/neurolink.js +49 -2
  50. package/dist/providers/googleAiStudio.d.ts +11 -3
  51. package/dist/providers/googleAiStudio.js +291 -339
  52. package/dist/providers/googleNativeGemini3.d.ts +83 -1
  53. package/dist/providers/googleNativeGemini3.js +208 -4
  54. package/dist/providers/googleVertex.d.ts +116 -129
  55. package/dist/providers/googleVertex.js +2824 -1967
  56. package/dist/providers/openRouter.js +7 -3
  57. package/dist/types/aliases.d.ts +14 -0
  58. package/dist/types/common.d.ts +0 -3
  59. package/dist/types/conversation.d.ts +10 -3
  60. package/dist/types/generate.d.ts +14 -0
  61. package/dist/types/index.d.ts +1 -0
  62. package/dist/types/index.js +1 -0
  63. package/dist/types/memory.d.ts +96 -0
  64. package/dist/types/memory.js +22 -0
  65. package/dist/types/providers.d.ts +140 -2
  66. package/dist/types/stream.d.ts +6 -0
  67. package/dist/utils/lifecycleCallbacks.d.ts +13 -0
  68. package/dist/utils/lifecycleCallbacks.js +43 -0
  69. package/dist/utils/messageBuilder.d.ts +10 -0
  70. package/dist/utils/messageBuilder.js +40 -5
  71. package/dist/utils/modelDetection.d.ts +11 -0
  72. package/dist/utils/modelDetection.js +27 -0
  73. package/dist/utils/providerHealth.js +7 -7
  74. package/dist/utils/schemaConversion.d.ts +1 -1
  75. package/dist/utils/schemaConversion.js +59 -4
  76. package/dist/utils/tokenLimits.js +23 -32
  77. package/package.json +11 -4
@@ -1,22 +1,15 @@
1
- import { createGoogleGenerativeAI } from "@ai-sdk/google";
2
- import { embed, embedMany, stepCountIs, streamText, } from "ai";
3
1
  import { ErrorCategory, ErrorSeverity, GoogleAIModels, } from "../constants/enums.js";
4
2
  import { BaseProvider } from "../core/baseProvider.js";
5
- import { DEFAULT_MAX_STEPS } from "../core/constants.js";
6
- import { streamAnalyticsCollector } from "../core/streamAnalytics.js";
7
- import { markStreamProviderEmittedGenerationEnd, } from "../neurolink.js";
8
- import { SpanStatusCode } from "@opentelemetry/api";
3
+ import { IMAGE_GENERATION_MODELS } from "../core/constants.js";
4
+ import { processUnifiedFilesArray } from "../utils/messageBuilder.js";
9
5
  import { ATTR, tracers, withClientSpan } from "../telemetry/index.js";
10
- import { AuthenticationError, NetworkError, ProviderError, RateLimitError, } from "../types/index.js";
6
+ import { AuthenticationError, InvalidModelError, NetworkError, ProviderError, RateLimitError, } from "../types/index.js";
11
7
  import { ERROR_CODES, NeuroLinkError } from "../utils/errorHandling.js";
12
8
  import { logger } from "../utils/logger.js";
13
- import { isGemini3Model } from "../utils/modelDetection.js";
14
9
  import { composeAbortSignals, createTimeoutController, TimeoutError, } from "../utils/timeout.js";
15
10
  import { estimateTokens } from "../utils/tokenEstimation.js";
16
- import { resolveToolChoice } from "../utils/toolChoice.js";
17
- import { emitToolEndFromStepFinish } from "../utils/toolEndEmitter.js";
18
- import { buildNativeConfig, buildNativeToolDeclarations, collectStreamChunks, collectStreamChunksIncremental, computeMaxSteps, createTextChannel, executeNativeToolCalls, extractTextFromParts, handleMaxStepsTermination, pushModelResponseToHistory, sanitizeToolsForGemini, } from "./googleNativeGemini3.js";
19
- import { toAnalyticsStreamResult } from "./providerTypeUtils.js";
11
+ import { buildGeminiResponseSchema, buildNativeConfig, buildNativeToolDeclarations, collectStreamChunks, collectStreamChunksIncremental, computeMaxSteps, createTextChannel, buildUserPartsWithMultimodal, executeNativeToolCalls, extractTextFromParts, handleMaxStepsTermination, prependConversationMessages, pushModelResponseToHistory, } from "./googleNativeGemini3.js";
12
+ import { createProxyFetch } from "../proxy/proxyFetch.js";
20
13
  // Google AI Live API types now imported from ../types/providerSpecific.js
21
14
  // Import proper types for multimodal message handling
22
15
  // Create Google GenAI client
@@ -34,7 +27,13 @@ async function createGoogleGenAIClient(apiKey) {
34
27
  });
35
28
  }
36
29
  const Ctor = ctor;
37
- return new Ctor({ apiKey });
30
+ // Include httpOptions with proxy fetch for corporate network support
31
+ return new Ctor({
32
+ apiKey,
33
+ httpOptions: {
34
+ fetch: createProxyFetch(),
35
+ },
36
+ });
38
37
  }
39
38
  /**
40
39
  * Google AI Studio provider implementation using BaseProvider
@@ -88,12 +87,18 @@ export class GoogleAIStudioProvider extends BaseProvider {
88
87
  return process.env.GOOGLE_AI_MODEL || GoogleAIModels.GEMINI_2_5_FLASH;
89
88
  }
90
89
  /**
91
- * 🔧 PHASE 2: Return AI SDK model instance for tool calling
90
+ * AI SDK model instance no longer used.
91
+ * All models are routed through native @google/genai SDK directly.
92
92
  */
93
93
  getAISDKModel() {
94
- const apiKey = this.getApiKey();
95
- const google = createGoogleGenerativeAI({ apiKey });
96
- return google(this.modelName);
94
+ throw new NeuroLinkError({
95
+ code: ERROR_CODES.INVALID_CONFIGURATION,
96
+ message: "GoogleAIStudioProvider no longer uses @ai-sdk/google. All models use native @google/genai SDK.",
97
+ category: ErrorCategory.CONFIGURATION,
98
+ severity: ErrorSeverity.CRITICAL,
99
+ retriable: false,
100
+ context: { provider: this.providerName, model: this.modelName },
101
+ });
97
102
  }
98
103
  formatProviderError(error) {
99
104
  if (error instanceof TimeoutError) {
@@ -103,12 +108,53 @@ export class GoogleAIStudioProvider extends BaseProvider {
103
108
  const message = typeof errorRecord?.message === "string"
104
109
  ? errorRecord.message
105
110
  : "Unknown error";
106
- if (message.includes("API_KEY_INVALID")) {
111
+ const statusCode = typeof errorRecord?.status === "number"
112
+ ? errorRecord.status
113
+ : typeof errorRecord?.statusCode === "number"
114
+ ? errorRecord.statusCode
115
+ : undefined;
116
+ // Authentication errors
117
+ if (message.includes("API_KEY_INVALID") ||
118
+ message.includes("Invalid API key") ||
119
+ statusCode === 401) {
107
120
  return new AuthenticationError("Invalid Google AI API key. Please check your GOOGLE_AI_API_KEY environment variable.", this.providerName);
108
121
  }
109
- if (message.includes("RATE_LIMIT_EXCEEDED")) {
122
+ // Rate limit errors
123
+ if (message.includes("RATE_LIMIT_EXCEEDED") ||
124
+ message.includes("rate limit") ||
125
+ message.includes("429") ||
126
+ statusCode === 429) {
110
127
  return new RateLimitError("Google AI rate limit exceeded. Please try again later.", this.providerName);
111
128
  }
129
+ // Model not found errors — gate on a 404 status when available; fall
130
+ // back to literal phrase matching only when we have no status code at
131
+ // all. Avoids misclassifying permission/validation errors that happen
132
+ // to mention model resource paths (e.g. "...models/foo permission...").
133
+ if (statusCode === 404 ||
134
+ (statusCode === undefined &&
135
+ (message.includes("model not found") ||
136
+ message.includes("Model not found")))) {
137
+ return new InvalidModelError(`Model '${this.modelName}' not found. Please check the model name and ensure it is available.`, this.providerName);
138
+ }
139
+ // Network connectivity errors
140
+ if (message.includes("ECONNRESET") ||
141
+ message.includes("ENOTFOUND") ||
142
+ message.includes("ETIMEDOUT") ||
143
+ message.includes("ECONNREFUSED") ||
144
+ message.includes("network") ||
145
+ message.includes("connection")) {
146
+ return new NetworkError(`Connection error: ${message}`, this.providerName);
147
+ }
148
+ // Server errors (5xx)
149
+ if (message.includes("500") ||
150
+ message.includes("502") ||
151
+ message.includes("503") ||
152
+ message.includes("504") ||
153
+ message.includes("server error") ||
154
+ message.includes("Internal Server Error") ||
155
+ (statusCode && statusCode >= 500 && statusCode < 600)) {
156
+ return new ProviderError(`Google AI server error: ${message}. Please try again later.`, this.providerName);
157
+ }
112
158
  return new ProviderError(`Google AI error: ${message}`, this.providerName);
113
159
  }
114
160
  /**
@@ -388,198 +434,49 @@ export class GoogleAIStudioProvider extends BaseProvider {
388
434
  }
389
435
  // executeGenerate removed - BaseProvider handles all generation with tools
390
436
  async executeStream(options, analysisSchema) {
391
- // Check if this is a Gemini 3 model with tools - use native SDK for thought_signature
392
- const gemini3CheckModelName = options.model || this.modelName;
393
- // Structured output (analysisSchema, JSON format, or schema) is incompatible with tools on Gemini.
394
- // Compute once and reuse in both the native Gemini 3 gate and the streamText fallback path.
395
- const wantsStructuredOutput = analysisSchema || options.output?.format === "json" || options.schema;
396
- // Check for tools from options AND from SDK (MCP tools)
397
- // Need to check early if we should route to native SDK
398
- const gemini3CheckShouldUseTools = !options.disableTools && this.supportsTools() && !wantsStructuredOutput;
399
- const tools = options.tools || {};
400
- const hasTools = gemini3CheckShouldUseTools && Object.keys(tools).length > 0;
401
- if (isGemini3Model(gemini3CheckModelName) && hasTools) {
402
- // Merge SDK tools into options for native SDK path
403
- let mergedOptions = {
404
- ...options,
405
- tools: tools,
406
- };
407
- // Check for tools + JSON schema conflict (Gemini limitation)
408
- const wantsJsonOutput = options.output?.format === "json" || options.schema;
409
- if (wantsJsonOutput &&
410
- mergedOptions.tools &&
411
- Object.keys(mergedOptions.tools).length > 0 &&
412
- !mergedOptions.disableTools) {
413
- logger.warn("[GoogleAIStudio] Gemini does not support tools and JSON schema output simultaneously. Disabling tools for this request.");
414
- mergedOptions = { ...mergedOptions, disableTools: true, tools: {} };
415
- }
416
- // Only route to native path if tools are still active after conflict check
417
- const hasActiveTools = !mergedOptions.disableTools &&
418
- mergedOptions.tools &&
419
- Object.keys(mergedOptions.tools).length > 0;
420
- if (hasActiveTools) {
421
- logger.info("[GoogleAIStudio] Routing Gemini 3 to native SDK for tool calling", {
422
- model: gemini3CheckModelName,
423
- totalToolCount: Object.keys(mergedOptions.tools ?? {}).length,
424
- });
425
- return this.executeNativeGemini3Stream(mergedOptions);
426
- }
427
- // Fall through to standard stream path using merged options (tools disabled for schema)
428
- options = mergedOptions;
429
- }
437
+ const modelName = options.model || this.modelName;
430
438
  // Phase 1: if audio input present, bridge to Gemini Live (Studio) using @google/genai
431
439
  if (options.input?.audio) {
432
440
  return await this.executeAudioStreamViaGeminiLive(options);
433
441
  }
434
- this.validateStreamOptions(options);
435
- const startTime = Date.now();
436
- const model = await this.getAISDKModelWithMiddleware(options);
437
- const timeout = this.getTimeout(options);
438
- const timeoutController = createTimeoutController(timeout, this.providerName, "stream");
439
- try {
440
- // Get tools consistently with generate method (include user-provided RAG tools)
441
- // wantsStructuredOutput already computed before the Gemini 3 native-routing gate
442
- if (wantsStructuredOutput &&
443
- !options.disableTools &&
444
- this.supportsTools()) {
445
- logger.warn("[GoogleAIStudio] Structured output active — disabling tools (Gemini limitation).");
446
- }
447
- const shouldUseTools = !options.disableTools && this.supportsTools() && !wantsStructuredOutput;
448
- const filteredTools = shouldUseTools
449
- ? (options.tools ?? {})
450
- : {};
451
- // Sanitize tool schemas for Gemini proto compatibility (converts anyOf/oneOf unions to string)
452
- let tools;
453
- if (Object.keys(filteredTools).length > 0) {
454
- const sanitized = sanitizeToolsForGemini(filteredTools);
455
- if (sanitized.dropped.length > 0) {
456
- logger.warn(`[GoogleAIStudio] Dropped ${sanitized.dropped.length} incompatible tool(s): ${sanitized.dropped.join(", ")}`);
457
- }
458
- tools =
459
- Object.keys(sanitized.tools).length > 0 ? sanitized.tools : undefined;
460
- }
461
- else {
462
- tools = undefined;
463
- }
464
- // Build message array from options with multimodal support
465
- // Using protected helper from BaseProvider to eliminate code duplication
466
- const messages = await this.buildMessagesForStream(options);
467
- const collectedToolCalls = [];
468
- const collectedToolResults = [];
469
- // Reviewer follow-up: capture upstream provider errors via onError
470
- // so the post-stream NoOutput sentinel carries the real cause.
471
- let capturedProviderError;
472
- const result = await streamText({
473
- model,
474
- messages: messages,
475
- temperature: options.temperature,
476
- maxOutputTokens: options.maxTokens, // No default limit - unlimited unless specified
477
- tools,
478
- stopWhen: stepCountIs(options.maxSteps || DEFAULT_MAX_STEPS),
479
- toolChoice: resolveToolChoice(options, tools, shouldUseTools),
480
- abortSignal: composeAbortSignals(options.abortSignal, timeoutController?.controller.signal),
481
- experimental_telemetry: this.telemetryHandler.getTelemetryConfig(options),
482
- experimental_repairToolCall: this.getToolCallRepairFn(options),
483
- onError: (event) => {
484
- capturedProviderError = event.error;
485
- logger.error("GoogleAiStudio: Stream error", {
486
- error: event.error instanceof Error
487
- ? event.error.message
488
- : String(event.error),
489
- });
490
- },
491
- // Gemini 3: use thinkingLevel via providerOptions
492
- // Gemini 2.5: use thinkingBudget via providerOptions
493
- ...(options.thinkingConfig?.enabled && {
494
- providerOptions: {
495
- google: {
496
- thinkingConfig: {
497
- ...(options.thinkingConfig.thinkingLevel && {
498
- thinkingLevel: options.thinkingConfig.thinkingLevel,
499
- }),
500
- ...(options.thinkingConfig.budgetTokens &&
501
- !options.thinkingConfig.thinkingLevel && {
502
- thinkingBudget: options.thinkingConfig.budgetTokens,
503
- }),
504
- includeThoughts: true,
505
- },
506
- },
507
- },
508
- }),
509
- onStepFinish: ({ toolCalls, toolResults }) => {
510
- for (const toolCall of toolCalls) {
511
- collectedToolCalls.push({
512
- toolCallId: toolCall.toolCallId,
513
- toolName: toolCall.toolName,
514
- args: toolCall.args ??
515
- toolCall.input ??
516
- toolCall
517
- .parameters ??
518
- {},
519
- });
520
- }
521
- for (const toolResult of toolResults) {
522
- const rawToolResult = toolResult;
523
- collectedToolResults.push({
524
- toolName: toolResult.toolName,
525
- status: rawToolResult.error ? "failure" : "success",
526
- output: (rawToolResult.output ??
527
- rawToolResult.result) ??
528
- undefined,
529
- error: rawToolResult.error,
530
- id: rawToolResult.toolCallId ?? toolResult.toolName,
531
- });
532
- }
533
- // Emit tool:end for each completed tool result so Pipeline B
534
- // captures telemetry for AI-SDK-driven tool calls (gap S2).
535
- emitToolEndFromStepFinish(this.neurolink?.getEventEmitter(), toolResults);
536
- this.handleToolExecutionStorage(toolCalls, toolResults, options, new Date()).catch((error) => {
537
- logger.warn("[GoogleAiStudioProvider] Failed to store tool executions", {
538
- provider: this.providerName,
539
- error: error instanceof Error ? error.message : String(error),
540
- });
541
- });
542
- },
543
- });
544
- // Defer timeout cleanup until the stream completes or errors.
545
- // Guard against NoOutputGeneratedError becoming an unhandled rejection.
546
- Promise.resolve(result.text)
547
- .catch((err) => {
548
- logger.debug("Stream text promise rejected (expected for empty streams)", {
549
- error: err instanceof Error ? err.message : String(err),
550
- });
551
- })
552
- .finally(() => timeoutController?.cleanup());
553
- // Transform string stream to content object stream using BaseProvider method
554
- const transformedStream = this.createTextStream(result, () => capturedProviderError);
555
- // Create analytics promise that resolves after stream completion
556
- const analyticsPromise = streamAnalyticsCollector.createAnalytics(this.providerName, this.modelName, toAnalyticsStreamResult(result), Date.now() - startTime, {
557
- requestId: `google-ai-stream-${Date.now()}`,
558
- streamingMode: true,
559
- });
560
- return {
561
- stream: transformedStream,
562
- provider: this.providerName,
563
- model: this.modelName,
564
- ...(shouldUseTools && {
565
- toolCalls: collectedToolCalls,
566
- toolResults: collectedToolResults,
567
- }),
568
- analytics: analyticsPromise,
569
- metadata: {
570
- startTime,
571
- streamId: `google-ai-${Date.now()}`,
572
- },
573
- };
442
+ // Structured output (analysisSchema, JSON format, or schema) is incompatible with tools on Gemini.
443
+ const wantsStructuredOutput = analysisSchema || options.output?.format === "json" || options.schema;
444
+ // Tool filter (a0269210): trust options.tools — caller (BaseProvider.stream)
445
+ // already merged MCP/built-in tools with user tools and applied any
446
+ // enabledToolNames filter. Re-attaching getAllTools() here would clobber
447
+ // that filter and re-introduce filtered-out tools.
448
+ const shouldUseTools = !options.disableTools && this.supportsTools() && !wantsStructuredOutput;
449
+ const optionTools = options.tools || {};
450
+ // Merge into options for native SDK path
451
+ let mergedOptions = {
452
+ ...options,
453
+ tools: optionTools,
454
+ };
455
+ // Check for tools + JSON schema conflict (Gemini limitation)
456
+ const wantsJsonOutput = options.output?.format === "json" || options.schema;
457
+ if (wantsJsonOutput &&
458
+ mergedOptions.tools &&
459
+ Object.keys(mergedOptions.tools).length > 0 &&
460
+ !mergedOptions.disableTools) {
461
+ logger.warn("[GoogleAIStudio] Gemini does not support tools and JSON schema output simultaneously. Disabling tools for this request.");
462
+ mergedOptions = { ...mergedOptions, disableTools: true, tools: {} };
574
463
  }
575
- catch (error) {
576
- timeoutController?.cleanup();
577
- throw this.handleProviderError(error);
464
+ const hasActiveTools = shouldUseTools &&
465
+ !mergedOptions.disableTools &&
466
+ mergedOptions.tools &&
467
+ Object.keys(mergedOptions.tools).length > 0;
468
+ if (hasActiveTools) {
469
+ logger.info("[GoogleAIStudio] Routing to native @google/genai SDK for tool calling", {
470
+ model: modelName,
471
+ totalToolCount: Object.keys(mergedOptions.tools ?? {}).length,
472
+ });
578
473
  }
474
+ // Route ALL models through native @google/genai SDK (no more @ai-sdk/google dependency)
475
+ return this.executeNativeGemini3Stream(mergedOptions);
579
476
  }
580
477
  /**
581
- * Execute stream using native @google/genai SDK for Gemini 3 models
582
- * This bypasses @ai-sdk/google to properly handle thought_signature
478
+ * Execute stream using native @google/genai SDK
479
+ * Uses @google/genai directly for all Gemini models (2.0, 2.5, 3.x)
583
480
  */
584
481
  async executeNativeGemini3Stream(options) {
585
482
  const modelName = options.model || this.modelName;
@@ -603,8 +500,23 @@ export class GoogleAIStudioProvider extends BaseProvider {
603
500
  model: modelName,
604
501
  hasTools: !!options.tools && Object.keys(options.tools).length > 0,
605
502
  });
606
- // Build contents from input
607
- const currentContents = [{ role: "user", parts: [{ text: options.input.text }] }];
503
+ // Build contents from input. Prepend prior conversation turns so
504
+ // multi-turn callers (memory, loop REPL, agent flows) actually
505
+ // carry context — the previous build started fresh from the
506
+ // current user input only, which silently dropped history.
507
+ //
508
+ // `buildUserPartsWithMultimodal` is the shared helper that also
509
+ // attaches `input.images` and `input.pdfFiles` as `inlineData`
510
+ // parts. The previous AI Studio path pushed only `{ text }` and
511
+ // silently dropped both, which is why the model legitimately
512
+ // reported "no image attached" on multimodal calls.
513
+ const currentContents = [];
514
+ prependConversationMessages(currentContents, options.conversationMessages);
515
+ const userParts = await buildUserPartsWithMultimodal(options.input, options.input.text, "[GoogleAIStudio:stream]");
516
+ currentContents.push({
517
+ role: "user",
518
+ parts: userParts,
519
+ });
608
520
  // Convert tools
609
521
  let toolsConfig;
610
522
  let executeMap = new Map();
@@ -619,7 +531,21 @@ export class GoogleAIStudioProvider extends BaseProvider {
619
531
  toolNames: toolsConfig[0].functionDeclarations.map((t) => t.name),
620
532
  });
621
533
  }
622
- const config = buildNativeConfig(options, toolsConfig);
534
+ // Native JSON / schema enforcement: when no tools are being sent
535
+ // (the AI Studio orchestrator above already force-disables tools
536
+ // whenever JSON/schema output is requested), enforce the response
537
+ // shape natively via responseMimeType / responseSchema. Without
538
+ // this, JSON output was best-effort prompting only.
539
+ const wantsNativeJson = !toolsConfig &&
540
+ (options.output?.format === "json" || !!options.schema);
541
+ const nativeResponseSchema = wantsNativeJson && options.schema
542
+ ? buildGeminiResponseSchema(options.schema)
543
+ : undefined;
544
+ const config = buildNativeConfig({
545
+ ...options,
546
+ wantsJsonOutput: wantsNativeJson,
547
+ responseSchema: nativeResponseSchema,
548
+ }, toolsConfig);
623
549
  const maxSteps = computeMaxSteps(options.maxSteps);
624
550
  // Compose abort signal from user signal + timeout
625
551
  const composedSignal = composeAbortSignals(options.abortSignal, timeoutController?.controller.signal);
@@ -742,68 +668,9 @@ export class GoogleAIStudioProvider extends BaseProvider {
742
668
  requestDuration: responseTime,
743
669
  timestamp: new Date().toISOString(),
744
670
  });
745
- // Emit generation:end so Pipeline B (Langfuse) creates a GENERATION
746
- // observation. The native @google/genai stream path bypasses the Vercel
747
- // AI SDK so experimental_telemetry is never injected; we emit manually.
748
- const nativeStreamEmitter = this.neurolink?.getEventEmitter();
749
- if (nativeStreamEmitter) {
750
- // Curator P2-4 dedup: flag the per-stream context attached
751
- // to options so the orchestration skips its own emit.
752
- markStreamProviderEmittedGenerationEnd(options);
753
- nativeStreamEmitter.emit("generation:end", {
754
- provider: this.providerName,
755
- responseTime,
756
- timestamp: Date.now(),
757
- result: {
758
- content: "",
759
- usage: {
760
- input: totalInputTokens,
761
- output: totalOutputTokens,
762
- total: totalInputTokens + totalOutputTokens,
763
- },
764
- model: modelName,
765
- provider: this.providerName,
766
- finishReason: hitStepLimitWithoutFinalAnswer
767
- ? "max_steps"
768
- : "stop",
769
- },
770
- success: true,
771
- });
772
- }
773
671
  channel.close();
774
672
  }
775
673
  catch (err) {
776
- // Propagate error to OTel span so traces show ERROR status
777
- span.recordException(err instanceof Error ? err : new Error(String(err)));
778
- span.setStatus({
779
- code: SpanStatusCode.ERROR,
780
- message: err instanceof Error ? err.message : String(err),
781
- });
782
- // Emit failure generation:end so Pipeline B records the failed stream
783
- const errorEmitter = this.neurolink?.getEventEmitter();
784
- if (errorEmitter) {
785
- // Curator P2-4 dedup: flag the per-stream context attached
786
- // to options so the orchestration skips its own emit.
787
- markStreamProviderEmittedGenerationEnd(options);
788
- errorEmitter.emit("generation:end", {
789
- provider: this.providerName,
790
- responseTime: Date.now() - startTime,
791
- timestamp: Date.now(),
792
- result: {
793
- content: "",
794
- usage: {
795
- input: totalInputTokens,
796
- output: totalOutputTokens,
797
- total: totalInputTokens + totalOutputTokens,
798
- },
799
- model: modelName,
800
- provider: this.providerName,
801
- finishReason: "error",
802
- },
803
- success: false,
804
- error: err instanceof Error ? err.message : String(err),
805
- });
806
- }
807
674
  channel.error(err);
808
675
  analyticsReject(err);
809
676
  }
@@ -858,8 +725,21 @@ export class GoogleAIStudioProvider extends BaseProvider {
858
725
  // Prefer input.text over prompt — processCSVFilesForNativeSDK enriches
859
726
  // input.text with inlined CSV data, so using prompt first would discard it.
860
727
  const promptText = options.input?.text || options.prompt || "";
861
- const currentContents = [{ role: "user", parts: [{ text: promptText }] }];
862
- // Convert tools (merge SDK tools with options.tools)
728
+ // Prepend prior conversation turns so multi-turn generate calls
729
+ // see history; otherwise the native generate path silently drops
730
+ // every turn before the current prompt.
731
+ //
732
+ // `buildUserPartsWithMultimodal` also attaches inline image / PDF
733
+ // parts. Without it the request body was text-only and the model
734
+ // legitimately reported "no image / PDF attached".
735
+ const currentContents = [];
736
+ prependConversationMessages(currentContents, options.conversationMessages);
737
+ const userParts = await buildUserPartsWithMultimodal(options.input, promptText, "[GoogleAIStudio:generate]");
738
+ currentContents.push({
739
+ role: "user",
740
+ parts: userParts,
741
+ });
742
+ // Convert tools (a0269210: trust options.tools — already merged + filtered upstream)
863
743
  let toolsConfig;
864
744
  let executeMap = new Map();
865
745
  const shouldUseTools = !options.disableTools;
@@ -875,7 +755,19 @@ export class GoogleAIStudioProvider extends BaseProvider {
875
755
  });
876
756
  }
877
757
  }
878
- const config = buildNativeConfig(options, toolsConfig);
758
+ // Native JSON / schema enforcement (generate path). Mirrors the
759
+ // stream block above; only set when no tools are being sent
760
+ // because Gemini cannot combine function calling with JSON mime.
761
+ const wantsNativeJson = !toolsConfig &&
762
+ (options.output?.format === "json" || !!options.schema);
763
+ const nativeResponseSchema = wantsNativeJson && options.schema
764
+ ? buildGeminiResponseSchema(options.schema)
765
+ : undefined;
766
+ const config = buildNativeConfig({
767
+ ...options,
768
+ wantsJsonOutput: wantsNativeJson,
769
+ responseSchema: nativeResponseSchema,
770
+ }, toolsConfig);
879
771
  const composedSignal = composeAbortSignals(options.abortSignal, timeoutController?.controller.signal);
880
772
  const maxSteps = computeMaxSteps(options.maxSteps);
881
773
  let finalText = "";
@@ -946,31 +838,11 @@ export class GoogleAIStudioProvider extends BaseProvider {
946
838
  span.setAttribute(ATTR.GEN_AI_INPUT_TOKENS, totalInputTokens);
947
839
  span.setAttribute(ATTR.GEN_AI_OUTPUT_TOKENS, totalOutputTokens);
948
840
  span.setAttribute(ATTR.GEN_AI_FINISH_REASON, step >= maxSteps ? "max_steps" : "stop");
949
- // Emit generation:end so Pipeline B (Langfuse) creates a GENERATION
950
- // observation. The native @google/genai path bypasses the Vercel AI SDK
951
- // so experimental_telemetry is never injected; we emit the event manually.
952
- const nativeGenerateEmitter = this.neurolink?.getEventEmitter();
953
- if (nativeGenerateEmitter) {
954
- nativeGenerateEmitter.emit("generation:end", {
955
- provider: this.providerName,
956
- responseTime,
957
- timestamp: Date.now(),
958
- result: {
959
- content: finalText,
960
- usage: {
961
- input: totalInputTokens,
962
- output: totalOutputTokens,
963
- total: totalInputTokens + totalOutputTokens,
964
- },
965
- model: modelName,
966
- provider: this.providerName,
967
- finishReason: step >= maxSteps ? "max_steps" : "stop",
968
- },
969
- success: true,
970
- });
971
- }
972
- // Build EnhancedGenerateResult
973
- return {
841
+ // Build EnhancedGenerateResult and route through enhanceResult so
842
+ // analytics / evaluation / tracing stay attached. The native AI
843
+ // Studio generate path bypasses BaseProvider.generate(), so
844
+ // skipping enhanceResult would silently drop those features.
845
+ const baseResult = {
974
846
  content: finalText,
975
847
  provider: this.providerName,
976
848
  model: modelName,
@@ -984,6 +856,7 @@ export class GoogleAIStudioProvider extends BaseProvider {
984
856
  toolExecutions: toolExecutions,
985
857
  enhancedWithTools: allToolCalls.length > 0,
986
858
  };
859
+ return this.enhanceResult(baseResult, options, startTime);
987
860
  }
988
861
  finally {
989
862
  timeoutController?.cleanup();
@@ -999,40 +872,116 @@ export class GoogleAIStudioProvider extends BaseProvider {
999
872
  ? { prompt: optionsOrPrompt }
1000
873
  : optionsOrPrompt;
1001
874
  const modelName = options.model || this.modelName;
1002
- // Check if we should use native SDK for Gemini 3 with tools
1003
- const shouldUseTools = !options.disableTools && this.supportsTools();
1004
- const hasTools = shouldUseTools && options.tools && Object.keys(options.tools).length > 0;
1005
- if (isGemini3Model(modelName) && hasTools) {
1006
- // Merge SDK tools into options for native SDK path
1007
- let mergedOptions = {
1008
- ...options,
1009
- tools: options.tools,
1010
- };
1011
- // Check for tools + JSON schema conflict (Gemini limitation)
1012
- const wantsJsonOutput = options.output?.format === "json" || options.schema;
1013
- if (wantsJsonOutput &&
1014
- mergedOptions.tools &&
1015
- Object.keys(mergedOptions.tools).length > 0 &&
1016
- !mergedOptions.disableTools) {
1017
- logger.warn("[GoogleAIStudio] Gemini does not support tools and JSON schema output simultaneously. Disabling tools for this request.");
1018
- mergedOptions = { ...mergedOptions, disableTools: true, tools: {} };
875
+ // Image-generation models reject function-calling. Route them to
876
+ // executeImageGeneration without merging tools. This must happen
877
+ // BEFORE getToolsForStream to avoid leaking registered (MCP / built-in)
878
+ // tools into the image API request, which trips
879
+ // "Function calling is not enabled for this model".
880
+ // startsWith (not includes) so a hypothetical text model whose ID
881
+ // contains an image-model string as a substring isn't silently routed
882
+ // to executeImageGeneration and stripped of tool support.
883
+ const isImageModel = IMAGE_GENERATION_MODELS.some((m) => modelName.toLowerCase().startsWith(m.toLowerCase()));
884
+ if (isImageModel) {
885
+ logger.info("[GoogleAIStudio] Routing image generation model to executeImageGeneration", { model: modelName });
886
+ return this.executeImageGeneration(options);
887
+ }
888
+ // Process the unified `input.files` array before routing to the
889
+ // native SDK. BaseProvider.generate() runs this preprocessing via
890
+ // buildMultimodalMessagesArray, but AI Studio's override skips it,
891
+ // which would otherwise drop text-file content (and the
892
+ // mimetype-hint contract) on the floor. Mutates options.input.text /
893
+ // options.input.images / options.input.pdfFiles in place.
894
+ if (options.input?.files && options.input.files.length > 0) {
895
+ try {
896
+ await processUnifiedFilesArray(options, 100 * 1024 * 1024, this.providerName);
1019
897
  }
1020
- // Only route to native path if tools are still active after conflict check
1021
- const hasActiveTools = !mergedOptions.disableTools &&
1022
- mergedOptions.tools &&
1023
- Object.keys(mergedOptions.tools).length > 0;
1024
- if (hasActiveTools) {
1025
- logger.info("[GoogleAIStudio] Routing Gemini 3 generate to native SDK for tool calling", {
1026
- model: modelName,
1027
- totalToolCount: Object.keys(mergedOptions.tools ?? {}).length,
1028
- });
1029
- return this.executeNativeGemini3Generate(mergedOptions);
898
+ catch (fileError) {
899
+ logger.warn(`[GoogleAIStudio] processUnifiedFilesArray threw, continuing without file content: ${fileError instanceof Error ? fileError.message : String(fileError)}`);
1030
900
  }
1031
- // Fall through to standard generate path using merged options (tools disabled for schema)
1032
- return super.generate(mergedOptions);
1033
901
  }
1034
- // Fall back to BaseProvider implementation
1035
- return super.generate(options);
902
+ // Merge registered (built-in / MCP) tools with caller-supplied tools.
903
+ // AI Studio's generate() bypasses BaseProvider.generate(), so the
904
+ // ToolsManager-driven merge that normally injects sdk.registerTool()
905
+ // entries never runs here. Without this call, registered tools never
906
+ // reach the native function-calling path.
907
+ const baseTools = !options.disableTools
908
+ ? await this.getToolsForStream(options)
909
+ : {};
910
+ let mergedOptions = {
911
+ ...options,
912
+ tools: baseTools,
913
+ };
914
+ // Check for tools + JSON schema conflict (Gemini limitation)
915
+ const wantsJsonOutput = options.output?.format === "json" || options.schema;
916
+ if (wantsJsonOutput &&
917
+ mergedOptions.tools &&
918
+ Object.keys(mergedOptions.tools).length > 0 &&
919
+ !mergedOptions.disableTools) {
920
+ logger.warn("[GoogleAIStudio] Gemini does not support tools and JSON schema output simultaneously. Disabling tools for this request.");
921
+ mergedOptions = { ...mergedOptions, disableTools: true, tools: {} };
922
+ }
923
+ const hasActiveTools = !mergedOptions.disableTools &&
924
+ mergedOptions.tools &&
925
+ Object.keys(mergedOptions.tools).length > 0;
926
+ if (hasActiveTools) {
927
+ logger.info("[GoogleAIStudio] Routing generate to native @google/genai SDK for tool calling", {
928
+ model: modelName,
929
+ totalToolCount: Object.keys(mergedOptions.tools ?? {}).length,
930
+ });
931
+ }
932
+ // Route ALL models through native @google/genai SDK (no more @ai-sdk/google dependency).
933
+ // Emit Pipeline B `generation:end` so the observability listener
934
+ // creates a `model.generation` span — AI Studio's native path bypasses
935
+ // the AI SDK + experimental_telemetry plumbing the same way Vertex's
936
+ // does, so the event has to be emitted manually.
937
+ const generateStartTime = Date.now();
938
+ const inputPrompt = mergedOptions.input?.text ||
939
+ mergedOptions.prompt ||
940
+ "";
941
+ try {
942
+ const result = await this.executeNativeGemini3Generate(mergedOptions);
943
+ this.emitPipelineBGenerationEvent(modelName, result, generateStartTime, true, undefined, inputPrompt);
944
+ return result;
945
+ }
946
+ catch (error) {
947
+ this.emitPipelineBGenerationEvent(modelName, null, generateStartTime, false, error, inputPrompt);
948
+ throw error;
949
+ }
950
+ }
951
+ /**
952
+ * Emit `generation:end` so the Pipeline B observability listener creates
953
+ * a `model.generation` span for native Google AI Studio generate calls.
954
+ * Without this hand-off the native path silently disappears from
955
+ * Pipeline B exporters (Langfuse, custom OTEL collectors).
956
+ */
957
+ emitPipelineBGenerationEvent(modelName, result, startTime, success, error, prompt) {
958
+ const emitter = this.neurolink?.getEventEmitter();
959
+ if (!emitter) {
960
+ return;
961
+ }
962
+ const usage = result?.usage && typeof result.usage === "object"
963
+ ? result.usage
964
+ : { input: 0, output: 0, total: 0 };
965
+ emitter.emit("generation:end", {
966
+ provider: this.providerName,
967
+ responseTime: Date.now() - startTime,
968
+ timestamp: Date.now(),
969
+ // The Pipeline B listener reads `data.prompt` to populate the
970
+ // `input` span attribute. Without this, the Observability Spans
971
+ // test fails with "input capture not working".
972
+ prompt: prompt || "",
973
+ result: {
974
+ content: result?.content || "",
975
+ usage,
976
+ model: modelName,
977
+ provider: this.providerName,
978
+ finishReason: success ? "stop" : "error",
979
+ },
980
+ success,
981
+ ...(error
982
+ ? { error: error instanceof Error ? error.message : String(error) }
983
+ : {}),
984
+ });
1036
985
  }
1037
986
  // ===================
1038
987
  // HELPER METHODS
@@ -1240,18 +1189,21 @@ export class GoogleAIStudioProvider extends BaseProvider {
1240
1189
  });
1241
1190
  try {
1242
1191
  const apiKey = this.getApiKey();
1243
- const google = createGoogleGenerativeAI({ apiKey });
1244
- const embeddingModel = google.textEmbeddingModel(embeddingModelName);
1245
- const result = await embed({
1246
- model: embeddingModel,
1247
- value: text,
1192
+ const client = await createGoogleGenAIClient(apiKey);
1193
+ const result = await client.models.embedContent({
1194
+ model: embeddingModelName,
1195
+ contents: [text],
1248
1196
  });
1197
+ const embedding = result.embeddings?.[0]?.values;
1198
+ if (!embedding) {
1199
+ throw new ProviderError("No embedding returned from Google AI", this.providerName);
1200
+ }
1249
1201
  logger.debug("Embedding generated successfully", {
1250
1202
  provider: this.providerName,
1251
1203
  model: embeddingModelName,
1252
- embeddingDimension: result.embedding.length,
1204
+ embeddingDimension: embedding.length,
1253
1205
  });
1254
- return result.embedding;
1206
+ return embedding;
1255
1207
  }
1256
1208
  catch (error) {
1257
1209
  logger.error("Embedding generation failed", {
@@ -1277,19 +1229,19 @@ export class GoogleAIStudioProvider extends BaseProvider {
1277
1229
  });
1278
1230
  try {
1279
1231
  const apiKey = this.getApiKey();
1280
- const google = createGoogleGenerativeAI({ apiKey });
1281
- const embeddingModel = google.textEmbeddingModel(embeddingModelName);
1282
- const result = await embedMany({
1283
- model: embeddingModel,
1284
- values: texts,
1232
+ const client = await createGoogleGenAIClient(apiKey);
1233
+ const result = await client.models.embedContent({
1234
+ model: embeddingModelName,
1235
+ contents: texts,
1285
1236
  });
1237
+ const embeddings = (result.embeddings || []).map((e) => e.values || []);
1286
1238
  logger.debug("Batch embeddings generated successfully", {
1287
1239
  provider: this.providerName,
1288
1240
  model: embeddingModelName,
1289
- count: result.embeddings.length,
1290
- embeddingDimension: result.embeddings[0]?.length,
1241
+ count: embeddings.length,
1242
+ embeddingDimension: embeddings[0]?.length,
1291
1243
  });
1292
- return result.embeddings;
1244
+ return embeddings;
1293
1245
  }
1294
1246
  catch (error) {
1295
1247
  logger.error("Batch embedding generation failed", {