@juspay/neurolink 9.22.2 → 9.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/adapters/video/directorPipeline.d.ts +31 -0
- package/dist/adapters/video/directorPipeline.js +516 -0
- package/dist/adapters/video/ffmpegAdapter.d.ts +78 -0
- package/dist/adapters/video/ffmpegAdapter.js +206 -0
- package/dist/adapters/video/frameExtractor.d.ts +28 -0
- package/dist/adapters/video/frameExtractor.js +143 -0
- package/dist/adapters/video/vertexVideoHandler.d.ts +25 -25
- package/dist/adapters/video/vertexVideoHandler.js +173 -42
- package/dist/adapters/video/videoMerger.d.ts +22 -0
- package/dist/adapters/video/videoMerger.js +171 -0
- package/dist/constants/index.d.ts +1 -0
- package/dist/constants/index.js +2 -0
- package/dist/constants/videoErrors.d.ts +45 -0
- package/dist/constants/videoErrors.js +46 -0
- package/dist/core/baseProvider.js +42 -1
- package/dist/lib/adapters/video/directorPipeline.d.ts +31 -0
- package/dist/lib/adapters/video/directorPipeline.js +517 -0
- package/dist/lib/adapters/video/ffmpegAdapter.d.ts +78 -0
- package/dist/lib/adapters/video/ffmpegAdapter.js +207 -0
- package/dist/lib/adapters/video/frameExtractor.d.ts +28 -0
- package/dist/lib/adapters/video/frameExtractor.js +144 -0
- package/dist/lib/adapters/video/vertexVideoHandler.d.ts +25 -25
- package/dist/lib/adapters/video/vertexVideoHandler.js +173 -42
- package/dist/lib/adapters/video/videoMerger.d.ts +22 -0
- package/dist/lib/adapters/video/videoMerger.js +172 -0
- package/dist/lib/constants/index.d.ts +1 -0
- package/dist/lib/constants/index.js +2 -0
- package/dist/lib/constants/videoErrors.d.ts +45 -0
- package/dist/lib/constants/videoErrors.js +47 -0
- package/dist/lib/core/baseProvider.js +42 -1
- package/dist/lib/types/content.d.ts +1 -1
- package/dist/lib/types/generateTypes.d.ts +18 -1
- package/dist/lib/types/multimodal.d.ts +64 -4
- package/dist/lib/types/multimodal.js +36 -1
- package/dist/lib/utils/parameterValidation.d.ts +8 -1
- package/dist/lib/utils/parameterValidation.js +80 -1
- package/dist/types/content.d.ts +1 -1
- package/dist/types/generateTypes.d.ts +18 -1
- package/dist/types/multimodal.d.ts +64 -4
- package/dist/types/multimodal.js +36 -1
- package/dist/utils/parameterValidation.d.ts +8 -1
- package/dist/utils/parameterValidation.js +80 -1
- package/package.json +1 -1
|
@@ -7,7 +7,7 @@ import type { Content, ImageWithAltText } from "./content.js";
|
|
|
7
7
|
import type { ChatMessage, ConversationMemoryConfig } from "./conversation.js";
|
|
8
8
|
import type { EvaluationData } from "./evaluation.js";
|
|
9
9
|
import type { MiddlewareFactoryOptions } from "./middlewareTypes.js";
|
|
10
|
-
import type { VideoGenerationResult, VideoOutputOptions } from "./multimodal.js";
|
|
10
|
+
import type { DirectorModeOptions, DirectorSegment, VideoGenerationResult, VideoOutputOptions } from "./multimodal.js";
|
|
11
11
|
import type { PPTGenerationResult, PPTOutputOptions } from "./pptTypes.js";
|
|
12
12
|
import type { TTSOptions, TTSResult } from "./ttsTypes.js";
|
|
13
13
|
import type { StandardRecord, ValidationSchema, ZodUnknownSchema } from "./typeAliases.js";
|
|
@@ -41,6 +41,12 @@ export type GenerateOptions = {
|
|
|
41
41
|
videoFiles?: Array<Buffer | string>;
|
|
42
42
|
files?: Array<Buffer | string | import("./fileTypes.js").FileWithMetadata>;
|
|
43
43
|
content?: Content[];
|
|
44
|
+
/**
|
|
45
|
+
* Director Mode segments. When provided, Director Mode is activated automatically.
|
|
46
|
+
* Each segment contains its own prompt and image.
|
|
47
|
+
* Must contain 2-10 segments.
|
|
48
|
+
*/
|
|
49
|
+
segments?: DirectorSegment[];
|
|
44
50
|
};
|
|
45
51
|
/**
|
|
46
52
|
* Output configuration options
|
|
@@ -83,6 +89,11 @@ export type GenerateOptions = {
|
|
|
83
89
|
* Generates slides based on text prompt
|
|
84
90
|
*/
|
|
85
91
|
ppt?: PPTOutputOptions;
|
|
92
|
+
/**
|
|
93
|
+
* Director Mode configuration (only used when input.segments is provided)
|
|
94
|
+
* Controls transition prompts, durations, and concurrency.
|
|
95
|
+
*/
|
|
96
|
+
director?: DirectorModeOptions;
|
|
86
97
|
};
|
|
87
98
|
csvOptions?: {
|
|
88
99
|
maxRows?: number;
|
|
@@ -591,6 +602,8 @@ export type TextGenerationOptions = {
|
|
|
591
602
|
images?: Array<Buffer | string | import("./content.js").ImageWithAltText>;
|
|
592
603
|
pdfFiles?: Array<Buffer | string>;
|
|
593
604
|
files?: Array<Buffer | string | import("./fileTypes.js").FileWithMetadata>;
|
|
605
|
+
/** Director Mode segments (2-10). When provided, Director Mode is activated. */
|
|
606
|
+
segments?: import("./multimodal.js").DirectorSegment[];
|
|
594
607
|
};
|
|
595
608
|
provider?: AIProviderName;
|
|
596
609
|
model?: string;
|
|
@@ -627,6 +640,10 @@ export type TextGenerationOptions = {
|
|
|
627
640
|
* PowerPoint generation configuration (used when mode is "ppt")
|
|
628
641
|
*/
|
|
629
642
|
ppt?: PPTOutputOptions;
|
|
643
|
+
/**
|
|
644
|
+
* Director Mode configuration (only used when input.segments is provided)
|
|
645
|
+
*/
|
|
646
|
+
director?: DirectorModeOptions;
|
|
630
647
|
};
|
|
631
648
|
tools?: Record<string, Tool>;
|
|
632
649
|
timeout?: number | string;
|
|
@@ -150,6 +150,34 @@ export type VideoOutputOptions = {
|
|
|
150
150
|
/** Enable audio generation (default: true) */
|
|
151
151
|
audio?: boolean;
|
|
152
152
|
};
|
|
153
|
+
/**
|
|
154
|
+
* A single segment in Director Mode, representing one video clip.
|
|
155
|
+
*/
|
|
156
|
+
export type DirectorSegment = {
|
|
157
|
+
/** Prompt describing the video content for this segment */
|
|
158
|
+
prompt: string;
|
|
159
|
+
/** Input image for this segment (Buffer, URL string, file path, or ImageWithAltText) */
|
|
160
|
+
image: Buffer | string | ImageWithAltText;
|
|
161
|
+
};
|
|
162
|
+
/**
|
|
163
|
+
* Director Mode configuration options.
|
|
164
|
+
* Used when `input.segments` is provided to control transition generation.
|
|
165
|
+
*/
|
|
166
|
+
export type DirectorModeOptions = {
|
|
167
|
+
/**
|
|
168
|
+
* Prompts for generating transition clips (array of N-1 entries for N segments).
|
|
169
|
+
* transitionPrompts[i] is used for the transition between segment i and segment i+1.
|
|
170
|
+
* If omitted, defaults to "Smooth cinematic transition between scenes".
|
|
171
|
+
*/
|
|
172
|
+
transitionPrompts?: string[];
|
|
173
|
+
/**
|
|
174
|
+
* Duration of each transition clip in seconds (array of N-1 entries for N segments).
|
|
175
|
+
* Each value must be 4, 6, or 8 (4 recommended for seamless feel).
|
|
176
|
+
* If omitted, all transitions default to 4 seconds.
|
|
177
|
+
* @default [4, 4, ...]
|
|
178
|
+
*/
|
|
179
|
+
transitionDurations?: Array<4 | 6 | 8>;
|
|
180
|
+
};
|
|
153
181
|
/**
|
|
154
182
|
* Result type for generated video content
|
|
155
183
|
*
|
|
@@ -201,6 +229,27 @@ export type VideoGenerationResult = {
|
|
|
201
229
|
audioEnabled?: boolean;
|
|
202
230
|
/** Processing time in milliseconds */
|
|
203
231
|
processingTime?: number;
|
|
232
|
+
/** Number of main segments in the video */
|
|
233
|
+
segmentCount?: number;
|
|
234
|
+
/** Number of transition clips generated */
|
|
235
|
+
transitionCount?: number;
|
|
236
|
+
/** Duration of each main clip in seconds */
|
|
237
|
+
clipDuration?: number;
|
|
238
|
+
/** Durations of each transition in seconds (one per transition) */
|
|
239
|
+
transitionDurations?: number[];
|
|
240
|
+
/** Per-segment metadata */
|
|
241
|
+
segments?: Array<{
|
|
242
|
+
index: number;
|
|
243
|
+
duration: number;
|
|
244
|
+
processingTime: number;
|
|
245
|
+
}>;
|
|
246
|
+
/** Per-transition metadata */
|
|
247
|
+
transitions?: Array<{
|
|
248
|
+
fromSegment: number;
|
|
249
|
+
toSegment: number;
|
|
250
|
+
duration: number;
|
|
251
|
+
processingTime: number;
|
|
252
|
+
}>;
|
|
204
253
|
};
|
|
205
254
|
};
|
|
206
255
|
/**
|
|
@@ -295,6 +344,21 @@ export type MultimodalInput = {
|
|
|
295
344
|
audioFiles?: Array<Buffer | string>;
|
|
296
345
|
/** Video files for file-based video processing (future) */
|
|
297
346
|
videoFiles?: Array<Buffer | string>;
|
|
347
|
+
/**
|
|
348
|
+
* Director Mode segments for multi-clip video generation.
|
|
349
|
+
* Each segment contains a prompt and image for generating one video clip.
|
|
350
|
+
* Automatically enables Director Mode when provided.
|
|
351
|
+
*
|
|
352
|
+
* @example
|
|
353
|
+
* ```typescript
|
|
354
|
+
* segments: [
|
|
355
|
+
* { prompt: "Product reveal", image: imageBuffer1 },
|
|
356
|
+
* { prompt: "Feature showcase", image: "./image2.jpg" },
|
|
357
|
+
* { prompt: "Call to action", image: { data: imageBuffer3, altText: "CTA" } }
|
|
358
|
+
* ]
|
|
359
|
+
* ```
|
|
360
|
+
*/
|
|
361
|
+
segments?: DirectorSegment[];
|
|
298
362
|
};
|
|
299
363
|
/**
|
|
300
364
|
* Content format for multimodal messages (used internally)
|
|
@@ -389,10 +453,6 @@ export declare function isAudioContent(content: Content): content is AudioConten
|
|
|
389
453
|
* Type guard to check if content is VideoContent
|
|
390
454
|
*/
|
|
391
455
|
export declare function isVideoContent(content: Content): content is VideoContent;
|
|
392
|
-
/**
|
|
393
|
-
* Type guard to check if input contains multimodal content
|
|
394
|
-
* Now includes audio and video detection
|
|
395
|
-
*/
|
|
396
456
|
export declare function isMultimodalInput(input: unknown): input is MultimodalInput;
|
|
397
457
|
/**
|
|
398
458
|
* Type guard to check if message content is multimodal (array)
|
|
@@ -82,6 +82,38 @@ export function isVideoContent(content) {
|
|
|
82
82
|
* Type guard to check if input contains multimodal content
|
|
83
83
|
* Now includes audio and video detection
|
|
84
84
|
*/
|
|
85
|
+
/**
|
|
86
|
+
* Type guard to validate if an object matches the DirectorSegment shape.
|
|
87
|
+
* Checks for required prompt (string) and image (Buffer, string, or ImageWithAltText).
|
|
88
|
+
*/
|
|
89
|
+
function isDirectorSegment(segment) {
|
|
90
|
+
if (!segment || typeof segment !== "object") {
|
|
91
|
+
return false;
|
|
92
|
+
}
|
|
93
|
+
const maybeSegment = segment;
|
|
94
|
+
// Check for required prompt field
|
|
95
|
+
if (typeof maybeSegment.prompt !== "string" || !maybeSegment.prompt) {
|
|
96
|
+
return false;
|
|
97
|
+
}
|
|
98
|
+
// Check for required image field
|
|
99
|
+
const { image } = maybeSegment;
|
|
100
|
+
if (!image) {
|
|
101
|
+
return false;
|
|
102
|
+
}
|
|
103
|
+
// Validate image type: Buffer, string (URL/path), or ImageWithAltText
|
|
104
|
+
if (Buffer.isBuffer(image)) {
|
|
105
|
+
return true;
|
|
106
|
+
}
|
|
107
|
+
if (typeof image === "string") {
|
|
108
|
+
return true;
|
|
109
|
+
}
|
|
110
|
+
// Check for ImageWithAltText structure
|
|
111
|
+
if (typeof image === "object" && "data" in image) {
|
|
112
|
+
const imgData = image.data;
|
|
113
|
+
return Buffer.isBuffer(imgData) || typeof imgData === "string";
|
|
114
|
+
}
|
|
115
|
+
return false;
|
|
116
|
+
}
|
|
85
117
|
export function isMultimodalInput(input) {
|
|
86
118
|
const maybeInput = input;
|
|
87
119
|
return !!(maybeInput?.images?.length ||
|
|
@@ -90,7 +122,10 @@ export function isMultimodalInput(input) {
|
|
|
90
122
|
maybeInput?.files?.length ||
|
|
91
123
|
maybeInput?.content?.length ||
|
|
92
124
|
maybeInput?.audioFiles?.length ||
|
|
93
|
-
maybeInput?.videoFiles?.length
|
|
125
|
+
maybeInput?.videoFiles?.length ||
|
|
126
|
+
(maybeInput?.segments?.length &&
|
|
127
|
+
Array.isArray(maybeInput.segments) &&
|
|
128
|
+
maybeInput.segments.every(isDirectorSegment)));
|
|
94
129
|
}
|
|
95
130
|
/**
|
|
96
131
|
* Type guard to check if message content is multimodal (array)
|
|
@@ -8,7 +8,7 @@ import type { VideoOutputOptions } from "../types/multimodal.js";
|
|
|
8
8
|
import type { PPTOutputOptions } from "../types/pptTypes.js";
|
|
9
9
|
import type { EnhancedValidationResult } from "../types/tools.js";
|
|
10
10
|
import type { StringArray, ValidationSchema } from "../types/typeAliases.js";
|
|
11
|
-
import { NeuroLinkError } from "./errorHandling.js";
|
|
11
|
+
import { type NeuroLinkError } from "./errorHandling.js";
|
|
12
12
|
/**
|
|
13
13
|
* Custom error class for parameter validation failures
|
|
14
14
|
* Provides detailed information about validation errors including field context and suggestions
|
|
@@ -134,6 +134,13 @@ export declare function validateImageForVideo(image: Buffer | string, maxSize?:
|
|
|
134
134
|
* ```
|
|
135
135
|
*/
|
|
136
136
|
export declare function validateVideoGenerationInput(options: GenerateOptions): EnhancedValidationResult;
|
|
137
|
+
/**
|
|
138
|
+
* Validate Director Mode input: segments, transition prompts, and durations.
|
|
139
|
+
*
|
|
140
|
+
* @param options - GenerateOptions with input.segments and output.director
|
|
141
|
+
* @returns EnhancedValidationResult with errors, warnings, and suggestions
|
|
142
|
+
*/
|
|
143
|
+
export declare function validateDirectorModeInput(options: GenerateOptions): EnhancedValidationResult;
|
|
137
144
|
export declare const MIN_PPT_PAGES = 5;
|
|
138
145
|
export declare const MAX_PPT_PAGES = 50;
|
|
139
146
|
export declare const MIN_PPT_PROMPT_LENGTH = 10;
|
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
* Parameter Validation Utilities
|
|
3
3
|
* Provides consistent parameter validation across all tool interfaces
|
|
4
4
|
*/
|
|
5
|
+
import { VIDEO_ERROR_CODES } from "../constants/videoErrors.js";
|
|
5
6
|
import { SYSTEM_LIMITS } from "../core/constants.js";
|
|
6
|
-
import { ErrorFactory
|
|
7
|
+
import { ErrorFactory } from "./errorHandling.js";
|
|
7
8
|
import { isNonNullObject } from "./typeUtils.js";
|
|
8
9
|
// ============================================================================
|
|
9
10
|
// VALIDATION ERROR TYPES
|
|
@@ -606,6 +607,84 @@ export function validateVideoGenerationInput(options) {
|
|
|
606
607
|
return { isValid: errors.length === 0, errors, warnings, suggestions };
|
|
607
608
|
}
|
|
608
609
|
// ============================================================================
|
|
610
|
+
// DIRECTOR MODE VALIDATION
|
|
611
|
+
// ============================================================================
|
|
612
|
+
const MIN_DIRECTOR_SEGMENTS = 2;
|
|
613
|
+
const MAX_DIRECTOR_SEGMENTS = 10;
|
|
614
|
+
const VALID_TRANSITION_DURATIONS = [4, 6, 8];
|
|
615
|
+
/**
|
|
616
|
+
* Validate Director Mode input: segments, transition prompts, and durations.
|
|
617
|
+
*
|
|
618
|
+
* @param options - GenerateOptions with input.segments and output.director
|
|
619
|
+
* @returns EnhancedValidationResult with errors, warnings, and suggestions
|
|
620
|
+
*/
|
|
621
|
+
export function validateDirectorModeInput(options) {
|
|
622
|
+
const errors = [];
|
|
623
|
+
const warnings = [];
|
|
624
|
+
const suggestions = [];
|
|
625
|
+
const segments = options.input?.segments;
|
|
626
|
+
if (!segments || !Array.isArray(segments)) {
|
|
627
|
+
errors.push(new ValidationError("Director Mode requires an input.segments array", "input.segments", VIDEO_ERROR_CODES.DIRECTOR_SEGMENT_MISMATCH));
|
|
628
|
+
return { isValid: false, errors, warnings, suggestions };
|
|
629
|
+
}
|
|
630
|
+
if (segments.length < MIN_DIRECTOR_SEGMENTS ||
|
|
631
|
+
segments.length > MAX_DIRECTOR_SEGMENTS) {
|
|
632
|
+
errors.push(new ValidationError(`Director Mode requires ${MIN_DIRECTOR_SEGMENTS}-${MAX_DIRECTOR_SEGMENTS} segments, got ${segments.length}`, "input.segments", segments.length > MAX_DIRECTOR_SEGMENTS
|
|
633
|
+
? VIDEO_ERROR_CODES.DIRECTOR_SEGMENT_LIMIT_EXCEEDED
|
|
634
|
+
: VIDEO_ERROR_CODES.DIRECTOR_SEGMENT_MISMATCH));
|
|
635
|
+
return { isValid: false, errors, warnings, suggestions };
|
|
636
|
+
}
|
|
637
|
+
// Validate each segment
|
|
638
|
+
for (let i = 0; i < segments.length; i++) {
|
|
639
|
+
const seg = segments[i];
|
|
640
|
+
if (!seg || typeof seg !== "object") {
|
|
641
|
+
errors.push(new ValidationError(`Segment ${i} must be an object with prompt and image`, `input.segments[${i}]`, VIDEO_ERROR_CODES.DIRECTOR_SEGMENT_MISMATCH));
|
|
642
|
+
continue;
|
|
643
|
+
}
|
|
644
|
+
if (!seg.prompt ||
|
|
645
|
+
typeof seg.prompt !== "string" ||
|
|
646
|
+
seg.prompt.trim().length === 0) {
|
|
647
|
+
errors.push(new ValidationError(`Segment ${i} requires a non-empty prompt`, `input.segments[${i}].prompt`, VIDEO_ERROR_CODES.DIRECTOR_SEGMENT_MISMATCH));
|
|
648
|
+
}
|
|
649
|
+
if (seg.image === undefined || seg.image === null) {
|
|
650
|
+
errors.push(new ValidationError(`Segment ${i} requires a valid image (Buffer, URL, path, or ImageWithAltText)`, `input.segments[${i}].image`, VIDEO_ERROR_CODES.DIRECTOR_SEGMENT_MISMATCH));
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
// Validate director options
|
|
654
|
+
const director = options.output?.director;
|
|
655
|
+
const expectedTransitions = segments.length - 1;
|
|
656
|
+
if (director?.transitionPrompts) {
|
|
657
|
+
if (director.transitionPrompts.length !== expectedTransitions) {
|
|
658
|
+
errors.push(new ValidationError(`Expected ${expectedTransitions} transition prompts, got ${director.transitionPrompts.length}`, "output.director.transitionPrompts", VIDEO_ERROR_CODES.DIRECTOR_SEGMENT_MISMATCH));
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
if (director?.transitionDurations) {
|
|
662
|
+
if (director.transitionDurations.length !== expectedTransitions) {
|
|
663
|
+
errors.push(new ValidationError(`Expected ${expectedTransitions} transition durations, got ${director.transitionDurations.length}`, "output.director.transitionDurations", VIDEO_ERROR_CODES.DIRECTOR_SEGMENT_MISMATCH));
|
|
664
|
+
}
|
|
665
|
+
else {
|
|
666
|
+
for (let i = 0; i < director.transitionDurations.length; i++) {
|
|
667
|
+
const d = director.transitionDurations[i];
|
|
668
|
+
if (!VALID_TRANSITION_DURATIONS.includes(d)) {
|
|
669
|
+
errors.push(new ValidationError(`Invalid transition duration at index ${i}: ${d}. Use 4, 6, or 8`, `output.director.transitionDurations[${i}]`, VIDEO_ERROR_CODES.DIRECTOR_INVALID_TRANSITION_DURATION));
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
// Validate video output options if provided
|
|
675
|
+
if (options.output?.video) {
|
|
676
|
+
const videoError = validateVideoOutputOptions(options.output.video);
|
|
677
|
+
if (videoError) {
|
|
678
|
+
errors.push(toValidationError(videoError));
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
if (errors.length === 0) {
|
|
682
|
+
const totalCalls = segments.length + expectedTransitions;
|
|
683
|
+
suggestions.push(`Director Mode will make ${totalCalls} API calls (${segments.length} clips + ${expectedTransitions} transitions). Ensure adequate timeout.`);
|
|
684
|
+
}
|
|
685
|
+
return { isValid: errors.length === 0, errors, warnings, suggestions };
|
|
686
|
+
}
|
|
687
|
+
// ============================================================================
|
|
609
688
|
// PPT VALIDATION (Presentation Generation)
|
|
610
689
|
// ============================================================================
|
|
611
690
|
/**
|
package/dist/types/content.d.ts
CHANGED
|
@@ -14,5 +14,5 @@
|
|
|
14
14
|
* import type { MultimodalInput } from './types/multimodal.js';
|
|
15
15
|
* ```
|
|
16
16
|
*/
|
|
17
|
-
export type { TextContent, ImageContent, CSVContent, PDFContent, AudioContent, VideoContent, VideoOutputOptions, VideoGenerationResult, Content, ImageWithAltText, MultimodalInput, MultimodalMessage, VisionCapability, ProviderImageFormat, ProcessedImage, ProviderMultimodalPayload, } from "./multimodal.js";
|
|
17
|
+
export type { TextContent, ImageContent, CSVContent, PDFContent, AudioContent, VideoContent, VideoOutputOptions, VideoGenerationResult, DirectorSegment, DirectorModeOptions, Content, ImageWithAltText, MultimodalInput, MultimodalMessage, VisionCapability, ProviderImageFormat, ProcessedImage, ProviderMultimodalPayload, } from "./multimodal.js";
|
|
18
18
|
export { isTextContent, isImageContent, isCSVContent, isPDFContent, isAudioContent, isVideoContent, isMultimodalInput, } from "./multimodal.js";
|
|
@@ -7,7 +7,7 @@ import type { Content, ImageWithAltText } from "./content.js";
|
|
|
7
7
|
import type { ChatMessage, ConversationMemoryConfig } from "./conversation.js";
|
|
8
8
|
import type { EvaluationData } from "./evaluation.js";
|
|
9
9
|
import type { MiddlewareFactoryOptions } from "./middlewareTypes.js";
|
|
10
|
-
import type { VideoGenerationResult, VideoOutputOptions } from "./multimodal.js";
|
|
10
|
+
import type { DirectorModeOptions, DirectorSegment, VideoGenerationResult, VideoOutputOptions } from "./multimodal.js";
|
|
11
11
|
import type { PPTGenerationResult, PPTOutputOptions } from "./pptTypes.js";
|
|
12
12
|
import type { TTSOptions, TTSResult } from "./ttsTypes.js";
|
|
13
13
|
import type { StandardRecord, ValidationSchema, ZodUnknownSchema } from "./typeAliases.js";
|
|
@@ -41,6 +41,12 @@ export type GenerateOptions = {
|
|
|
41
41
|
videoFiles?: Array<Buffer | string>;
|
|
42
42
|
files?: Array<Buffer | string | import("./fileTypes.js").FileWithMetadata>;
|
|
43
43
|
content?: Content[];
|
|
44
|
+
/**
|
|
45
|
+
* Director Mode segments. When provided, Director Mode is activated automatically.
|
|
46
|
+
* Each segment contains its own prompt and image.
|
|
47
|
+
* Must contain 2-10 segments.
|
|
48
|
+
*/
|
|
49
|
+
segments?: DirectorSegment[];
|
|
44
50
|
};
|
|
45
51
|
/**
|
|
46
52
|
* Output configuration options
|
|
@@ -83,6 +89,11 @@ export type GenerateOptions = {
|
|
|
83
89
|
* Generates slides based on text prompt
|
|
84
90
|
*/
|
|
85
91
|
ppt?: PPTOutputOptions;
|
|
92
|
+
/**
|
|
93
|
+
* Director Mode configuration (only used when input.segments is provided)
|
|
94
|
+
* Controls transition prompts, durations, and concurrency.
|
|
95
|
+
*/
|
|
96
|
+
director?: DirectorModeOptions;
|
|
86
97
|
};
|
|
87
98
|
csvOptions?: {
|
|
88
99
|
maxRows?: number;
|
|
@@ -591,6 +602,8 @@ export type TextGenerationOptions = {
|
|
|
591
602
|
images?: Array<Buffer | string | import("./content.js").ImageWithAltText>;
|
|
592
603
|
pdfFiles?: Array<Buffer | string>;
|
|
593
604
|
files?: Array<Buffer | string | import("./fileTypes.js").FileWithMetadata>;
|
|
605
|
+
/** Director Mode segments (2-10). When provided, Director Mode is activated. */
|
|
606
|
+
segments?: import("./multimodal.js").DirectorSegment[];
|
|
594
607
|
};
|
|
595
608
|
provider?: AIProviderName;
|
|
596
609
|
model?: string;
|
|
@@ -627,6 +640,10 @@ export type TextGenerationOptions = {
|
|
|
627
640
|
* PowerPoint generation configuration (used when mode is "ppt")
|
|
628
641
|
*/
|
|
629
642
|
ppt?: PPTOutputOptions;
|
|
643
|
+
/**
|
|
644
|
+
* Director Mode configuration (only used when input.segments is provided)
|
|
645
|
+
*/
|
|
646
|
+
director?: DirectorModeOptions;
|
|
630
647
|
};
|
|
631
648
|
tools?: Record<string, Tool>;
|
|
632
649
|
timeout?: number | string;
|
|
@@ -150,6 +150,34 @@ export type VideoOutputOptions = {
|
|
|
150
150
|
/** Enable audio generation (default: true) */
|
|
151
151
|
audio?: boolean;
|
|
152
152
|
};
|
|
153
|
+
/**
|
|
154
|
+
* A single segment in Director Mode, representing one video clip.
|
|
155
|
+
*/
|
|
156
|
+
export type DirectorSegment = {
|
|
157
|
+
/** Prompt describing the video content for this segment */
|
|
158
|
+
prompt: string;
|
|
159
|
+
/** Input image for this segment (Buffer, URL string, file path, or ImageWithAltText) */
|
|
160
|
+
image: Buffer | string | ImageWithAltText;
|
|
161
|
+
};
|
|
162
|
+
/**
|
|
163
|
+
* Director Mode configuration options.
|
|
164
|
+
* Used when `input.segments` is provided to control transition generation.
|
|
165
|
+
*/
|
|
166
|
+
export type DirectorModeOptions = {
|
|
167
|
+
/**
|
|
168
|
+
* Prompts for generating transition clips (array of N-1 entries for N segments).
|
|
169
|
+
* transitionPrompts[i] is used for the transition between segment i and segment i+1.
|
|
170
|
+
* If omitted, defaults to "Smooth cinematic transition between scenes".
|
|
171
|
+
*/
|
|
172
|
+
transitionPrompts?: string[];
|
|
173
|
+
/**
|
|
174
|
+
* Duration of each transition clip in seconds (array of N-1 entries for N segments).
|
|
175
|
+
* Each value must be 4, 6, or 8 (4 recommended for seamless feel).
|
|
176
|
+
* If omitted, all transitions default to 4 seconds.
|
|
177
|
+
* @default [4, 4, ...]
|
|
178
|
+
*/
|
|
179
|
+
transitionDurations?: Array<4 | 6 | 8>;
|
|
180
|
+
};
|
|
153
181
|
/**
|
|
154
182
|
* Result type for generated video content
|
|
155
183
|
*
|
|
@@ -201,6 +229,27 @@ export type VideoGenerationResult = {
|
|
|
201
229
|
audioEnabled?: boolean;
|
|
202
230
|
/** Processing time in milliseconds */
|
|
203
231
|
processingTime?: number;
|
|
232
|
+
/** Number of main segments in the video */
|
|
233
|
+
segmentCount?: number;
|
|
234
|
+
/** Number of transition clips generated */
|
|
235
|
+
transitionCount?: number;
|
|
236
|
+
/** Duration of each main clip in seconds */
|
|
237
|
+
clipDuration?: number;
|
|
238
|
+
/** Durations of each transition in seconds (one per transition) */
|
|
239
|
+
transitionDurations?: number[];
|
|
240
|
+
/** Per-segment metadata */
|
|
241
|
+
segments?: Array<{
|
|
242
|
+
index: number;
|
|
243
|
+
duration: number;
|
|
244
|
+
processingTime: number;
|
|
245
|
+
}>;
|
|
246
|
+
/** Per-transition metadata */
|
|
247
|
+
transitions?: Array<{
|
|
248
|
+
fromSegment: number;
|
|
249
|
+
toSegment: number;
|
|
250
|
+
duration: number;
|
|
251
|
+
processingTime: number;
|
|
252
|
+
}>;
|
|
204
253
|
};
|
|
205
254
|
};
|
|
206
255
|
/**
|
|
@@ -295,6 +344,21 @@ export type MultimodalInput = {
|
|
|
295
344
|
audioFiles?: Array<Buffer | string>;
|
|
296
345
|
/** Video files for file-based video processing (future) */
|
|
297
346
|
videoFiles?: Array<Buffer | string>;
|
|
347
|
+
/**
|
|
348
|
+
* Director Mode segments for multi-clip video generation.
|
|
349
|
+
* Each segment contains a prompt and image for generating one video clip.
|
|
350
|
+
* Automatically enables Director Mode when provided.
|
|
351
|
+
*
|
|
352
|
+
* @example
|
|
353
|
+
* ```typescript
|
|
354
|
+
* segments: [
|
|
355
|
+
* { prompt: "Product reveal", image: imageBuffer1 },
|
|
356
|
+
* { prompt: "Feature showcase", image: "./image2.jpg" },
|
|
357
|
+
* { prompt: "Call to action", image: { data: imageBuffer3, altText: "CTA" } }
|
|
358
|
+
* ]
|
|
359
|
+
* ```
|
|
360
|
+
*/
|
|
361
|
+
segments?: DirectorSegment[];
|
|
298
362
|
};
|
|
299
363
|
/**
|
|
300
364
|
* Content format for multimodal messages (used internally)
|
|
@@ -389,10 +453,6 @@ export declare function isAudioContent(content: Content): content is AudioConten
|
|
|
389
453
|
* Type guard to check if content is VideoContent
|
|
390
454
|
*/
|
|
391
455
|
export declare function isVideoContent(content: Content): content is VideoContent;
|
|
392
|
-
/**
|
|
393
|
-
* Type guard to check if input contains multimodal content
|
|
394
|
-
* Now includes audio and video detection
|
|
395
|
-
*/
|
|
396
456
|
export declare function isMultimodalInput(input: unknown): input is MultimodalInput;
|
|
397
457
|
/**
|
|
398
458
|
* Type guard to check if message content is multimodal (array)
|
package/dist/types/multimodal.js
CHANGED
|
@@ -82,6 +82,38 @@ export function isVideoContent(content) {
|
|
|
82
82
|
* Type guard to check if input contains multimodal content
|
|
83
83
|
* Now includes audio and video detection
|
|
84
84
|
*/
|
|
85
|
+
/**
|
|
86
|
+
* Type guard to validate if an object matches the DirectorSegment shape.
|
|
87
|
+
* Checks for required prompt (string) and image (Buffer, string, or ImageWithAltText).
|
|
88
|
+
*/
|
|
89
|
+
function isDirectorSegment(segment) {
|
|
90
|
+
if (!segment || typeof segment !== "object") {
|
|
91
|
+
return false;
|
|
92
|
+
}
|
|
93
|
+
const maybeSegment = segment;
|
|
94
|
+
// Check for required prompt field
|
|
95
|
+
if (typeof maybeSegment.prompt !== "string" || !maybeSegment.prompt) {
|
|
96
|
+
return false;
|
|
97
|
+
}
|
|
98
|
+
// Check for required image field
|
|
99
|
+
const { image } = maybeSegment;
|
|
100
|
+
if (!image) {
|
|
101
|
+
return false;
|
|
102
|
+
}
|
|
103
|
+
// Validate image type: Buffer, string (URL/path), or ImageWithAltText
|
|
104
|
+
if (Buffer.isBuffer(image)) {
|
|
105
|
+
return true;
|
|
106
|
+
}
|
|
107
|
+
if (typeof image === "string") {
|
|
108
|
+
return true;
|
|
109
|
+
}
|
|
110
|
+
// Check for ImageWithAltText structure
|
|
111
|
+
if (typeof image === "object" && "data" in image) {
|
|
112
|
+
const imgData = image.data;
|
|
113
|
+
return Buffer.isBuffer(imgData) || typeof imgData === "string";
|
|
114
|
+
}
|
|
115
|
+
return false;
|
|
116
|
+
}
|
|
85
117
|
export function isMultimodalInput(input) {
|
|
86
118
|
const maybeInput = input;
|
|
87
119
|
return !!(maybeInput?.images?.length ||
|
|
@@ -90,7 +122,10 @@ export function isMultimodalInput(input) {
|
|
|
90
122
|
maybeInput?.files?.length ||
|
|
91
123
|
maybeInput?.content?.length ||
|
|
92
124
|
maybeInput?.audioFiles?.length ||
|
|
93
|
-
maybeInput?.videoFiles?.length
|
|
125
|
+
maybeInput?.videoFiles?.length ||
|
|
126
|
+
(maybeInput?.segments?.length &&
|
|
127
|
+
Array.isArray(maybeInput.segments) &&
|
|
128
|
+
maybeInput.segments.every(isDirectorSegment)));
|
|
94
129
|
}
|
|
95
130
|
/**
|
|
96
131
|
* Type guard to check if message content is multimodal (array)
|
|
@@ -8,7 +8,7 @@ import type { VideoOutputOptions } from "../types/multimodal.js";
|
|
|
8
8
|
import type { PPTOutputOptions } from "../types/pptTypes.js";
|
|
9
9
|
import type { EnhancedValidationResult } from "../types/tools.js";
|
|
10
10
|
import type { StringArray, ValidationSchema } from "../types/typeAliases.js";
|
|
11
|
-
import { NeuroLinkError } from "./errorHandling.js";
|
|
11
|
+
import { type NeuroLinkError } from "./errorHandling.js";
|
|
12
12
|
/**
|
|
13
13
|
* Custom error class for parameter validation failures
|
|
14
14
|
* Provides detailed information about validation errors including field context and suggestions
|
|
@@ -134,6 +134,13 @@ export declare function validateImageForVideo(image: Buffer | string, maxSize?:
|
|
|
134
134
|
* ```
|
|
135
135
|
*/
|
|
136
136
|
export declare function validateVideoGenerationInput(options: GenerateOptions): EnhancedValidationResult;
|
|
137
|
+
/**
|
|
138
|
+
* Validate Director Mode input: segments, transition prompts, and durations.
|
|
139
|
+
*
|
|
140
|
+
* @param options - GenerateOptions with input.segments and output.director
|
|
141
|
+
* @returns EnhancedValidationResult with errors, warnings, and suggestions
|
|
142
|
+
*/
|
|
143
|
+
export declare function validateDirectorModeInput(options: GenerateOptions): EnhancedValidationResult;
|
|
137
144
|
export declare const MIN_PPT_PAGES = 5;
|
|
138
145
|
export declare const MAX_PPT_PAGES = 50;
|
|
139
146
|
export declare const MIN_PPT_PROMPT_LENGTH = 10;
|
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
* Parameter Validation Utilities
|
|
3
3
|
* Provides consistent parameter validation across all tool interfaces
|
|
4
4
|
*/
|
|
5
|
+
import { VIDEO_ERROR_CODES } from "../constants/videoErrors.js";
|
|
5
6
|
import { SYSTEM_LIMITS } from "../core/constants.js";
|
|
6
|
-
import { ErrorFactory
|
|
7
|
+
import { ErrorFactory } from "./errorHandling.js";
|
|
7
8
|
import { isNonNullObject } from "./typeUtils.js";
|
|
8
9
|
// ============================================================================
|
|
9
10
|
// VALIDATION ERROR TYPES
|
|
@@ -606,6 +607,84 @@ export function validateVideoGenerationInput(options) {
|
|
|
606
607
|
return { isValid: errors.length === 0, errors, warnings, suggestions };
|
|
607
608
|
}
|
|
608
609
|
// ============================================================================
|
|
610
|
+
// DIRECTOR MODE VALIDATION
|
|
611
|
+
// ============================================================================
|
|
612
|
+
const MIN_DIRECTOR_SEGMENTS = 2;
|
|
613
|
+
const MAX_DIRECTOR_SEGMENTS = 10;
|
|
614
|
+
const VALID_TRANSITION_DURATIONS = [4, 6, 8];
|
|
615
|
+
/**
|
|
616
|
+
* Validate Director Mode input: segments, transition prompts, and durations.
|
|
617
|
+
*
|
|
618
|
+
* @param options - GenerateOptions with input.segments and output.director
|
|
619
|
+
* @returns EnhancedValidationResult with errors, warnings, and suggestions
|
|
620
|
+
*/
|
|
621
|
+
export function validateDirectorModeInput(options) {
|
|
622
|
+
const errors = [];
|
|
623
|
+
const warnings = [];
|
|
624
|
+
const suggestions = [];
|
|
625
|
+
const segments = options.input?.segments;
|
|
626
|
+
if (!segments || !Array.isArray(segments)) {
|
|
627
|
+
errors.push(new ValidationError("Director Mode requires an input.segments array", "input.segments", VIDEO_ERROR_CODES.DIRECTOR_SEGMENT_MISMATCH));
|
|
628
|
+
return { isValid: false, errors, warnings, suggestions };
|
|
629
|
+
}
|
|
630
|
+
if (segments.length < MIN_DIRECTOR_SEGMENTS ||
|
|
631
|
+
segments.length > MAX_DIRECTOR_SEGMENTS) {
|
|
632
|
+
errors.push(new ValidationError(`Director Mode requires ${MIN_DIRECTOR_SEGMENTS}-${MAX_DIRECTOR_SEGMENTS} segments, got ${segments.length}`, "input.segments", segments.length > MAX_DIRECTOR_SEGMENTS
|
|
633
|
+
? VIDEO_ERROR_CODES.DIRECTOR_SEGMENT_LIMIT_EXCEEDED
|
|
634
|
+
: VIDEO_ERROR_CODES.DIRECTOR_SEGMENT_MISMATCH));
|
|
635
|
+
return { isValid: false, errors, warnings, suggestions };
|
|
636
|
+
}
|
|
637
|
+
// Validate each segment
|
|
638
|
+
for (let i = 0; i < segments.length; i++) {
|
|
639
|
+
const seg = segments[i];
|
|
640
|
+
if (!seg || typeof seg !== "object") {
|
|
641
|
+
errors.push(new ValidationError(`Segment ${i} must be an object with prompt and image`, `input.segments[${i}]`, VIDEO_ERROR_CODES.DIRECTOR_SEGMENT_MISMATCH));
|
|
642
|
+
continue;
|
|
643
|
+
}
|
|
644
|
+
if (!seg.prompt ||
|
|
645
|
+
typeof seg.prompt !== "string" ||
|
|
646
|
+
seg.prompt.trim().length === 0) {
|
|
647
|
+
errors.push(new ValidationError(`Segment ${i} requires a non-empty prompt`, `input.segments[${i}].prompt`, VIDEO_ERROR_CODES.DIRECTOR_SEGMENT_MISMATCH));
|
|
648
|
+
}
|
|
649
|
+
if (seg.image === undefined || seg.image === null) {
|
|
650
|
+
errors.push(new ValidationError(`Segment ${i} requires a valid image (Buffer, URL, path, or ImageWithAltText)`, `input.segments[${i}].image`, VIDEO_ERROR_CODES.DIRECTOR_SEGMENT_MISMATCH));
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
// Validate director options
|
|
654
|
+
const director = options.output?.director;
|
|
655
|
+
const expectedTransitions = segments.length - 1;
|
|
656
|
+
if (director?.transitionPrompts) {
|
|
657
|
+
if (director.transitionPrompts.length !== expectedTransitions) {
|
|
658
|
+
errors.push(new ValidationError(`Expected ${expectedTransitions} transition prompts, got ${director.transitionPrompts.length}`, "output.director.transitionPrompts", VIDEO_ERROR_CODES.DIRECTOR_SEGMENT_MISMATCH));
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
if (director?.transitionDurations) {
|
|
662
|
+
if (director.transitionDurations.length !== expectedTransitions) {
|
|
663
|
+
errors.push(new ValidationError(`Expected ${expectedTransitions} transition durations, got ${director.transitionDurations.length}`, "output.director.transitionDurations", VIDEO_ERROR_CODES.DIRECTOR_SEGMENT_MISMATCH));
|
|
664
|
+
}
|
|
665
|
+
else {
|
|
666
|
+
for (let i = 0; i < director.transitionDurations.length; i++) {
|
|
667
|
+
const d = director.transitionDurations[i];
|
|
668
|
+
if (!VALID_TRANSITION_DURATIONS.includes(d)) {
|
|
669
|
+
errors.push(new ValidationError(`Invalid transition duration at index ${i}: ${d}. Use 4, 6, or 8`, `output.director.transitionDurations[${i}]`, VIDEO_ERROR_CODES.DIRECTOR_INVALID_TRANSITION_DURATION));
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
// Validate video output options if provided
|
|
675
|
+
if (options.output?.video) {
|
|
676
|
+
const videoError = validateVideoOutputOptions(options.output.video);
|
|
677
|
+
if (videoError) {
|
|
678
|
+
errors.push(toValidationError(videoError));
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
if (errors.length === 0) {
|
|
682
|
+
const totalCalls = segments.length + expectedTransitions;
|
|
683
|
+
suggestions.push(`Director Mode will make ${totalCalls} API calls (${segments.length} clips + ${expectedTransitions} transitions). Ensure adequate timeout.`);
|
|
684
|
+
}
|
|
685
|
+
return { isValid: errors.length === 0, errors, warnings, suggestions };
|
|
686
|
+
}
|
|
687
|
+
// ============================================================================
|
|
609
688
|
// PPT VALIDATION (Presentation Generation)
|
|
610
689
|
// ============================================================================
|
|
611
690
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@juspay/neurolink",
|
|
3
|
-
"version": "9.
|
|
3
|
+
"version": "9.23.0",
|
|
4
4
|
"description": "Universal AI Development Platform with working MCP integration, multi-provider support, and professional CLI. Built-in tools operational, 58+ external MCP servers discoverable. Connect to filesystem, GitHub, database operations, and more. Build, test, and deploy AI applications with 13 providers: OpenAI, Anthropic, Google AI, AWS Bedrock, Azure, Hugging Face, Ollama, and Mistral AI.",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Juspay Technologies",
|