@juspay/neurolink 10.10.6 → 10.10.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +37 -8
- package/dist/browser/neurolink.min.js +399 -399
- package/dist/cli/factories/commandFactory.js +8 -4
- package/dist/constants/contextWindows.js +10 -1
- package/dist/context/anthropicLoopGuard.d.ts +1 -0
- package/dist/context/anthropicLoopGuard.js +30 -12
- package/dist/context/contextCompactor.js +19 -0
- package/dist/context/geminiLoopGuard.d.ts +54 -0
- package/dist/context/geminiLoopGuard.js +140 -0
- package/dist/core/redisConversationMemoryManager.d.ts +27 -0
- package/dist/core/redisConversationMemoryManager.js +146 -25
- package/dist/lib/constants/contextWindows.js +10 -1
- package/dist/lib/context/anthropicLoopGuard.d.ts +1 -0
- package/dist/lib/context/anthropicLoopGuard.js +30 -12
- package/dist/lib/context/contextCompactor.js +19 -0
- package/dist/lib/context/geminiLoopGuard.d.ts +54 -0
- package/dist/lib/context/geminiLoopGuard.js +141 -0
- package/dist/lib/core/redisConversationMemoryManager.d.ts +27 -0
- package/dist/lib/core/redisConversationMemoryManager.js +146 -25
- package/dist/lib/processors/media/VideoProcessor.d.ts +13 -3
- package/dist/lib/processors/media/VideoProcessor.js +53 -12
- package/dist/lib/providers/googleAiStudio/client.d.ts +0 -31
- package/dist/lib/providers/googleAiStudio/client.js +118 -1
- package/dist/lib/providers/googleNativeGemini3/utils.d.ts +9 -0
- package/dist/lib/providers/googleNativeGemini3/utils.js +12 -0
- package/dist/lib/providers/googleVertex/client.d.ts +0 -45
- package/dist/lib/providers/googleVertex/client.js +201 -21
- package/dist/lib/types/context.d.ts +9 -0
- package/dist/lib/types/file.d.ts +37 -0
- package/dist/lib/types/generate.d.ts +4 -0
- package/dist/lib/types/stream.d.ts +4 -0
- package/dist/lib/utils/errorHandling.d.ts +21 -0
- package/dist/lib/utils/errorHandling.js +53 -0
- package/dist/lib/utils/fileDetector.js +9 -6
- package/dist/lib/utils/messageBuilder.js +111 -36
- package/dist/lib/utils/pdfProcessor.d.ts +11 -0
- package/dist/lib/utils/pdfProcessor.js +17 -0
- package/dist/lib/utils/redis.d.ts +60 -1
- package/dist/lib/utils/redis.js +143 -12
- package/dist/processors/media/VideoProcessor.d.ts +13 -3
- package/dist/processors/media/VideoProcessor.js +53 -12
- package/dist/providers/googleAiStudio/client.d.ts +0 -31
- package/dist/providers/googleAiStudio/client.js +118 -1
- package/dist/providers/googleNativeGemini3/utils.d.ts +9 -0
- package/dist/providers/googleNativeGemini3/utils.js +12 -0
- package/dist/providers/googleVertex/client.d.ts +0 -45
- package/dist/providers/googleVertex/client.js +201 -21
- package/dist/types/context.d.ts +9 -0
- package/dist/types/file.d.ts +37 -0
- package/dist/types/generate.d.ts +4 -0
- package/dist/types/stream.d.ts +4 -0
- package/dist/utils/errorHandling.d.ts +21 -0
- package/dist/utils/errorHandling.js +53 -0
- package/dist/utils/fileDetector.js +9 -6
- package/dist/utils/messageBuilder.js +111 -36
- package/dist/utils/pdfProcessor.d.ts +11 -0
- package/dist/utils/pdfProcessor.js +17 -0
- package/dist/utils/redis.d.ts +60 -1
- package/dist/utils/redis.js +143 -12
- package/package.json +3 -1
|
@@ -15,7 +15,7 @@ import { withTimeout } from "../utils/errorHandling.js";
|
|
|
15
15
|
import { buildContextFromPointer, getEffectiveTokenThreshold, } from "../utils/conversationMemory.js";
|
|
16
16
|
import { runWithCurrentLangfuseContext } from "../services/server/ai/observability/instrumentation.js";
|
|
17
17
|
import { logger } from "../utils/logger.js";
|
|
18
|
-
import { createRedisClient, deserializeConversation, getNormalizedConfig, getPooledRedisClient, getSessionKey, getUserSessionsKey, releasePooledRedisClient, scanKeys, serializeConversation, } from "../utils/redis.js";
|
|
18
|
+
import { createRedisClient, deserializeConversation, encodeStoredMessages, getSessionMessagesKey, MESSAGES_KEY_SUFFIX, isSessionMessagesKey, parseStoredMessages, serializeConversationMetadata, usesSplitMessageStorage, getNormalizedConfig, getPooledRedisClient, getSessionKey, getUserSessionsKey, releasePooledRedisClient, scanKeys, serializeConversation, } from "../utils/redis.js";
|
|
19
19
|
const redisTracer = tracers.redis;
|
|
20
20
|
const REDIS_TIMEOUT_MS = 5000;
|
|
21
21
|
/**
|
|
@@ -140,7 +140,7 @@ export class RedisConversationMemoryManager {
|
|
|
140
140
|
try {
|
|
141
141
|
const redisKey = getSessionKey(this.redisConfig, sessionId, userId);
|
|
142
142
|
const conversationData = await withTimeout(redisClient.get(redisKey), REDIS_TIMEOUT_MS);
|
|
143
|
-
const conversation = deserializeConversation(conversationData || null);
|
|
143
|
+
const conversation = await this.hydrateMessages(deserializeConversation(conversationData || null), redisKey);
|
|
144
144
|
if (!conversation) {
|
|
145
145
|
span.setAttribute("session.found", false);
|
|
146
146
|
return undefined;
|
|
@@ -221,7 +221,7 @@ export class RedisConversationMemoryManager {
|
|
|
221
221
|
}
|
|
222
222
|
const redisKey = getSessionKey(this.redisConfig, sessionId, userId);
|
|
223
223
|
const conversationData = await this.redisClient.get(redisKey);
|
|
224
|
-
return deserializeConversation(conversationData || null);
|
|
224
|
+
return this.hydrateMessages(deserializeConversation(conversationData || null), redisKey);
|
|
225
225
|
}
|
|
226
226
|
catch (error) {
|
|
227
227
|
logger.error("[RedisConversationMemoryManager] Failed to get raw session", {
|
|
@@ -391,7 +391,13 @@ export class RedisConversationMemoryManager {
|
|
|
391
391
|
}
|
|
392
392
|
const redisKey = getSessionKey(this.redisConfig, options.sessionId, options.userId);
|
|
393
393
|
const conversationData = await this.redisClient.get(redisKey);
|
|
394
|
-
let conversation = deserializeConversation(conversationData);
|
|
394
|
+
let conversation = await this.hydrateMessages(deserializeConversation(conversationData), redisKey);
|
|
395
|
+
// Split-storage bookkeeping: whether this session already keeps its
|
|
396
|
+
// messages in the companion LIST, and how many it held before this
|
|
397
|
+
// turn — so only the new ones are appended rather than rewriting the
|
|
398
|
+
// whole conversation on every turn.
|
|
399
|
+
const wasSplitStorage = usesSplitMessageStorage(conversation);
|
|
400
|
+
const messageCountBeforeTurn = conversation?.messages.length ?? 0;
|
|
395
401
|
const currentTime = new Date().toISOString();
|
|
396
402
|
const normalizedUserId = options.userId || "randomUser";
|
|
397
403
|
if (!conversation) {
|
|
@@ -486,6 +492,16 @@ export class RedisConversationMemoryManager {
|
|
|
486
492
|
const shouldSummarize = options.enableSummarization !== undefined
|
|
487
493
|
? options.enableSummarization
|
|
488
494
|
: this.config.enableSummarization;
|
|
495
|
+
// Append-only: push just this turn's messages and persist a SMALL
|
|
496
|
+
// metadata blob. A session not yet using split storage (new, or a
|
|
497
|
+
// legacy blob) is converted once here, then every later turn appends.
|
|
498
|
+
await this.persistConversation(conversation, options.sessionId, options.userId, wasSplitStorage ? messageCountBeforeTurn : undefined);
|
|
499
|
+
// Scheduled AFTER the write, never before it. On the single turn that
|
|
500
|
+
// converts a legacy session, `checkAndSummarize` re-reads the blob and
|
|
501
|
+
// writes it back; observing the pre-conversion blob would make it
|
|
502
|
+
// re-serialize the record WITHOUT the split marker, so a later SET
|
|
503
|
+
// would strip the marker while the companion LIST already held the
|
|
504
|
+
// messages — the next read would then serve the stale inline copy.
|
|
489
505
|
if (shouldSummarize) {
|
|
490
506
|
const normalizedUserId = options.userId || "randomUser";
|
|
491
507
|
const summarizationKey = `${options.sessionId}:${normalizedUserId}`;
|
|
@@ -514,10 +530,8 @@ export class RedisConversationMemoryManager {
|
|
|
514
530
|
});
|
|
515
531
|
}
|
|
516
532
|
}
|
|
517
|
-
const serializedData = serializeConversation(conversation);
|
|
518
|
-
await this.redisClient.set(redisKey, serializedData);
|
|
519
533
|
// Log turn storage metadata for observability
|
|
520
|
-
const blobSizeBytes = Buffer.byteLength(
|
|
534
|
+
const blobSizeBytes = Buffer.byteLength(serializeConversationMetadata(conversation), "utf8");
|
|
521
535
|
logger.info("[ConversationMemory] Turn stored", {
|
|
522
536
|
requestId: options.requestId,
|
|
523
537
|
sessionId: options.sessionId,
|
|
@@ -611,7 +625,14 @@ export class RedisConversationMemoryManager {
|
|
|
611
625
|
conversation.summarizedMessage;
|
|
612
626
|
latestConversation.lastTokenCount = conversation.lastTokenCount;
|
|
613
627
|
latestConversation.lastCountedAt = conversation.lastCountedAt;
|
|
614
|
-
|
|
628
|
+
// Only summarization metadata changed, so the companion LIST is
|
|
629
|
+
// left alone. Critical: this re-read was NOT hydrated, so for a
|
|
630
|
+
// split session `messages` is empty — writing it with
|
|
631
|
+
// serializeConversation would drop the marker and make the record
|
|
632
|
+
// look like a legacy blob with zero messages (silent data loss).
|
|
633
|
+
const freshSerialized = usesSplitMessageStorage(latestConversation)
|
|
634
|
+
? serializeConversationMetadata(latestConversation)
|
|
635
|
+
: serializeConversation(latestConversation);
|
|
615
636
|
await this.redisClient.set(redisKey, freshSerialized);
|
|
616
637
|
if (this.redisConfig.ttl > 0) {
|
|
617
638
|
await this.redisClient.expire(redisKey, this.redisConfig.ttl);
|
|
@@ -631,6 +652,95 @@ export class RedisConversationMemoryManager {
|
|
|
631
652
|
this.summarizationInProgress.delete(summarizationKey);
|
|
632
653
|
}
|
|
633
654
|
}
|
|
655
|
+
/**
|
|
656
|
+
* True only for keys holding a conversation BLOB — the sole key type these
|
|
657
|
+
* scan-then-GET paths may read.
|
|
658
|
+
*
|
|
659
|
+
* `${keyPrefix}*` also matches the companion message LISTs and, when a
|
|
660
|
+
* custom key prefix does not end in `conversation:`, the user-index SETs
|
|
661
|
+
* (whose derived prefix then collapses onto `keyPrefix`). `GET` against
|
|
662
|
+
* either raises WRONGTYPE, and counting them would inflate session totals.
|
|
663
|
+
*/
|
|
664
|
+
isConversationBlobKey(key) {
|
|
665
|
+
return (!isSessionMessagesKey(key) &&
|
|
666
|
+
!key.endsWith(":sessions") &&
|
|
667
|
+
!key.startsWith(this.redisConfig.userSessionsKeyPrefix));
|
|
668
|
+
}
|
|
669
|
+
/**
|
|
670
|
+
* Hydrate a deserialized blob's messages from the companion LIST when the
|
|
671
|
+
* session uses split storage. Legacy blobs (messages inline) pass through
|
|
672
|
+
* untouched — that is what makes the migration backward compatible.
|
|
673
|
+
*/
|
|
674
|
+
async hydrateMessages(conversation, redisKey) {
|
|
675
|
+
if (!conversation || !usesSplitMessageStorage(conversation)) {
|
|
676
|
+
return conversation;
|
|
677
|
+
}
|
|
678
|
+
if (!this.redisClient) {
|
|
679
|
+
return conversation;
|
|
680
|
+
}
|
|
681
|
+
// `getSession` and `buildContextMessages` guard their own GET with
|
|
682
|
+
// `withTimeout`; both now also depend on this read, so leaving it unguarded
|
|
683
|
+
// would let a hung LIST read block the request thread anyway.
|
|
684
|
+
const entries = await withTimeout(this.redisClient.lRange(`${redisKey}${MESSAGES_KEY_SUFFIX}`, 0, -1), REDIS_TIMEOUT_MS);
|
|
685
|
+
// `lRange` is typed `(string | Buffer)[]` — the client returns Buffers when
|
|
686
|
+
// a connection is opened in binary mode. Normalize rather than assert, so
|
|
687
|
+
// a binary-mode client reads its history instead of throwing on parse.
|
|
688
|
+
conversation.messages = parseStoredMessages(entries.map((entry) => entry.toString()));
|
|
689
|
+
return conversation;
|
|
690
|
+
}
|
|
691
|
+
/** Message count without materializing them — LLEN for split sessions. */
|
|
692
|
+
async countMessages(conversation, redisKey) {
|
|
693
|
+
if (!usesSplitMessageStorage(conversation) || !this.redisClient) {
|
|
694
|
+
return conversation.messages.length;
|
|
695
|
+
}
|
|
696
|
+
// `lLen` is typed `number | \`${number}\`` (RESP3 can deliver it as a
|
|
697
|
+
// string), and callers add it to numbers — coerce so the count never
|
|
698
|
+
// concatenates instead of summing.
|
|
699
|
+
return Number(await withTimeout(this.redisClient.lLen(`${redisKey}${MESSAGES_KEY_SUFFIX}`), REDIS_TIMEOUT_MS));
|
|
700
|
+
}
|
|
701
|
+
/** Load a session, messages included, regardless of storage format. */
|
|
702
|
+
async loadConversation(sessionId, userId) {
|
|
703
|
+
if (!this.redisClient) {
|
|
704
|
+
return null;
|
|
705
|
+
}
|
|
706
|
+
const redisKey = getSessionKey(this.redisConfig, sessionId, userId);
|
|
707
|
+
return this.hydrateMessages(deserializeConversation(await this.redisClient.get(redisKey)), redisKey);
|
|
708
|
+
}
|
|
709
|
+
/**
|
|
710
|
+
* Persist a conversation, splitting messages into the companion LIST.
|
|
711
|
+
* `appendFrom` appends only messages from that index onward (the per-turn
|
|
712
|
+
* fast path); omit it to rewrite the LIST wholesale, which is also how a
|
|
713
|
+
* legacy blob gets converted.
|
|
714
|
+
*/
|
|
715
|
+
async persistConversation(conversation, sessionId, userId, appendFrom) {
|
|
716
|
+
if (!this.redisClient) {
|
|
717
|
+
return;
|
|
718
|
+
}
|
|
719
|
+
const redisKey = getSessionKey(this.redisConfig, sessionId, userId);
|
|
720
|
+
const messagesKey = getSessionMessagesKey(this.redisConfig, sessionId, userId);
|
|
721
|
+
// One MULTI, not four to six independent commands. The replace path is a
|
|
722
|
+
// DEL followed by an RPUSH, and on the legacy-conversion path the inline
|
|
723
|
+
// blob is the only copy of the history until the SET lands — so a failure
|
|
724
|
+
// between any two of them leaves the session either empty or unrecoverable.
|
|
725
|
+
// Grouping them also collapses the round trips, which is the point of the
|
|
726
|
+
// append-only layout.
|
|
727
|
+
const tx = this.redisClient.multi();
|
|
728
|
+
if (appendFrom === undefined) {
|
|
729
|
+
tx.del(messagesKey);
|
|
730
|
+
}
|
|
731
|
+
const toPush = appendFrom === undefined
|
|
732
|
+
? conversation.messages
|
|
733
|
+
: conversation.messages.slice(appendFrom);
|
|
734
|
+
if (toPush.length > 0) {
|
|
735
|
+
tx.rPush(messagesKey, encodeStoredMessages(toPush));
|
|
736
|
+
}
|
|
737
|
+
tx.set(redisKey, serializeConversationMetadata(conversation));
|
|
738
|
+
if (this.redisConfig.ttl > 0) {
|
|
739
|
+
tx.expire(redisKey, this.redisConfig.ttl);
|
|
740
|
+
tx.expire(messagesKey, this.redisConfig.ttl);
|
|
741
|
+
}
|
|
742
|
+
await withTimeout(tx.exec(), REDIS_TIMEOUT_MS);
|
|
743
|
+
}
|
|
634
744
|
/**
|
|
635
745
|
* Build context messages for AI prompt injection (TOKEN-BASED)
|
|
636
746
|
* Returns messages from pointer onwards (or all if no pointer)
|
|
@@ -664,7 +774,7 @@ export class RedisConversationMemoryManager {
|
|
|
664
774
|
});
|
|
665
775
|
const redisKey = getSessionKey(this.redisConfig, sessionId, userId);
|
|
666
776
|
const conversationData = await withTimeout(redisClient.get(redisKey), REDIS_TIMEOUT_MS);
|
|
667
|
-
const conversation = deserializeConversation(conversationData || null);
|
|
777
|
+
const conversation = await this.hydrateMessages(deserializeConversation(conversationData || null), redisKey);
|
|
668
778
|
logger.debug("[RedisConversationMemoryManager] Retrieved conversation for context building", {
|
|
669
779
|
sessionId,
|
|
670
780
|
userId,
|
|
@@ -895,7 +1005,7 @@ export class RedisConversationMemoryManager {
|
|
|
895
1005
|
return null;
|
|
896
1006
|
}
|
|
897
1007
|
// Deserialize the complete conversation object
|
|
898
|
-
const conversation = deserializeConversation(conversationData);
|
|
1008
|
+
const conversation = await this.hydrateMessages(deserializeConversation(conversationData), sessionKey);
|
|
899
1009
|
if (!conversation) {
|
|
900
1010
|
logger.debug("[RedisConversationMemoryManager] Failed to deserialize conversation data", {
|
|
901
1011
|
userId,
|
|
@@ -1014,7 +1124,7 @@ User message: "${userMessage}"`;
|
|
|
1014
1124
|
try {
|
|
1015
1125
|
const redisKey = getSessionKey(this.redisConfig, sessionId, userId);
|
|
1016
1126
|
const conversationData = await this.redisClient.get(redisKey);
|
|
1017
|
-
const conversation = deserializeConversation(conversationData || null);
|
|
1127
|
+
const conversation = await this.hydrateMessages(deserializeConversation(conversationData || null), redisKey);
|
|
1018
1128
|
return conversation?.messages ?? [];
|
|
1019
1129
|
}
|
|
1020
1130
|
catch (error) {
|
|
@@ -1050,11 +1160,8 @@ User message: "${userMessage}"`;
|
|
|
1050
1160
|
conversation.summarizedMessage = undefined;
|
|
1051
1161
|
conversation.lastTokenCount = undefined;
|
|
1052
1162
|
conversation.lastCountedAt = undefined;
|
|
1053
|
-
|
|
1054
|
-
await this.
|
|
1055
|
-
if (this.redisConfig.ttl > 0) {
|
|
1056
|
-
await this.redisClient.expire(redisKey, this.redisConfig.ttl);
|
|
1057
|
-
}
|
|
1163
|
+
// Wholesale replacement: rewrite the LIST rather than appending.
|
|
1164
|
+
await this.persistConversation(conversation, sessionId, userId);
|
|
1058
1165
|
logger.debug("[RedisConversationMemoryManager] Session messages replaced", {
|
|
1059
1166
|
sessionId,
|
|
1060
1167
|
userId,
|
|
@@ -1097,11 +1204,16 @@ User message: "${userMessage}"`;
|
|
|
1097
1204
|
pattern,
|
|
1098
1205
|
keyCount: keys.length,
|
|
1099
1206
|
});
|
|
1207
|
+
// Companion message LISTs share the session key prefix, so the SCAN above
|
|
1208
|
+
// returns them too. GET on a LIST raises WRONGTYPE, and counting them as
|
|
1209
|
+
// sessions would double `totalSessions` — filter them out exactly as the
|
|
1210
|
+
// session listing already skips `:sessions` index keys.
|
|
1211
|
+
const sessionKeys = keys.filter((key) => this.isConversationBlobKey(key));
|
|
1100
1212
|
// Count messages in each session
|
|
1101
1213
|
let totalTurns = 0;
|
|
1102
|
-
for (const key of
|
|
1214
|
+
for (const key of sessionKeys) {
|
|
1103
1215
|
const conversationData = await this.redisClient.get(key);
|
|
1104
|
-
const conversation = deserializeConversation(conversationData);
|
|
1216
|
+
const conversation = await this.hydrateMessages(deserializeConversation(conversationData), key);
|
|
1105
1217
|
if (conversation?.messages) {
|
|
1106
1218
|
// Pinned skill messages are extra rows inside a turn — exclude them
|
|
1107
1219
|
// so a skill-activating turn still counts as one turn.
|
|
@@ -1110,7 +1222,7 @@ User message: "${userMessage}"`;
|
|
|
1110
1222
|
}
|
|
1111
1223
|
}
|
|
1112
1224
|
return {
|
|
1113
|
-
totalSessions:
|
|
1225
|
+
totalSessions: sessionKeys.length,
|
|
1114
1226
|
totalTurns,
|
|
1115
1227
|
};
|
|
1116
1228
|
}
|
|
@@ -1133,7 +1245,9 @@ User message: "${userMessage}"`;
|
|
|
1133
1245
|
}, async (span) => {
|
|
1134
1246
|
try {
|
|
1135
1247
|
const redisKey = getSessionKey(this.redisConfig, sessionId, userId);
|
|
1136
|
-
|
|
1248
|
+
// Delete the companion message LIST as well: leaving it behind would
|
|
1249
|
+
// leak the messages and let a re-created session inherit them.
|
|
1250
|
+
const result = await withTimeout(redisClient.del([redisKey, `${redisKey}${MESSAGES_KEY_SUFFIX}`]), REDIS_TIMEOUT_MS);
|
|
1137
1251
|
if (Number(result) > 0) {
|
|
1138
1252
|
// Remove session from user's session set
|
|
1139
1253
|
if (userId) {
|
|
@@ -1307,8 +1421,9 @@ User message: "${userMessage}"`;
|
|
|
1307
1421
|
// List all sessions across all users by scanning Redis keys
|
|
1308
1422
|
const keys = await scanKeys(this.redisClient, `${this.redisConfig.keyPrefix}*`);
|
|
1309
1423
|
for (const key of keys) {
|
|
1310
|
-
// Skip user session index keys (they end with :sessions)
|
|
1311
|
-
|
|
1424
|
+
// Skip user session index keys (they end with :sessions) and the
|
|
1425
|
+
// companion message LISTs — GET on a LIST raises WRONGTYPE.
|
|
1426
|
+
if (!this.isConversationBlobKey(key)) {
|
|
1312
1427
|
continue;
|
|
1313
1428
|
}
|
|
1314
1429
|
const raw = await this.redisClient.get(key);
|
|
@@ -1325,7 +1440,9 @@ User message: "${userMessage}"`;
|
|
|
1325
1440
|
createdAt: session.createdAt,
|
|
1326
1441
|
updatedAt: session.updatedAt,
|
|
1327
1442
|
userId: session.userId,
|
|
1328
|
-
|
|
1443
|
+
// LLEN for split sessions: the blob's inline array is empty here
|
|
1444
|
+
// because this branch reads the raw key without hydrating.
|
|
1445
|
+
messageCount: await this.countMessages(session, key),
|
|
1329
1446
|
lastActive: this.formatTimeAgo(now - new Date(session.updatedAt).getTime()),
|
|
1330
1447
|
});
|
|
1331
1448
|
}
|
|
@@ -1571,8 +1688,12 @@ User message: "${userMessage}"`;
|
|
|
1571
1688
|
logger.debug("[RedisConversationMemoryManager] Added new agentic loop report", { sessionId, reportId: report.reportId });
|
|
1572
1689
|
}
|
|
1573
1690
|
conversation.updatedAt = new Date().toISOString();
|
|
1574
|
-
// Write back to Redis
|
|
1575
|
-
|
|
1691
|
+
// Write back to Redis. Metadata-only change, so the companion LIST is
|
|
1692
|
+
// untouched — but the split marker must survive (see the summarization
|
|
1693
|
+
// writeback above for why dropping it loses messages).
|
|
1694
|
+
const serializedData = usesSplitMessageStorage(conversation)
|
|
1695
|
+
? serializeConversationMetadata(conversation)
|
|
1696
|
+
: serializeConversation(conversation);
|
|
1576
1697
|
await withTimeout(this.redisClient.set(redisKey, serializedData), 5000);
|
|
1577
1698
|
if (this.redisConfig.ttl > 0) {
|
|
1578
1699
|
await withTimeout(this.redisClient.expire(redisKey, this.redisConfig.ttl), 5000);
|
|
@@ -44,7 +44,7 @@
|
|
|
44
44
|
* ```
|
|
45
45
|
*/
|
|
46
46
|
import { BaseFileProcessor } from "../base/BaseFileProcessor.js";
|
|
47
|
-
import type { FileInfo, ProcessedVideo, ProcessorFileProcessingResult, ProcessOptions } from "../../types/index.js";
|
|
47
|
+
import type { FileInfo, ProcessedVideo, ProcessorFileProcessingResult, ProcessOptions, VideoProcessorOptions } from "../../types/index.js";
|
|
48
48
|
/**
|
|
49
49
|
* Narrow a loaded `fluent-ffmpeg` export to the shape this file actually uses:
|
|
50
50
|
* a callable carrying the `ffprobe` and `setFfmpegPath` statics.
|
|
@@ -116,7 +116,7 @@ export declare class VideoProcessor extends BaseFileProcessor<ProcessedVideo> {
|
|
|
116
116
|
* @param options - Optional processing options
|
|
117
117
|
* @returns Processing result with extracted video data or error
|
|
118
118
|
*/
|
|
119
|
-
processFile(fileInfo: FileInfo, options?: ProcessOptions): Promise<ProcessorFileProcessingResult<ProcessedVideo>>;
|
|
119
|
+
processFile(fileInfo: FileInfo, options?: ProcessOptions & VideoProcessorOptions): Promise<ProcessorFileProcessingResult<ProcessedVideo>>;
|
|
120
120
|
/**
|
|
121
121
|
* Probe a video file to extract metadata using ffprobe.
|
|
122
122
|
*
|
|
@@ -137,6 +137,11 @@ export declare class VideoProcessor extends BaseFileProcessor<ProcessedVideo> {
|
|
|
137
137
|
* @returns Structured video metadata
|
|
138
138
|
*/
|
|
139
139
|
private buildMetadata;
|
|
140
|
+
/**
|
|
141
|
+
* Clamp a caller-supplied frame quality into sharp's valid 1-100 range,
|
|
142
|
+
* falling back to the default when absent or non-numeric (#478).
|
|
143
|
+
*/
|
|
144
|
+
private static resolveFrameQuality;
|
|
140
145
|
/**
|
|
141
146
|
* Extract keyframes from a video at calculated intervals.
|
|
142
147
|
*
|
|
@@ -153,10 +158,15 @@ export declare class VideoProcessor extends BaseFileProcessor<ProcessedVideo> {
|
|
|
153
158
|
* The interval is adaptive: if the tier interval would exceed MAX_FRAMES,
|
|
154
159
|
* the interval widens to duration/MAX_FRAMES for full-video coverage.
|
|
155
160
|
*
|
|
161
|
+
* A caller-supplied `options.frames` overrides the tier schedule entirely:
|
|
162
|
+
* that many frames are spread evenly across the clip, still capped at
|
|
163
|
+
* MAX_FRAMES. `options.quality` and `options.format` reach the encoder (#478).
|
|
164
|
+
*
|
|
156
165
|
* @param videoPath - Path to the video file
|
|
157
166
|
* @param tempDir - Temp directory for frame output
|
|
158
167
|
* @param durationSec - Video duration in seconds
|
|
159
|
-
* @
|
|
168
|
+
* @param options - Caller frame budget / encoder settings
|
|
169
|
+
* @returns Array of encoded frame buffers (JPEG unless png was requested)
|
|
160
170
|
*/
|
|
161
171
|
private extractKeyframes;
|
|
162
172
|
/**
|
|
@@ -313,7 +313,10 @@ export class VideoProcessor extends BaseFileProcessor {
|
|
|
313
313
|
* @param options - Optional processing options
|
|
314
314
|
* @returns Processing result with extracted video data or error
|
|
315
315
|
*/
|
|
316
|
-
async processFile(fileInfo,
|
|
316
|
+
async processFile(fileInfo,
|
|
317
|
+
// #478: widened with the keyframe knobs so `--video-frames`/`-quality`/
|
|
318
|
+
// `-format` can reach the encoder instead of being silently discarded.
|
|
319
|
+
options) {
|
|
317
320
|
const filename = this.getFilename(fileInfo);
|
|
318
321
|
const sizeBytes = fileInfo.size || fileInfo.buffer?.length || 0;
|
|
319
322
|
return withSpan({
|
|
@@ -438,7 +441,7 @@ export class VideoProcessor extends BaseFileProcessor {
|
|
|
438
441
|
// Step 5: Extract keyframes
|
|
439
442
|
let keyframes = [];
|
|
440
443
|
try {
|
|
441
|
-
keyframes = await this.extractKeyframes(tempVideoPath, tempDir, metadata.duration);
|
|
444
|
+
keyframes = await this.extractKeyframes(tempVideoPath, tempDir, metadata.duration, options);
|
|
442
445
|
}
|
|
443
446
|
catch {
|
|
444
447
|
// Non-fatal: continue without keyframes if extraction fails
|
|
@@ -642,6 +645,16 @@ export class VideoProcessor extends BaseFileProcessor {
|
|
|
642
645
|
// ===========================================================================
|
|
643
646
|
// KEYFRAME EXTRACTION
|
|
644
647
|
// ===========================================================================
|
|
648
|
+
/**
|
|
649
|
+
* Clamp a caller-supplied frame quality into sharp's valid 1-100 range,
|
|
650
|
+
* falling back to the default when absent or non-numeric (#478).
|
|
651
|
+
*/
|
|
652
|
+
static resolveFrameQuality(quality) {
|
|
653
|
+
if (typeof quality !== "number" || !Number.isFinite(quality)) {
|
|
654
|
+
return VIDEO_CONFIG.FRAME_JPEG_QUALITY;
|
|
655
|
+
}
|
|
656
|
+
return Math.min(100, Math.max(1, Math.round(quality)));
|
|
657
|
+
}
|
|
645
658
|
/**
|
|
646
659
|
* Extract keyframes from a video at calculated intervals.
|
|
647
660
|
*
|
|
@@ -658,20 +671,45 @@ export class VideoProcessor extends BaseFileProcessor {
|
|
|
658
671
|
* The interval is adaptive: if the tier interval would exceed MAX_FRAMES,
|
|
659
672
|
* the interval widens to duration/MAX_FRAMES for full-video coverage.
|
|
660
673
|
*
|
|
674
|
+
* A caller-supplied `options.frames` overrides the tier schedule entirely:
|
|
675
|
+
* that many frames are spread evenly across the clip, still capped at
|
|
676
|
+
* MAX_FRAMES. `options.quality` and `options.format` reach the encoder (#478).
|
|
677
|
+
*
|
|
661
678
|
* @param videoPath - Path to the video file
|
|
662
679
|
* @param tempDir - Temp directory for frame output
|
|
663
680
|
* @param durationSec - Video duration in seconds
|
|
664
|
-
* @
|
|
681
|
+
* @param options - Caller frame budget / encoder settings
|
|
682
|
+
* @returns Array of encoded frame buffers (JPEG unless png was requested)
|
|
665
683
|
*/
|
|
666
|
-
async extractKeyframes(videoPath, tempDir, durationSec) {
|
|
684
|
+
async extractKeyframes(videoPath, tempDir, durationSec, options) {
|
|
667
685
|
if (durationSec <= 0) {
|
|
668
686
|
return [];
|
|
669
687
|
}
|
|
670
|
-
//
|
|
671
|
-
|
|
688
|
+
// #478: honor the caller's frame budget, still bounded by MAX_FRAMES so a
|
|
689
|
+
// CLI flag can lower the cost but never raise it past the processor's own
|
|
690
|
+
// ceiling. A non-positive/non-finite request falls back to the default.
|
|
691
|
+
const requestedFrames = options?.frames;
|
|
692
|
+
const hasExplicitBudget = typeof requestedFrames === "number" &&
|
|
693
|
+
Number.isFinite(requestedFrames) &&
|
|
694
|
+
requestedFrames > 0;
|
|
695
|
+
const frameBudget = hasExplicitBudget
|
|
696
|
+
? Math.min(Math.floor(requestedFrames), VIDEO_CONFIG.MAX_FRAMES)
|
|
697
|
+
: VIDEO_CONFIG.MAX_FRAMES;
|
|
698
|
+
// Determine extraction interval based on duration. When the caller asked
|
|
699
|
+
// for a specific frame count, spread that many evenly across the whole
|
|
700
|
+
// video instead of using the duration tier — otherwise a short interval
|
|
701
|
+
// would hit the budget early and only cover the opening seconds.
|
|
702
|
+
//
|
|
703
|
+
// Keyed on whether a budget was REQUESTED, not on whether it happens to be
|
|
704
|
+
// below MAX_FRAMES: asking for exactly MAX_FRAMES is still an explicit
|
|
705
|
+
// request and must produce that many frames, not silently fall back to the
|
|
706
|
+
// tier schedule (which yields far fewer on a short clip).
|
|
707
|
+
const intervalSec = hasExplicitBudget
|
|
708
|
+
? Math.max(durationSec / frameBudget, Number.EPSILON)
|
|
709
|
+
: this.getFrameInterval(durationSec);
|
|
672
710
|
// Calculate timestamps to extract
|
|
673
711
|
const timestamps = [];
|
|
674
|
-
for (let t = 0; t < durationSec && timestamps.length <
|
|
712
|
+
for (let t = 0; t < durationSec && timestamps.length < frameBudget; t += intervalSec) {
|
|
675
713
|
timestamps.push(t);
|
|
676
714
|
}
|
|
677
715
|
if (timestamps.length === 0) {
|
|
@@ -691,13 +729,16 @@ export class VideoProcessor extends BaseFileProcessor {
|
|
|
691
729
|
const rawFrame = await fs.readFile(framePath);
|
|
692
730
|
// Resize to fit within max dimension while preserving aspect ratio
|
|
693
731
|
const sharp = (await import("sharp")).default;
|
|
694
|
-
const
|
|
695
|
-
.resize(VIDEO_CONFIG.FRAME_MAX_DIMENSION, VIDEO_CONFIG.FRAME_MAX_DIMENSION, {
|
|
732
|
+
const pipeline = sharp(rawFrame).resize(VIDEO_CONFIG.FRAME_MAX_DIMENSION, VIDEO_CONFIG.FRAME_MAX_DIMENSION, {
|
|
696
733
|
fit: "inside",
|
|
697
734
|
withoutEnlargement: true,
|
|
698
|
-
})
|
|
699
|
-
|
|
700
|
-
|
|
735
|
+
});
|
|
736
|
+
// #478: `--video-quality` / `--video-format` were accepted by the CLI
|
|
737
|
+
// and then dropped on the floor; both now reach the encoder.
|
|
738
|
+
const quality = VideoProcessor.resolveFrameQuality(options?.quality);
|
|
739
|
+
const resized = await (options?.format === "png"
|
|
740
|
+
? pipeline.png({ quality })
|
|
741
|
+
: pipeline.jpeg({ quality })).toBuffer();
|
|
701
742
|
keyframes.push(resized);
|
|
702
743
|
}
|
|
703
744
|
catch {
|
|
@@ -2,37 +2,6 @@ import { type AIProviderName } from "../../constants/enums.js";
|
|
|
2
2
|
import { BaseProvider } from "../../core/baseProvider.js";
|
|
3
3
|
import type { ZodUnknownSchema, EnhancedGenerateResult, TextGenerationOptions, StreamOptions, StreamResult } from "../../types/index.js";
|
|
4
4
|
import type { LanguageModel, Schema } from "../../types/index.js";
|
|
5
|
-
/**
|
|
6
|
-
* Google AI Studio provider implementation using BaseProvider
|
|
7
|
-
* Migrated from original GoogleAIStudio class to new factory pattern
|
|
8
|
-
*
|
|
9
|
-
* @important Structured Output Limitation
|
|
10
|
-
* Google Gemini models cannot combine function calling (tools) with structured
|
|
11
|
-
* output (JSON schema). When using schemas with output.format: "json", you MUST
|
|
12
|
-
* set disableTools: true.
|
|
13
|
-
*
|
|
14
|
-
* Error without disableTools:
|
|
15
|
-
* "Function calling with a response mime type: 'application/json' is unsupported"
|
|
16
|
-
*
|
|
17
|
-
* This is a Google API limitation documented at:
|
|
18
|
-
* https://ai.google.dev/gemini-api/docs/function-calling
|
|
19
|
-
*
|
|
20
|
-
* @example
|
|
21
|
-
* ```typescript
|
|
22
|
-
* // ✅ Correct usage with schemas
|
|
23
|
-
* const provider = new GoogleAIStudioProvider("gemini-2.5-flash");
|
|
24
|
-
* const result = await provider.generate({
|
|
25
|
-
* input: { text: "Analyze data" },
|
|
26
|
-
* schema: MySchema,
|
|
27
|
-
* output: { format: "json" },
|
|
28
|
-
* disableTools: true // Required
|
|
29
|
-
* });
|
|
30
|
-
* ```
|
|
31
|
-
*
|
|
32
|
-
* @note Gemini 3 Pro Preview (November 2025) will support combining tools + schemas
|
|
33
|
-
* @note "Too many states for serving" errors can occur with complex schemas + tools.
|
|
34
|
-
* Solution: Simplify schema or use disableTools: true
|
|
35
|
-
*/
|
|
36
5
|
export declare class GoogleAIStudioProvider extends BaseProvider {
|
|
37
6
|
private credentials?;
|
|
38
7
|
constructor(modelName?: string, sdk?: unknown, credentials?: {
|