@juspay/neurolink 9.5.3 → 9.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +29 -25
- package/dist/adapters/video/vertexVideoHandler.js +3 -3
- package/dist/agent/directTools.d.ts +5 -5
- package/dist/cli/commands/config.d.ts +9 -9
- package/dist/cli/commands/serve.d.ts +37 -0
- package/dist/cli/commands/serve.js +302 -229
- package/dist/cli/commands/setup-anthropic.d.ts +2 -2
- package/dist/cli/commands/setup-azure.d.ts +2 -2
- package/dist/cli/commands/setup-bedrock.d.ts +2 -2
- package/dist/cli/commands/setup-gcp.d.ts +2 -2
- package/dist/cli/commands/setup-google-ai.d.ts +2 -2
- package/dist/cli/commands/setup-huggingface.d.ts +2 -2
- package/dist/cli/commands/setup-mistral.d.ts +2 -2
- package/dist/cli/commands/setup-openai.d.ts +2 -2
- package/dist/cli/commands/setup.d.ts +2 -2
- package/dist/cli/factories/commandFactory.js +16 -2
- package/dist/cli/loop/optionsSchema.d.ts +2 -2
- package/dist/cli/loop/optionsSchema.js +4 -0
- package/dist/cli/loop/session.d.ts +4 -0
- package/dist/cli/loop/session.js +49 -4
- package/dist/cli/utils/interactiveSetup.d.ts +4 -4
- package/dist/config/conversationMemory.d.ts +2 -0
- package/dist/config/conversationMemory.js +5 -5
- package/dist/constants/contextWindows.d.ts +46 -0
- package/dist/constants/contextWindows.js +156 -0
- package/dist/context/budgetChecker.d.ts +18 -0
- package/dist/context/budgetChecker.js +71 -0
- package/dist/context/contextCompactor.d.ts +22 -0
- package/dist/context/contextCompactor.js +106 -0
- package/dist/context/effectiveHistory.d.ts +52 -0
- package/dist/context/effectiveHistory.js +105 -0
- package/dist/context/errorDetection.d.ts +14 -0
- package/dist/context/errorDetection.js +124 -0
- package/dist/context/fileSummarizationService.d.ts +54 -0
- package/dist/context/fileSummarizationService.js +255 -0
- package/dist/context/fileSummarizer.d.ts +56 -0
- package/dist/context/fileSummarizer.js +145 -0
- package/dist/context/fileTokenBudget.d.ts +53 -0
- package/dist/context/fileTokenBudget.js +127 -0
- package/dist/context/prompts/summarizationPrompt.d.ts +17 -0
- package/dist/context/prompts/summarizationPrompt.js +110 -0
- package/dist/context/stages/fileReadDeduplicator.d.ts +10 -0
- package/dist/context/stages/fileReadDeduplicator.js +66 -0
- package/dist/context/stages/slidingWindowTruncator.d.ts +11 -0
- package/dist/context/stages/slidingWindowTruncator.js +42 -0
- package/dist/context/stages/structuredSummarizer.d.ts +10 -0
- package/dist/context/stages/structuredSummarizer.js +49 -0
- package/dist/context/stages/toolOutputPruner.d.ts +10 -0
- package/dist/context/stages/toolOutputPruner.js +52 -0
- package/dist/context/summarizationEngine.d.ts +45 -0
- package/dist/context/summarizationEngine.js +110 -0
- package/dist/context/toolOutputLimits.d.ts +17 -0
- package/dist/context/toolOutputLimits.js +84 -0
- package/dist/context/toolPairRepair.d.ts +16 -0
- package/dist/context/toolPairRepair.js +66 -0
- package/dist/core/analytics.js +11 -4
- package/dist/core/baseProvider.d.ts +6 -0
- package/dist/core/baseProvider.js +83 -14
- package/dist/core/conversationMemoryManager.d.ts +14 -11
- package/dist/core/conversationMemoryManager.js +36 -68
- package/dist/core/dynamicModels.js +3 -2
- package/dist/core/modules/GenerationHandler.js +2 -0
- package/dist/core/modules/MessageBuilder.d.ts +1 -1
- package/dist/core/modules/MessageBuilder.js +2 -0
- package/dist/core/modules/TelemetryHandler.d.ts +2 -3
- package/dist/core/modules/TelemetryHandler.js +3 -3
- package/dist/core/modules/ToolsManager.d.ts +2 -2
- package/dist/core/redisConversationMemoryManager.d.ts +19 -14
- package/dist/core/redisConversationMemoryManager.js +94 -86
- package/dist/factories/providerFactory.d.ts +2 -2
- package/dist/files/fileReferenceRegistry.d.ts +276 -0
- package/dist/files/fileReferenceRegistry.js +1543 -0
- package/dist/files/fileTools.d.ts +423 -0
- package/dist/files/fileTools.js +449 -0
- package/dist/files/index.d.ts +14 -0
- package/dist/files/index.js +13 -0
- package/dist/files/streamingReader.d.ts +93 -0
- package/dist/files/streamingReader.js +321 -0
- package/dist/files/types.d.ts +23 -0
- package/dist/files/types.js +23 -0
- package/dist/image-gen/imageGenTools.d.ts +2 -2
- package/dist/image-gen/types.d.ts +12 -12
- package/dist/index.d.ts +4 -0
- package/dist/index.js +5 -0
- package/dist/lib/adapters/video/vertexVideoHandler.js +3 -3
- package/dist/lib/agent/directTools.d.ts +7 -7
- package/dist/lib/config/conversationMemory.d.ts +2 -0
- package/dist/lib/config/conversationMemory.js +5 -5
- package/dist/lib/constants/contextWindows.d.ts +46 -0
- package/dist/lib/constants/contextWindows.js +157 -0
- package/dist/lib/context/budgetChecker.d.ts +18 -0
- package/dist/lib/context/budgetChecker.js +72 -0
- package/dist/lib/context/contextCompactor.d.ts +22 -0
- package/dist/lib/context/contextCompactor.js +107 -0
- package/dist/lib/context/effectiveHistory.d.ts +52 -0
- package/dist/lib/context/effectiveHistory.js +106 -0
- package/dist/lib/context/errorDetection.d.ts +14 -0
- package/dist/lib/context/errorDetection.js +125 -0
- package/dist/lib/context/fileSummarizationService.d.ts +54 -0
- package/dist/lib/context/fileSummarizationService.js +256 -0
- package/dist/lib/context/fileSummarizer.d.ts +56 -0
- package/dist/lib/context/fileSummarizer.js +146 -0
- package/dist/lib/context/fileTokenBudget.d.ts +53 -0
- package/dist/lib/context/fileTokenBudget.js +128 -0
- package/dist/lib/context/prompts/summarizationPrompt.d.ts +17 -0
- package/dist/lib/context/prompts/summarizationPrompt.js +111 -0
- package/dist/lib/context/stages/fileReadDeduplicator.d.ts +10 -0
- package/dist/lib/context/stages/fileReadDeduplicator.js +67 -0
- package/dist/lib/context/stages/slidingWindowTruncator.d.ts +11 -0
- package/dist/lib/context/stages/slidingWindowTruncator.js +43 -0
- package/dist/lib/context/stages/structuredSummarizer.d.ts +10 -0
- package/dist/lib/context/stages/structuredSummarizer.js +50 -0
- package/dist/lib/context/stages/toolOutputPruner.d.ts +10 -0
- package/dist/lib/context/stages/toolOutputPruner.js +53 -0
- package/dist/lib/context/summarizationEngine.d.ts +45 -0
- package/dist/lib/context/summarizationEngine.js +111 -0
- package/dist/lib/context/toolOutputLimits.d.ts +17 -0
- package/dist/lib/context/toolOutputLimits.js +85 -0
- package/dist/lib/context/toolPairRepair.d.ts +16 -0
- package/dist/lib/context/toolPairRepair.js +67 -0
- package/dist/lib/core/analytics.js +11 -4
- package/dist/lib/core/baseProvider.d.ts +6 -0
- package/dist/lib/core/baseProvider.js +83 -14
- package/dist/lib/core/conversationMemoryManager.d.ts +14 -11
- package/dist/lib/core/conversationMemoryManager.js +36 -68
- package/dist/lib/core/dynamicModels.js +3 -2
- package/dist/lib/core/modules/GenerationHandler.js +2 -0
- package/dist/lib/core/modules/MessageBuilder.d.ts +1 -1
- package/dist/lib/core/modules/MessageBuilder.js +2 -0
- package/dist/lib/core/modules/TelemetryHandler.d.ts +2 -3
- package/dist/lib/core/modules/TelemetryHandler.js +3 -3
- package/dist/lib/core/modules/ToolsManager.d.ts +2 -2
- package/dist/lib/core/redisConversationMemoryManager.d.ts +19 -14
- package/dist/lib/core/redisConversationMemoryManager.js +94 -86
- package/dist/lib/factories/providerFactory.d.ts +2 -2
- package/dist/lib/files/fileReferenceRegistry.d.ts +276 -0
- package/dist/lib/files/fileReferenceRegistry.js +1544 -0
- package/dist/lib/files/fileTools.d.ts +423 -0
- package/dist/lib/files/fileTools.js +450 -0
- package/dist/lib/files/index.d.ts +14 -0
- package/dist/lib/files/index.js +14 -0
- package/dist/lib/files/streamingReader.d.ts +93 -0
- package/dist/lib/files/streamingReader.js +322 -0
- package/dist/lib/files/types.d.ts +23 -0
- package/dist/lib/files/types.js +24 -0
- package/dist/lib/image-gen/imageGenTools.d.ts +2 -2
- package/dist/lib/image-gen/types.d.ts +12 -12
- package/dist/lib/index.d.ts +4 -0
- package/dist/lib/index.js +5 -0
- package/dist/lib/mcp/httpRetryHandler.js +6 -2
- package/dist/lib/memory/mem0Initializer.d.ts +2 -2
- package/dist/lib/neurolink.d.ts +66 -2
- package/dist/lib/neurolink.js +777 -315
- package/dist/lib/processors/archive/ArchiveProcessor.d.ts +327 -0
- package/dist/lib/processors/archive/ArchiveProcessor.js +1309 -0
- package/dist/lib/processors/archive/index.d.ts +33 -0
- package/dist/lib/processors/archive/index.js +43 -0
- package/dist/lib/processors/base/BaseFileProcessor.js +2 -1
- package/dist/lib/processors/base/types.d.ts +70 -64
- package/dist/lib/processors/base/types.js +6 -0
- package/dist/lib/processors/cli/fileProcessorCli.d.ts +8 -8
- package/dist/lib/processors/cli/fileProcessorCli.js +5 -5
- package/dist/lib/processors/config/mimeTypes.js +25 -0
- package/dist/lib/processors/config/sizeLimits.d.ts +52 -40
- package/dist/lib/processors/config/sizeLimits.js +56 -44
- package/dist/lib/processors/document/ExcelProcessor.d.ts +14 -0
- package/dist/lib/processors/document/ExcelProcessor.js +72 -1
- package/dist/lib/processors/document/PptxProcessor.d.ts +63 -0
- package/dist/lib/processors/document/PptxProcessor.js +158 -0
- package/dist/lib/processors/document/index.d.ts +1 -0
- package/dist/lib/processors/document/index.js +6 -0
- package/dist/lib/processors/errors/FileErrorCode.d.ts +2 -2
- package/dist/lib/processors/errors/errorHelpers.d.ts +2 -2
- package/dist/lib/processors/errors/errorHelpers.js +12 -4
- package/dist/lib/processors/errors/errorSerializer.d.ts +4 -4
- package/dist/lib/processors/index.d.ts +8 -2
- package/dist/lib/processors/index.js +5 -2
- package/dist/lib/processors/integration/FileProcessorIntegration.d.ts +8 -8
- package/dist/lib/processors/integration/FileProcessorIntegration.js +7 -7
- package/dist/lib/processors/media/AudioProcessor.d.ts +328 -0
- package/dist/lib/processors/media/AudioProcessor.js +708 -0
- package/dist/lib/processors/media/VideoProcessor.d.ts +350 -0
- package/dist/lib/processors/media/VideoProcessor.js +992 -0
- package/dist/lib/processors/media/index.d.ts +27 -0
- package/dist/lib/processors/media/index.js +37 -0
- package/dist/lib/processors/registry/ProcessorRegistry.d.ts +19 -5
- package/dist/lib/processors/registry/ProcessorRegistry.js +103 -8
- package/dist/lib/processors/registry/index.d.ts +1 -1
- package/dist/lib/processors/registry/index.js +1 -1
- package/dist/lib/processors/registry/types.d.ts +2 -2
- package/dist/lib/providers/amazonBedrock.js +2 -1
- package/dist/lib/providers/anthropic.js +2 -2
- package/dist/lib/providers/anthropicBaseProvider.js +10 -4
- package/dist/lib/providers/azureOpenai.js +14 -25
- package/dist/lib/providers/googleAiStudio.js +136 -457
- package/dist/lib/providers/googleNativeGemini3.d.ts +119 -0
- package/dist/lib/providers/googleNativeGemini3.js +264 -0
- package/dist/lib/providers/googleVertex.d.ts +15 -1
- package/dist/lib/providers/googleVertex.js +341 -775
- package/dist/lib/providers/huggingFace.js +20 -5
- package/dist/lib/providers/litellm.js +6 -4
- package/dist/lib/providers/mistral.js +3 -2
- package/dist/lib/providers/openAI.js +2 -2
- package/dist/lib/providers/openRouter.js +8 -7
- package/dist/lib/providers/openaiCompatible.js +10 -4
- package/dist/lib/providers/sagemaker/detection.d.ts +6 -6
- package/dist/lib/providers/sagemaker/diagnostics.d.ts +4 -4
- package/dist/lib/providers/sagemaker/parsers.d.ts +4 -4
- package/dist/lib/rag/chunkers/RecursiveChunker.js +2 -2
- package/dist/lib/rag/document/loaders.d.ts +6 -71
- package/dist/lib/rag/document/loaders.js +5 -5
- package/dist/lib/rag/graphRag/graphRAG.js +26 -9
- package/dist/lib/rag/metadata/MetadataExtractorFactory.d.ts +5 -55
- package/dist/lib/rag/metadata/metadataExtractor.js +6 -3
- package/dist/lib/rag/pipeline/RAGPipeline.d.ts +8 -126
- package/dist/lib/rag/pipeline/RAGPipeline.js +11 -11
- package/dist/lib/rag/pipeline/contextAssembly.d.ts +3 -42
- package/dist/lib/rag/pipeline/contextAssembly.js +6 -3
- package/dist/lib/rag/reranker/RerankerFactory.d.ts +5 -60
- package/dist/lib/rag/resilience/CircuitBreaker.d.ts +3 -33
- package/dist/lib/rag/resilience/RetryHandler.d.ts +2 -21
- package/dist/lib/rag/resilience/RetryHandler.js +6 -2
- package/dist/lib/rag/retrieval/hybridSearch.d.ts +3 -41
- package/dist/lib/rag/retrieval/vectorQueryTool.d.ts +2 -13
- package/dist/lib/rag/retrieval/vectorQueryTool.js +4 -3
- package/dist/lib/rag/types.d.ts +3 -3
- package/dist/lib/sdk/toolRegistration.d.ts +2 -2
- package/dist/lib/server/middleware/cache.d.ts +2 -2
- package/dist/lib/server/middleware/rateLimit.d.ts +2 -2
- package/dist/lib/server/routes/mcpRoutes.js +277 -249
- package/dist/lib/server/routes/memoryRoutes.js +287 -281
- package/dist/lib/server/utils/validation.d.ts +10 -10
- package/dist/lib/services/server/ai/observability/instrumentation.d.ts +24 -2
- package/dist/lib/services/server/ai/observability/instrumentation.js +12 -1
- package/dist/lib/session/globalSessionState.d.ts +2 -2
- package/dist/lib/telemetry/telemetryService.d.ts +2 -2
- package/dist/lib/types/common.d.ts +39 -0
- package/dist/lib/types/contextTypes.d.ts +255 -0
- package/dist/lib/types/contextTypes.js +0 -2
- package/dist/lib/types/conversation.d.ts +62 -0
- package/dist/lib/types/conversationMemoryInterface.d.ts +27 -0
- package/dist/lib/types/conversationMemoryInterface.js +7 -0
- package/dist/lib/types/fileReferenceTypes.d.ts +222 -0
- package/dist/lib/types/fileReferenceTypes.js +9 -0
- package/dist/lib/types/fileTypes.d.ts +26 -3
- package/dist/lib/types/generateTypes.d.ts +50 -1
- package/dist/lib/types/index.d.ts +4 -5
- package/dist/lib/types/index.js +8 -10
- package/dist/lib/types/modelTypes.d.ts +2 -2
- package/dist/lib/types/processorTypes.d.ts +597 -0
- package/dist/lib/types/processorTypes.js +91 -0
- package/dist/lib/types/ragTypes.d.ts +489 -0
- package/dist/lib/types/ragTypes.js +8 -0
- package/dist/lib/types/sdkTypes.d.ts +17 -18
- package/dist/lib/types/streamTypes.d.ts +24 -1
- package/dist/lib/utils/async/retry.d.ts +2 -2
- package/dist/lib/utils/async/withTimeout.js +3 -1
- package/dist/lib/utils/conversationMemory.d.ts +12 -6
- package/dist/lib/utils/conversationMemory.js +91 -36
- package/dist/lib/utils/errorHandling.d.ts +5 -0
- package/dist/lib/utils/errorHandling.js +19 -0
- package/dist/lib/utils/fileDetector.d.ts +62 -0
- package/dist/lib/utils/fileDetector.js +1014 -14
- package/dist/lib/utils/json/safeParse.d.ts +2 -2
- package/dist/lib/utils/messageBuilder.js +806 -153
- package/dist/lib/utils/modelChoices.d.ts +2 -2
- package/dist/lib/utils/multimodalOptionsBuilder.d.ts +2 -1
- package/dist/lib/utils/multimodalOptionsBuilder.js +1 -0
- package/dist/lib/utils/pricing.d.ts +12 -0
- package/dist/lib/utils/pricing.js +134 -0
- package/dist/lib/utils/rateLimiter.d.ts +2 -2
- package/dist/lib/utils/redis.d.ts +17 -0
- package/dist/lib/utils/redis.js +105 -0
- package/dist/lib/utils/sanitizers/filename.d.ts +4 -4
- package/dist/lib/utils/sanitizers/svg.d.ts +2 -2
- package/dist/lib/utils/thinkingConfig.d.ts +6 -6
- package/dist/lib/utils/timeout.d.ts +10 -0
- package/dist/lib/utils/timeout.js +15 -0
- package/dist/lib/utils/tokenEstimation.d.ts +68 -0
- package/dist/lib/utils/tokenEstimation.js +113 -0
- package/dist/lib/utils/tokenUtils.d.ts +4 -4
- package/dist/lib/utils/ttsProcessor.d.ts +2 -2
- package/dist/lib/workflow/config.d.ts +150 -150
- package/dist/mcp/httpRetryHandler.js +6 -2
- package/dist/memory/mem0Initializer.d.ts +2 -2
- package/dist/neurolink.d.ts +66 -2
- package/dist/neurolink.js +777 -315
- package/dist/processors/archive/ArchiveProcessor.d.ts +327 -0
- package/dist/processors/archive/ArchiveProcessor.js +1308 -0
- package/dist/processors/archive/index.d.ts +33 -0
- package/dist/processors/archive/index.js +42 -0
- package/dist/processors/base/BaseFileProcessor.js +2 -1
- package/dist/processors/base/types.d.ts +70 -64
- package/dist/processors/base/types.js +6 -0
- package/dist/processors/cli/fileProcessorCli.d.ts +8 -8
- package/dist/processors/cli/fileProcessorCli.js +5 -5
- package/dist/processors/config/mimeTypes.js +25 -0
- package/dist/processors/config/sizeLimits.d.ts +52 -40
- package/dist/processors/config/sizeLimits.js +56 -44
- package/dist/processors/document/ExcelProcessor.d.ts +14 -0
- package/dist/processors/document/ExcelProcessor.js +72 -1
- package/dist/processors/document/PptxProcessor.d.ts +63 -0
- package/dist/processors/document/PptxProcessor.js +157 -0
- package/dist/processors/document/index.d.ts +1 -0
- package/dist/processors/document/index.js +6 -0
- package/dist/processors/errors/FileErrorCode.d.ts +2 -2
- package/dist/processors/errors/errorHelpers.d.ts +2 -2
- package/dist/processors/errors/errorHelpers.js +12 -4
- package/dist/processors/errors/errorSerializer.d.ts +4 -4
- package/dist/processors/index.d.ts +8 -2
- package/dist/processors/index.js +5 -2
- package/dist/processors/integration/FileProcessorIntegration.d.ts +8 -8
- package/dist/processors/integration/FileProcessorIntegration.js +7 -7
- package/dist/processors/media/AudioProcessor.d.ts +328 -0
- package/dist/processors/media/AudioProcessor.js +707 -0
- package/dist/processors/media/VideoProcessor.d.ts +350 -0
- package/dist/processors/media/VideoProcessor.js +991 -0
- package/dist/processors/media/ffprobe-static.d.ts +4 -0
- package/dist/processors/media/index.d.ts +27 -0
- package/dist/processors/media/index.js +36 -0
- package/dist/processors/registry/ProcessorRegistry.d.ts +19 -5
- package/dist/processors/registry/ProcessorRegistry.js +103 -8
- package/dist/processors/registry/index.d.ts +1 -1
- package/dist/processors/registry/index.js +1 -1
- package/dist/processors/registry/types.d.ts +2 -2
- package/dist/providers/amazonBedrock.js +2 -1
- package/dist/providers/anthropic.js +2 -2
- package/dist/providers/anthropicBaseProvider.js +10 -4
- package/dist/providers/azureOpenai.js +14 -25
- package/dist/providers/googleAiStudio.js +136 -457
- package/dist/providers/googleNativeGemini3.d.ts +119 -0
- package/dist/providers/googleNativeGemini3.js +263 -0
- package/dist/providers/googleVertex.d.ts +15 -1
- package/dist/providers/googleVertex.js +341 -775
- package/dist/providers/huggingFace.js +20 -5
- package/dist/providers/litellm.js +6 -4
- package/dist/providers/mistral.js +3 -2
- package/dist/providers/openAI.js +2 -2
- package/dist/providers/openRouter.js +8 -7
- package/dist/providers/openaiCompatible.js +10 -4
- package/dist/providers/sagemaker/detection.d.ts +6 -6
- package/dist/providers/sagemaker/diagnostics.d.ts +4 -4
- package/dist/providers/sagemaker/parsers.d.ts +4 -4
- package/dist/rag/chunkers/RecursiveChunker.js +2 -2
- package/dist/rag/document/loaders.d.ts +6 -71
- package/dist/rag/document/loaders.js +5 -5
- package/dist/rag/graphRag/graphRAG.js +26 -9
- package/dist/rag/metadata/MetadataExtractorFactory.d.ts +5 -55
- package/dist/rag/metadata/metadataExtractor.js +6 -3
- package/dist/rag/pipeline/RAGPipeline.d.ts +8 -126
- package/dist/rag/pipeline/RAGPipeline.js +11 -11
- package/dist/rag/pipeline/contextAssembly.d.ts +3 -42
- package/dist/rag/pipeline/contextAssembly.js +6 -3
- package/dist/rag/reranker/RerankerFactory.d.ts +5 -60
- package/dist/rag/resilience/CircuitBreaker.d.ts +3 -33
- package/dist/rag/resilience/RetryHandler.d.ts +2 -21
- package/dist/rag/resilience/RetryHandler.js +6 -2
- package/dist/rag/retrieval/hybridSearch.d.ts +3 -41
- package/dist/rag/retrieval/vectorQueryTool.d.ts +2 -13
- package/dist/rag/retrieval/vectorQueryTool.js +4 -3
- package/dist/rag/types.d.ts +3 -3
- package/dist/sdk/toolRegistration.d.ts +2 -2
- package/dist/server/middleware/cache.d.ts +2 -2
- package/dist/server/middleware/rateLimit.d.ts +2 -2
- package/dist/server/routes/mcpRoutes.js +277 -249
- package/dist/server/routes/memoryRoutes.js +287 -281
- package/dist/server/utils/validation.d.ts +4 -4
- package/dist/services/server/ai/observability/instrumentation.d.ts +24 -2
- package/dist/services/server/ai/observability/instrumentation.js +12 -1
- package/dist/session/globalSessionState.d.ts +2 -2
- package/dist/telemetry/telemetryService.d.ts +2 -2
- package/dist/types/common.d.ts +39 -0
- package/dist/types/contextTypes.d.ts +255 -0
- package/dist/types/contextTypes.js +0 -2
- package/dist/types/conversation.d.ts +62 -0
- package/dist/types/conversationMemoryInterface.d.ts +27 -0
- package/dist/types/conversationMemoryInterface.js +6 -0
- package/dist/types/fileReferenceTypes.d.ts +222 -0
- package/dist/types/fileReferenceTypes.js +8 -0
- package/dist/types/fileTypes.d.ts +26 -3
- package/dist/types/generateTypes.d.ts +50 -1
- package/dist/types/index.d.ts +4 -5
- package/dist/types/index.js +8 -10
- package/dist/types/processorTypes.d.ts +597 -0
- package/dist/types/processorTypes.js +90 -0
- package/dist/types/ragTypes.d.ts +489 -0
- package/dist/types/ragTypes.js +7 -0
- package/dist/types/sdkTypes.d.ts +17 -18
- package/dist/types/streamTypes.d.ts +24 -1
- package/dist/utils/async/retry.d.ts +2 -2
- package/dist/utils/async/withTimeout.js +3 -1
- package/dist/utils/conversationMemory.d.ts +12 -6
- package/dist/utils/conversationMemory.js +91 -36
- package/dist/utils/errorHandling.d.ts +5 -0
- package/dist/utils/errorHandling.js +19 -0
- package/dist/utils/fileDetector.d.ts +62 -0
- package/dist/utils/fileDetector.js +1014 -14
- package/dist/utils/json/safeParse.d.ts +2 -2
- package/dist/utils/messageBuilder.js +806 -153
- package/dist/utils/modelChoices.d.ts +2 -2
- package/dist/utils/multimodalOptionsBuilder.d.ts +2 -1
- package/dist/utils/multimodalOptionsBuilder.js +1 -0
- package/dist/utils/pricing.d.ts +12 -0
- package/dist/utils/pricing.js +133 -0
- package/dist/utils/rateLimiter.d.ts +2 -2
- package/dist/utils/redis.d.ts +17 -0
- package/dist/utils/redis.js +105 -0
- package/dist/utils/sanitizers/filename.d.ts +4 -4
- package/dist/utils/sanitizers/svg.d.ts +2 -2
- package/dist/utils/thinkingConfig.d.ts +6 -6
- package/dist/utils/timeout.d.ts +10 -0
- package/dist/utils/timeout.js +15 -0
- package/dist/utils/tokenEstimation.d.ts +68 -0
- package/dist/utils/tokenEstimation.js +112 -0
- package/dist/utils/tokenUtils.d.ts +4 -4
- package/dist/utils/ttsProcessor.d.ts +2 -2
- package/dist/workflow/config.d.ts +104 -104
- package/package.json +18 -6
- package/dist/lib/utils/conversationMemoryUtils.d.ts +0 -25
- package/dist/lib/utils/conversationMemoryUtils.js +0 -138
- package/dist/utils/conversationMemoryUtils.d.ts +0 -25
- package/dist/utils/conversationMemoryUtils.js +0 -137
|
@@ -0,0 +1,708 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Audio File Processor
|
|
3
|
+
*
|
|
4
|
+
* Handles downloading, validating, and processing audio files to extract metadata
|
|
5
|
+
* and build text content suitable for LLM consumption. Audio files cannot be sent
|
|
6
|
+
* raw to most LLMs, so this processor extracts structured metadata (duration, codec,
|
|
7
|
+
* bitrate, tags) and formats it as text.
|
|
8
|
+
*
|
|
9
|
+
* Uses the `music-metadata` library (pure JavaScript, no native dependencies) for
|
|
10
|
+
* metadata extraction. Supports all major audio formats: MP3, WAV, OGG, FLAC, M4A,
|
|
11
|
+
* AAC, WMA, WebM, AIFF, AMR, APE, WavPack, and more.
|
|
12
|
+
*
|
|
13
|
+
* Key features:
|
|
14
|
+
* - Metadata extraction: duration, codec, bitrate, sample rate, channels
|
|
15
|
+
* - Tag extraction: title, artist, album, year, genre, track number, composer
|
|
16
|
+
* - Embedded cover art extraction
|
|
17
|
+
* - Graceful degradation for corrupt or partially readable files
|
|
18
|
+
* - LLM-friendly text content generation
|
|
19
|
+
*
|
|
20
|
+
* @module processors/media/AudioProcessor
|
|
21
|
+
*
|
|
22
|
+
* @example
|
|
23
|
+
* ```typescript
|
|
24
|
+
* import { audioProcessor, processAudio, isAudioFile } from "./AudioProcessor.js";
|
|
25
|
+
*
|
|
26
|
+
* // Check if a file is an audio file
|
|
27
|
+
* if (isAudioFile(fileInfo.mimetype, fileInfo.name)) {
|
|
28
|
+
* const result = await processAudio(fileInfo);
|
|
29
|
+
*
|
|
30
|
+
* if (result.success) {
|
|
31
|
+
* console.log(`Duration: ${result.data.metadata.durationFormatted}`);
|
|
32
|
+
* console.log(`Codec: ${result.data.metadata.codec}`);
|
|
33
|
+
* console.log(`Artist: ${result.data.tags.artist}`);
|
|
34
|
+
* console.log(`Text for LLM: ${result.data.textContent}`);
|
|
35
|
+
* }
|
|
36
|
+
* }
|
|
37
|
+
* ```
|
|
38
|
+
*/
|
|
39
|
+
import { parseBuffer, selectCover } from "music-metadata";
|
|
40
|
+
import { BaseFileProcessor } from "../base/BaseFileProcessor.js";
|
|
41
|
+
import { SIZE_LIMITS_MB } from "../config/index.js";
|
|
42
|
+
import { FileErrorCode } from "../errors/index.js";
|
|
43
|
+
// =============================================================================
|
|
44
|
+
// CONSTANTS
|
|
45
|
+
// =============================================================================
|
|
46
|
+
/**
|
|
47
|
+
* Audio processor configuration constants.
|
|
48
|
+
*/
|
|
49
|
+
const AUDIO_CONFIG = {
|
|
50
|
+
/** Maximum audio file size in MB (uses centralized constant from sizeLimits) */
|
|
51
|
+
MAX_SIZE_MB: SIZE_LIMITS_MB.AUDIO_MAX_MB,
|
|
52
|
+
/** Processing timeout in milliseconds (audio metadata parsing is fast) */
|
|
53
|
+
TIMEOUT_MS: 30000,
|
|
54
|
+
/** Maximum file size for Whisper API transcription (25MB) */
|
|
55
|
+
WHISPER_MAX_SIZE_MB: 25,
|
|
56
|
+
/** Transcription timeout in milliseconds (120 seconds for large files) */
|
|
57
|
+
TRANSCRIPTION_TIMEOUT_MS: 120_000,
|
|
58
|
+
/** Whisper-supported audio formats */
|
|
59
|
+
WHISPER_SUPPORTED_FORMATS: [
|
|
60
|
+
"mp3",
|
|
61
|
+
"mp4",
|
|
62
|
+
"mpeg",
|
|
63
|
+
"mpga",
|
|
64
|
+
"m4a",
|
|
65
|
+
"wav",
|
|
66
|
+
"webm",
|
|
67
|
+
"flac",
|
|
68
|
+
"ogg",
|
|
69
|
+
],
|
|
70
|
+
};
|
|
71
|
+
/**
|
|
72
|
+
* Supported MIME types for audio files.
|
|
73
|
+
* Covers all major audio formats including common variants and aliases.
|
|
74
|
+
*/
|
|
75
|
+
const SUPPORTED_AUDIO_MIME_TYPES = [
|
|
76
|
+
"audio/mpeg",
|
|
77
|
+
"audio/mp3",
|
|
78
|
+
"audio/wav",
|
|
79
|
+
"audio/x-wav",
|
|
80
|
+
"audio/wave",
|
|
81
|
+
"audio/ogg",
|
|
82
|
+
"audio/vorbis",
|
|
83
|
+
"audio/opus",
|
|
84
|
+
"audio/flac",
|
|
85
|
+
"audio/x-flac",
|
|
86
|
+
"audio/mp4",
|
|
87
|
+
"audio/x-m4a",
|
|
88
|
+
"audio/aac",
|
|
89
|
+
"audio/x-ms-wma",
|
|
90
|
+
"audio/webm",
|
|
91
|
+
"audio/aiff",
|
|
92
|
+
"audio/x-aiff",
|
|
93
|
+
"audio/amr",
|
|
94
|
+
"audio/3gpp",
|
|
95
|
+
];
|
|
96
|
+
/**
|
|
97
|
+
* Supported file extensions for audio files.
|
|
98
|
+
* Includes common audio container formats and lossless variants.
|
|
99
|
+
*/
|
|
100
|
+
const SUPPORTED_AUDIO_EXTENSIONS = [
|
|
101
|
+
".mp3",
|
|
102
|
+
".wav",
|
|
103
|
+
".ogg",
|
|
104
|
+
".oga",
|
|
105
|
+
".opus",
|
|
106
|
+
".flac",
|
|
107
|
+
".m4a",
|
|
108
|
+
".aac",
|
|
109
|
+
".wma",
|
|
110
|
+
".webm",
|
|
111
|
+
".aiff",
|
|
112
|
+
".aif",
|
|
113
|
+
".amr",
|
|
114
|
+
".3gp",
|
|
115
|
+
".ape",
|
|
116
|
+
".wv",
|
|
117
|
+
];
|
|
118
|
+
// =============================================================================
|
|
119
|
+
// AUDIO PROCESSOR CLASS
|
|
120
|
+
// =============================================================================
|
|
121
|
+
/**
|
|
122
|
+
* Audio Processor - extracts metadata and tags from audio files for LLM consumption.
|
|
123
|
+
*
|
|
124
|
+
* Audio files cannot be directly sent to most language models. This processor
|
|
125
|
+
* parses audio file headers to extract structured metadata (duration, codec,
|
|
126
|
+
* bitrate, sample rate, channels) and embedded tags (title, artist, album, etc.),
|
|
127
|
+
* then builds a human-readable text summary for the AI to reason about.
|
|
128
|
+
*
|
|
129
|
+
* Uses the `music-metadata` library which is a pure JavaScript implementation
|
|
130
|
+
* with no native dependencies, making it safe for all deployment environments.
|
|
131
|
+
*
|
|
132
|
+
* @example
|
|
133
|
+
* ```typescript
|
|
134
|
+
* const processor = new AudioProcessor();
|
|
135
|
+
*
|
|
136
|
+
* const result = await processor.processFile({
|
|
137
|
+
* id: 'audio-123',
|
|
138
|
+
* name: 'song.mp3',
|
|
139
|
+
* mimetype: 'audio/mpeg',
|
|
140
|
+
* size: 5242880,
|
|
141
|
+
* buffer: audioBuffer,
|
|
142
|
+
* });
|
|
143
|
+
*
|
|
144
|
+
* if (result.success) {
|
|
145
|
+
* console.log(result.data.textContent);
|
|
146
|
+
* // "[Audio File: song.mp3]
|
|
147
|
+
* // Duration: 3:45 | Codec: MPEG 1 Layer 3 | Bitrate: 320 kbps | ..."
|
|
148
|
+
* }
|
|
149
|
+
* ```
|
|
150
|
+
*/
|
|
151
|
+
export class AudioProcessor extends BaseFileProcessor {
|
|
152
|
+
constructor() {
|
|
153
|
+
super({
|
|
154
|
+
maxSizeMB: AUDIO_CONFIG.MAX_SIZE_MB,
|
|
155
|
+
timeoutMs: AUDIO_CONFIG.TIMEOUT_MS,
|
|
156
|
+
supportedMimeTypes: [...SUPPORTED_AUDIO_MIME_TYPES],
|
|
157
|
+
supportedExtensions: [...SUPPORTED_AUDIO_EXTENSIONS],
|
|
158
|
+
fileTypeName: "audio",
|
|
159
|
+
defaultFilename: "audio.mp3",
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
// ===========================================================================
|
|
163
|
+
// PROCESSING OVERRIDE
|
|
164
|
+
// ===========================================================================
|
|
165
|
+
/**
|
|
166
|
+
* Override processFile for async audio metadata parsing with music-metadata.
|
|
167
|
+
*
|
|
168
|
+
* Processing pipeline:
|
|
169
|
+
* 1. Validate file type and size (base class)
|
|
170
|
+
* 2. Get file buffer (from direct buffer or download)
|
|
171
|
+
* 3. Parse audio metadata using music-metadata's parseBuffer()
|
|
172
|
+
* 4. Extract tags (title, artist, album, etc.)
|
|
173
|
+
* 5. Extract embedded cover art if present
|
|
174
|
+
* 6. Build LLM-friendly text content
|
|
175
|
+
*
|
|
176
|
+
* @param fileInfo - File information (can include URL or buffer)
|
|
177
|
+
* @param options - Optional processing options (auth headers, timeout, etc.)
|
|
178
|
+
* @returns Processing result with audio metadata or error
|
|
179
|
+
*/
|
|
180
|
+
async processFile(fileInfo, options) {
|
|
181
|
+
try {
|
|
182
|
+
// Step 1: Validate file type and size
|
|
183
|
+
const validationResult = this.validateFileWithResult(fileInfo);
|
|
184
|
+
if (!validationResult.success) {
|
|
185
|
+
return {
|
|
186
|
+
success: false,
|
|
187
|
+
error: validationResult.error,
|
|
188
|
+
};
|
|
189
|
+
}
|
|
190
|
+
// Step 2: Get file buffer (from direct buffer or download from URL)
|
|
191
|
+
let buffer;
|
|
192
|
+
if (fileInfo.buffer) {
|
|
193
|
+
buffer = fileInfo.buffer;
|
|
194
|
+
}
|
|
195
|
+
else if (fileInfo.url) {
|
|
196
|
+
const downloadResult = await this.downloadFileWithRetry(fileInfo, options);
|
|
197
|
+
if (!downloadResult.success) {
|
|
198
|
+
return {
|
|
199
|
+
success: false,
|
|
200
|
+
error: downloadResult.error,
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
if (!downloadResult.data) {
|
|
204
|
+
return {
|
|
205
|
+
success: false,
|
|
206
|
+
error: this.createError(FileErrorCode.DOWNLOAD_FAILED, {
|
|
207
|
+
reason: "Download succeeded but returned no data",
|
|
208
|
+
}),
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
buffer = downloadResult.data;
|
|
212
|
+
// Validate actual downloaded size against limit
|
|
213
|
+
if (!this.validateFileSize(buffer.length)) {
|
|
214
|
+
return {
|
|
215
|
+
success: false,
|
|
216
|
+
error: this.createError(FileErrorCode.FILE_TOO_LARGE, {
|
|
217
|
+
sizeMB: (buffer.length / (1024 * 1024)).toFixed(2),
|
|
218
|
+
maxMB: this.config.maxSizeMB,
|
|
219
|
+
type: this.config.fileTypeName,
|
|
220
|
+
}),
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
else {
|
|
225
|
+
return {
|
|
226
|
+
success: false,
|
|
227
|
+
error: this.createError(FileErrorCode.DOWNLOAD_FAILED, {
|
|
228
|
+
reason: "No buffer or URL provided for file",
|
|
229
|
+
}),
|
|
230
|
+
};
|
|
231
|
+
}
|
|
232
|
+
// Step 3: Parse audio metadata using music-metadata
|
|
233
|
+
const audioMetadata = await this.parseAudioMetadata(buffer, fileInfo);
|
|
234
|
+
// Step 4: Extract structured metadata from parsed result
|
|
235
|
+
const metadata = this.extractMetadata(audioMetadata, buffer.length);
|
|
236
|
+
// Step 5: Extract tags from common metadata
|
|
237
|
+
const tags = this.extractTags(audioMetadata);
|
|
238
|
+
// Step 6: Extract embedded cover art if present
|
|
239
|
+
const coverArt = this.extractCoverArt(audioMetadata);
|
|
240
|
+
// Step 7: Attempt transcription if API key is available
|
|
241
|
+
const filename = this.getFilename(fileInfo);
|
|
242
|
+
const transcriptionResult = await this.attemptTranscription(buffer, filename, fileInfo.mimetype);
|
|
243
|
+
// Step 8: Build LLM-friendly text content (includes transcript if available)
|
|
244
|
+
const textContent = this.buildTextContent(filename, metadata, tags, transcriptionResult.transcript);
|
|
245
|
+
return {
|
|
246
|
+
success: true,
|
|
247
|
+
data: {
|
|
248
|
+
textContent,
|
|
249
|
+
metadata,
|
|
250
|
+
tags,
|
|
251
|
+
transcript: transcriptionResult.transcript,
|
|
252
|
+
hasTranscript: transcriptionResult.hasTranscript,
|
|
253
|
+
transcriptionProvider: transcriptionResult.transcriptionProvider,
|
|
254
|
+
coverArt: coverArt ?? undefined,
|
|
255
|
+
buffer,
|
|
256
|
+
mimetype: fileInfo.mimetype || "audio/mpeg",
|
|
257
|
+
size: fileInfo.size,
|
|
258
|
+
filename,
|
|
259
|
+
},
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
catch (error) {
|
|
263
|
+
// Classify music-metadata parse errors as INVALID_FORMAT
|
|
264
|
+
// (corrupt/truncated files, unsupported codec variants, etc.)
|
|
265
|
+
const isParseError = error instanceof Error &&
|
|
266
|
+
(error.message.includes("parse") ||
|
|
267
|
+
error.message.includes("codec") ||
|
|
268
|
+
error.message.includes("header") ||
|
|
269
|
+
error.message.includes("format") ||
|
|
270
|
+
error.message.includes("unexpected end") ||
|
|
271
|
+
error.name === "CouldNotDetermineFileTypeError" ||
|
|
272
|
+
error.name === "UnsupportedFileTypeError");
|
|
273
|
+
const errorCode = isParseError
|
|
274
|
+
? FileErrorCode.INVALID_FORMAT
|
|
275
|
+
: FileErrorCode.PROCESSING_FAILED;
|
|
276
|
+
return {
|
|
277
|
+
success: false,
|
|
278
|
+
error: this.createError(errorCode, {
|
|
279
|
+
fileType: "audio",
|
|
280
|
+
error: error instanceof Error ? error.message : String(error),
|
|
281
|
+
}, error instanceof Error ? error : undefined),
|
|
282
|
+
};
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
// ===========================================================================
|
|
286
|
+
// PRIVATE: AUDIO TRANSCRIPTION
|
|
287
|
+
// ===========================================================================
|
|
288
|
+
/**
|
|
289
|
+
* Attempt to transcribe audio using the Vercel AI SDK's `transcribe()` function
|
|
290
|
+
* with the OpenAI Whisper model.
|
|
291
|
+
*
|
|
292
|
+
* Transcription is attempted when:
|
|
293
|
+
* 1. `OPENAI_API_KEY` environment variable is set
|
|
294
|
+
* 2. File size is within Whisper's 25MB limit
|
|
295
|
+
* 3. File format is supported by Whisper
|
|
296
|
+
*
|
|
297
|
+
* Gracefully degrades: if transcription fails for any reason, metadata-only
|
|
298
|
+
* output is returned (transcription is additive, never blocks processing).
|
|
299
|
+
*
|
|
300
|
+
* @param buffer - Audio file content
|
|
301
|
+
* @param filename - Original filename (used for format detection)
|
|
302
|
+
* @param mimetype - MIME type of the audio file
|
|
303
|
+
* @returns Transcription result with transcript text, or empty result
|
|
304
|
+
*/
|
|
305
|
+
async attemptTranscription(buffer, filename, mimetype) {
|
|
306
|
+
const emptyResult = {
|
|
307
|
+
transcript: undefined,
|
|
308
|
+
hasTranscript: false,
|
|
309
|
+
transcriptionProvider: undefined,
|
|
310
|
+
};
|
|
311
|
+
// Check if OPENAI_API_KEY is available
|
|
312
|
+
const apiKey = process.env.OPENAI_API_KEY;
|
|
313
|
+
if (!apiKey) {
|
|
314
|
+
return emptyResult;
|
|
315
|
+
}
|
|
316
|
+
// Check file size (Whisper limit is 25MB)
|
|
317
|
+
const fileSizeMB = buffer.length / (1024 * 1024);
|
|
318
|
+
if (fileSizeMB > AUDIO_CONFIG.WHISPER_MAX_SIZE_MB) {
|
|
319
|
+
return emptyResult;
|
|
320
|
+
}
|
|
321
|
+
// Check if file format is supported by Whisper
|
|
322
|
+
const ext = filename.split(".").pop()?.toLowerCase();
|
|
323
|
+
const isFormatSupported = ext && AUDIO_CONFIG.WHISPER_SUPPORTED_FORMATS.includes(ext);
|
|
324
|
+
const isMimeSupported = mimetype &&
|
|
325
|
+
(mimetype.startsWith("audio/mpeg") ||
|
|
326
|
+
mimetype.startsWith("audio/mp4") ||
|
|
327
|
+
mimetype.startsWith("audio/wav") ||
|
|
328
|
+
mimetype.startsWith("audio/webm") ||
|
|
329
|
+
mimetype.startsWith("audio/flac") ||
|
|
330
|
+
mimetype.startsWith("audio/ogg") ||
|
|
331
|
+
mimetype.startsWith("audio/x-m4a"));
|
|
332
|
+
if (!isFormatSupported && !isMimeSupported) {
|
|
333
|
+
return emptyResult;
|
|
334
|
+
}
|
|
335
|
+
try {
|
|
336
|
+
// Dynamic imports to avoid loading these modules when transcription is not needed
|
|
337
|
+
const [{ createOpenAI }, { experimental_transcribe }] = await Promise.all([import("@ai-sdk/openai"), import("ai")]);
|
|
338
|
+
const openai = createOpenAI({ apiKey });
|
|
339
|
+
const model = openai.transcription("whisper-1");
|
|
340
|
+
const result = await experimental_transcribe({
|
|
341
|
+
model,
|
|
342
|
+
audio: buffer,
|
|
343
|
+
});
|
|
344
|
+
if (result.text && result.text.trim().length > 0) {
|
|
345
|
+
return {
|
|
346
|
+
transcript: result.text.trim(),
|
|
347
|
+
hasTranscript: true,
|
|
348
|
+
transcriptionProvider: "openai-whisper",
|
|
349
|
+
};
|
|
350
|
+
}
|
|
351
|
+
return emptyResult;
|
|
352
|
+
}
|
|
353
|
+
catch {
|
|
354
|
+
// Transcription is best-effort — never fail the entire processing pipeline
|
|
355
|
+
// Common failures: rate limiting, network issues, unsupported audio encoding
|
|
356
|
+
return emptyResult;
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
// ===========================================================================
|
|
360
|
+
// STUB: buildProcessedResult (required by base class, unused due to override)
|
|
361
|
+
// ===========================================================================
|
|
362
|
+
/**
|
|
363
|
+
* Stub implementation required by BaseFileProcessor.
|
|
364
|
+
* Not used because processFile is fully overridden.
|
|
365
|
+
*
|
|
366
|
+
* @param buffer - File buffer
|
|
367
|
+
* @param fileInfo - File information
|
|
368
|
+
* @returns Empty ProcessedAudio structure
|
|
369
|
+
*/
|
|
370
|
+
buildProcessedResult(buffer, fileInfo) {
|
|
371
|
+
return {
|
|
372
|
+
textContent: "",
|
|
373
|
+
metadata: {
|
|
374
|
+
duration: 0,
|
|
375
|
+
durationFormatted: "0:00",
|
|
376
|
+
codec: "unknown",
|
|
377
|
+
lossless: false,
|
|
378
|
+
fileSize: buffer.length,
|
|
379
|
+
},
|
|
380
|
+
tags: {},
|
|
381
|
+
hasTranscript: false,
|
|
382
|
+
buffer,
|
|
383
|
+
mimetype: fileInfo.mimetype || "audio/mpeg",
|
|
384
|
+
size: fileInfo.size,
|
|
385
|
+
filename: this.getFilename(fileInfo),
|
|
386
|
+
};
|
|
387
|
+
}
|
|
388
|
+
// ===========================================================================
|
|
389
|
+
// PRIVATE: METADATA PARSING
|
|
390
|
+
// ===========================================================================
|
|
391
|
+
/**
|
|
392
|
+
* Parse audio metadata from a buffer using music-metadata.
|
|
393
|
+
*
|
|
394
|
+
* @param buffer - Audio file content
|
|
395
|
+
* @param fileInfo - File information (used for MIME type hint)
|
|
396
|
+
* @returns Parsed audio metadata from music-metadata
|
|
397
|
+
* @throws Error if the buffer cannot be parsed (corrupt file, unsupported format)
|
|
398
|
+
*/
|
|
399
|
+
async parseAudioMetadata(buffer, fileInfo) {
|
|
400
|
+
// Provide MIME type as a string hint to music-metadata for more accurate parsing.
|
|
401
|
+
// parseBuffer accepts (Uint8Array, fileInfo?: IFileInfo | string, options?)
|
|
402
|
+
// where string is interpreted as MIME type.
|
|
403
|
+
const mimeType = fileInfo.mimetype || undefined;
|
|
404
|
+
return parseBuffer(buffer, mimeType);
|
|
405
|
+
}
|
|
406
|
+
/**
|
|
407
|
+
* Extract structured metadata from the parsed audio format information.
|
|
408
|
+
*
|
|
409
|
+
* @param audioMetadata - Parsed audio metadata from music-metadata
|
|
410
|
+
* @param fileSize - File size in bytes
|
|
411
|
+
* @returns Structured metadata object
|
|
412
|
+
*/
|
|
413
|
+
extractMetadata(audioMetadata, fileSize) {
|
|
414
|
+
const format = audioMetadata.format;
|
|
415
|
+
const duration = format.duration ?? 0;
|
|
416
|
+
const durationFormatted = this.formatDuration(duration);
|
|
417
|
+
return {
|
|
418
|
+
duration,
|
|
419
|
+
durationFormatted,
|
|
420
|
+
codec: format.codec ?? format.container ?? "unknown",
|
|
421
|
+
codecProfile: format.codecProfile ?? undefined,
|
|
422
|
+
bitrate: format.bitrate ?? undefined,
|
|
423
|
+
sampleRate: format.sampleRate ?? undefined,
|
|
424
|
+
channels: format.numberOfChannels ?? undefined,
|
|
425
|
+
bitsPerSample: format.bitsPerSample ?? undefined,
|
|
426
|
+
lossless: format.lossless ?? false,
|
|
427
|
+
fileSize,
|
|
428
|
+
};
|
|
429
|
+
}
|
|
430
|
+
/**
|
|
431
|
+
* Extract common tags from the parsed audio metadata.
|
|
432
|
+
*
|
|
433
|
+
* Maps music-metadata's common tag format to our simplified tag structure.
|
|
434
|
+
* Handles array-to-scalar conversions (e.g., comment[] -> first comment string).
|
|
435
|
+
*
|
|
436
|
+
* @param audioMetadata - Parsed audio metadata from music-metadata
|
|
437
|
+
* @returns Simplified tag object
|
|
438
|
+
*/
|
|
439
|
+
extractTags(audioMetadata) {
|
|
440
|
+
const common = audioMetadata.common;
|
|
441
|
+
return {
|
|
442
|
+
title: common.title ?? undefined,
|
|
443
|
+
artist: common.artist ?? undefined,
|
|
444
|
+
album: common.album ?? undefined,
|
|
445
|
+
year: common.year ?? undefined,
|
|
446
|
+
genre: common.genre && common.genre.length > 0 ? common.genre : undefined,
|
|
447
|
+
track: common.track.no !== null || common.track.of !== null
|
|
448
|
+
? { no: common.track.no, of: common.track.of }
|
|
449
|
+
: undefined,
|
|
450
|
+
comment: common.comment && common.comment.length > 0
|
|
451
|
+
? (common.comment[0]?.text ?? undefined)
|
|
452
|
+
: undefined,
|
|
453
|
+
composer: common.composer && common.composer.length > 0
|
|
454
|
+
? common.composer[0]
|
|
455
|
+
: undefined,
|
|
456
|
+
};
|
|
457
|
+
}
|
|
458
|
+
/**
|
|
459
|
+
* Extract embedded cover art from the audio file.
|
|
460
|
+
*
|
|
461
|
+
* Uses music-metadata's selectCover() to pick the most appropriate
|
|
462
|
+
* cover image when multiple are embedded (e.g., front cover vs. back cover).
|
|
463
|
+
*
|
|
464
|
+
* @param audioMetadata - Parsed audio metadata from music-metadata
|
|
465
|
+
* @returns Cover art as Buffer, or null if no cover art is embedded
|
|
466
|
+
*/
|
|
467
|
+
extractCoverArt(audioMetadata) {
|
|
468
|
+
const pictures = audioMetadata.common.picture;
|
|
469
|
+
if (!pictures || pictures.length === 0) {
|
|
470
|
+
return null;
|
|
471
|
+
}
|
|
472
|
+
const cover = selectCover(pictures);
|
|
473
|
+
if (!cover) {
|
|
474
|
+
return null;
|
|
475
|
+
}
|
|
476
|
+
return Buffer.from(cover.data);
|
|
477
|
+
}
|
|
478
|
+
// ===========================================================================
|
|
479
|
+
// PRIVATE: TEXT CONTENT BUILDING
|
|
480
|
+
// ===========================================================================
|
|
481
|
+
/**
|
|
482
|
+
* Build an LLM-friendly text representation of the audio file.
|
|
483
|
+
*
|
|
484
|
+
* Produces a structured text block that gives the AI context about the
|
|
485
|
+
* audio file without requiring the actual audio stream. The format is
|
|
486
|
+
* designed to be scannable and information-dense.
|
|
487
|
+
*
|
|
488
|
+
* @param filename - Original filename
|
|
489
|
+
* @param metadata - Extracted audio metadata
|
|
490
|
+
* @param tags - Extracted audio tags
|
|
491
|
+
* @param transcript - Optional transcribed text from Whisper
|
|
492
|
+
* @returns Formatted text content string
|
|
493
|
+
*
|
|
494
|
+
* @example Output:
|
|
495
|
+
* ```
|
|
496
|
+
* [Audio File: song.mp3]
|
|
497
|
+
* Duration: 3:45 | Codec: MPEG 1 Layer 3 | Bitrate: 320 kbps | Sample Rate: 44100 Hz | Channels: 2 (Stereo) | Lossless: No
|
|
498
|
+
* File Size: 5.00 MB
|
|
499
|
+
* Title: Yesterday | Artist: The Beatles | Album: Help! | Year: 1965 | Genre: Rock, Pop
|
|
500
|
+
* Track: 1/14 | Composer: Lennon-McCartney
|
|
501
|
+
*
|
|
502
|
+
* --- Transcript ---
|
|
503
|
+
* [full transcribed text here]
|
|
504
|
+
* ```
|
|
505
|
+
*/
|
|
506
|
+
buildTextContent(filename, metadata, tags, transcript) {
|
|
507
|
+
const lines = [];
|
|
508
|
+
// Header line
|
|
509
|
+
lines.push(`[Audio File: ${filename}]`);
|
|
510
|
+
// Technical metadata line
|
|
511
|
+
const techParts = [];
|
|
512
|
+
techParts.push(`Duration: ${metadata.durationFormatted}`);
|
|
513
|
+
techParts.push(`Codec: ${metadata.codec}`);
|
|
514
|
+
if (metadata.codecProfile) {
|
|
515
|
+
techParts.push(`Profile: ${metadata.codecProfile}`);
|
|
516
|
+
}
|
|
517
|
+
if (metadata.bitrate) {
|
|
518
|
+
techParts.push(`Bitrate: ${this.formatBitrate(metadata.bitrate)}`);
|
|
519
|
+
}
|
|
520
|
+
if (metadata.sampleRate) {
|
|
521
|
+
techParts.push(`Sample Rate: ${metadata.sampleRate} Hz`);
|
|
522
|
+
}
|
|
523
|
+
if (metadata.channels) {
|
|
524
|
+
techParts.push(`Channels: ${metadata.channels} (${this.getChannelLabel(metadata.channels)})`);
|
|
525
|
+
}
|
|
526
|
+
if (metadata.bitsPerSample) {
|
|
527
|
+
techParts.push(`Bit Depth: ${metadata.bitsPerSample}-bit`);
|
|
528
|
+
}
|
|
529
|
+
techParts.push(`Lossless: ${metadata.lossless ? "Yes" : "No"}`);
|
|
530
|
+
lines.push(techParts.join(" | "));
|
|
531
|
+
// File size line
|
|
532
|
+
lines.push(`File Size: ${(metadata.fileSize / (1024 * 1024)).toFixed(2)} MB`);
|
|
533
|
+
// Tags line (only if any tags are present)
|
|
534
|
+
const tagParts = [];
|
|
535
|
+
if (tags.title) {
|
|
536
|
+
tagParts.push(`Title: ${tags.title}`);
|
|
537
|
+
}
|
|
538
|
+
if (tags.artist) {
|
|
539
|
+
tagParts.push(`Artist: ${tags.artist}`);
|
|
540
|
+
}
|
|
541
|
+
if (tags.album) {
|
|
542
|
+
tagParts.push(`Album: ${tags.album}`);
|
|
543
|
+
}
|
|
544
|
+
if (tags.year) {
|
|
545
|
+
tagParts.push(`Year: ${tags.year}`);
|
|
546
|
+
}
|
|
547
|
+
if (tags.genre && tags.genre.length > 0) {
|
|
548
|
+
tagParts.push(`Genre: ${tags.genre.join(", ")}`);
|
|
549
|
+
}
|
|
550
|
+
if (tagParts.length > 0) {
|
|
551
|
+
lines.push(tagParts.join(" | "));
|
|
552
|
+
}
|
|
553
|
+
// Secondary tags line (track, composer, comment)
|
|
554
|
+
const secondaryParts = [];
|
|
555
|
+
if (tags.track) {
|
|
556
|
+
const trackStr = tags.track.of !== null
|
|
557
|
+
? `${tags.track.no ?? "?"}/${tags.track.of}`
|
|
558
|
+
: `${tags.track.no ?? "?"}`;
|
|
559
|
+
secondaryParts.push(`Track: ${trackStr}`);
|
|
560
|
+
}
|
|
561
|
+
if (tags.composer) {
|
|
562
|
+
secondaryParts.push(`Composer: ${tags.composer}`);
|
|
563
|
+
}
|
|
564
|
+
if (tags.comment) {
|
|
565
|
+
secondaryParts.push(`Comment: ${tags.comment}`);
|
|
566
|
+
}
|
|
567
|
+
if (secondaryParts.length > 0) {
|
|
568
|
+
lines.push(secondaryParts.join(" | "));
|
|
569
|
+
}
|
|
570
|
+
// Transcript section (if transcription was performed)
|
|
571
|
+
if (transcript) {
|
|
572
|
+
lines.push("");
|
|
573
|
+
lines.push("--- Transcript ---");
|
|
574
|
+
lines.push(transcript);
|
|
575
|
+
}
|
|
576
|
+
return lines.join("\n");
|
|
577
|
+
}
|
|
578
|
+
// ===========================================================================
|
|
579
|
+
// PRIVATE: FORMATTING UTILITIES
|
|
580
|
+
// ===========================================================================
|
|
581
|
+
/**
|
|
582
|
+
* Format a duration in seconds to a human-readable string.
|
|
583
|
+
*
|
|
584
|
+
* @param seconds - Duration in seconds
|
|
585
|
+
* @returns Formatted string: "M:SS" for < 1 hour, "H:MM:SS" for >= 1 hour
|
|
586
|
+
*
|
|
587
|
+
* @example
|
|
588
|
+
* formatDuration(225) // "3:45"
|
|
589
|
+
* formatDuration(3750) // "1:02:30"
|
|
590
|
+
* formatDuration(0) // "0:00"
|
|
591
|
+
*/
|
|
592
|
+
formatDuration(seconds) {
|
|
593
|
+
if (!seconds || seconds <= 0) {
|
|
594
|
+
return "0:00";
|
|
595
|
+
}
|
|
596
|
+
const totalSeconds = Math.round(seconds);
|
|
597
|
+
const hours = Math.floor(totalSeconds / 3600);
|
|
598
|
+
const minutes = Math.floor((totalSeconds % 3600) / 60);
|
|
599
|
+
const secs = totalSeconds % 60;
|
|
600
|
+
if (hours > 0) {
|
|
601
|
+
return `${hours}:${String(minutes).padStart(2, "0")}:${String(secs).padStart(2, "0")}`;
|
|
602
|
+
}
|
|
603
|
+
return `${minutes}:${String(secs).padStart(2, "0")}`;
|
|
604
|
+
}
|
|
605
|
+
/**
|
|
606
|
+
* Format bitrate to a human-readable string.
|
|
607
|
+
*
|
|
608
|
+
* @param bitrate - Bitrate in bits per second
|
|
609
|
+
* @returns Formatted string (e.g., "320 kbps", "1411 kbps")
|
|
610
|
+
*/
|
|
611
|
+
formatBitrate(bitrate) {
|
|
612
|
+
const kbps = Math.round(bitrate / 1000);
|
|
613
|
+
return `${kbps} kbps`;
|
|
614
|
+
}
|
|
615
|
+
/**
|
|
616
|
+
* Get a human-readable label for the number of audio channels.
|
|
617
|
+
*
|
|
618
|
+
* @param channels - Number of audio channels
|
|
619
|
+
* @returns Channel label (e.g., "Mono", "Stereo", "5.1 Surround")
|
|
620
|
+
*/
|
|
621
|
+
getChannelLabel(channels) {
|
|
622
|
+
switch (channels) {
|
|
623
|
+
case 1:
|
|
624
|
+
return "Mono";
|
|
625
|
+
case 2:
|
|
626
|
+
return "Stereo";
|
|
627
|
+
case 6:
|
|
628
|
+
return "5.1 Surround";
|
|
629
|
+
case 8:
|
|
630
|
+
return "7.1 Surround";
|
|
631
|
+
default:
|
|
632
|
+
return `${channels}ch`;
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
// =============================================================================
|
|
637
|
+
// SINGLETON INSTANCE
|
|
638
|
+
// =============================================================================
|
|
639
|
+
/**
|
|
640
|
+
* Singleton Audio processor instance.
|
|
641
|
+
* Use this for standard audio processing operations.
|
|
642
|
+
*
|
|
643
|
+
* @example
|
|
644
|
+
* ```typescript
|
|
645
|
+
* import { audioProcessor } from "./AudioProcessor.js";
|
|
646
|
+
*
|
|
647
|
+
* const result = await audioProcessor.processFile(fileInfo);
|
|
648
|
+
* ```
|
|
649
|
+
*/
|
|
650
|
+
export const audioProcessor = new AudioProcessor();
|
|
651
|
+
// =============================================================================
|
|
652
|
+
// HELPER FUNCTIONS
|
|
653
|
+
// =============================================================================
|
|
654
|
+
/**
|
|
655
|
+
* Check if a file is an audio file.
|
|
656
|
+
* Matches by MIME type or file extension.
|
|
657
|
+
*
|
|
658
|
+
* @param mimetype - MIME type of the file
|
|
659
|
+
* @param filename - Filename (for extension-based detection)
|
|
660
|
+
* @returns true if the file is an audio file
|
|
661
|
+
*
|
|
662
|
+
* @example
|
|
663
|
+
* ```typescript
|
|
664
|
+
* if (isAudioFile('audio/mpeg', 'song.mp3')) {
|
|
665
|
+
* // Process as audio
|
|
666
|
+
* }
|
|
667
|
+
*
|
|
668
|
+
* if (isAudioFile('', 'recording.flac')) {
|
|
669
|
+
* // Also matches by extension
|
|
670
|
+
* }
|
|
671
|
+
* ```
|
|
672
|
+
*/
|
|
673
|
+
export function isAudioFile(mimetype, filename) {
|
|
674
|
+
return audioProcessor.isFileSupported(mimetype, filename);
|
|
675
|
+
}
|
|
676
|
+
/**
|
|
677
|
+
* Process a single audio file.
|
|
678
|
+
* Convenience function that uses the singleton processor.
|
|
679
|
+
*
|
|
680
|
+
* @param fileInfo - File information (can include URL or buffer)
|
|
681
|
+
* @param options - Optional processing options (auth headers, timeout, etc.)
|
|
682
|
+
* @returns Processing result with audio metadata or error
|
|
683
|
+
*
|
|
684
|
+
* @example
|
|
685
|
+
* ```typescript
|
|
686
|
+
* import { processAudio } from "./AudioProcessor.js";
|
|
687
|
+
*
|
|
688
|
+
* const result = await processAudio({
|
|
689
|
+
* id: 'audio-1',
|
|
690
|
+
* name: 'podcast.mp3',
|
|
691
|
+
* mimetype: 'audio/mpeg',
|
|
692
|
+
* size: 15728640,
|
|
693
|
+
* buffer: mp3Buffer,
|
|
694
|
+
* });
|
|
695
|
+
*
|
|
696
|
+
* if (result.success) {
|
|
697
|
+
* const { metadata, tags, textContent } = result.data;
|
|
698
|
+
* console.log(`${tags.title} by ${tags.artist} (${metadata.durationFormatted})`);
|
|
699
|
+
* // Send textContent to LLM for analysis
|
|
700
|
+
* } else {
|
|
701
|
+
* console.error(`Processing failed: ${result.error?.userMessage}`);
|
|
702
|
+
* }
|
|
703
|
+
* ```
|
|
704
|
+
*/
|
|
705
|
+
export async function processAudio(fileInfo, options) {
|
|
706
|
+
return audioProcessor.processFile(fileInfo, options);
|
|
707
|
+
}
|
|
708
|
+
//# sourceMappingURL=AudioProcessor.js.map
|