@aj-archipelago/cortex 1.0.2 → 1.0.4

package/tests/truncateMessages.test.js

@@ -0,0 +1,99 @@
+// ModelPlugin.test.js
+import test from 'ava';
+import ModelPlugin from '../graphql/plugins/modelPlugin.js';
+import { encode } from 'gpt-3-encoder';
+import { mockConfig, mockPathwayString } from './mocks.js';
+
+const config = mockConfig;
+const pathway = mockPathwayString;
+
+const modelPlugin = new ModelPlugin(config, pathway);
+
+const generateMessage = (role, content) => ({ role, content });
+
+test('truncateMessagesToTargetLength: should not modify messages if already within target length', (t) => {
+  const messages = [
+    generateMessage('user', 'Hello, how are you?'),
+    generateMessage('assistant', 'I am doing well, thank you!'),
+  ];
+  const targetTokenLength = encode(modelPlugin.messagesToChatML(messages, false)).length;
+
+  const result = modelPlugin.truncateMessagesToTargetLength(messages, targetTokenLength);
+  t.deepEqual(result, messages);
+});
+
+test('truncateMessagesToTargetLength: should remove messages from the front until target length is reached', (t) => {
+    const messages = [
+        generateMessage('user', 'Hello, how are you?'),
+        generateMessage('assistant', 'I am doing well, thank you!'),
+        generateMessage('user', 'What is your favorite color?'),
+    ];
+    const targetTokenLength = encode(modelPlugin.messagesToChatML(messages.slice(1), false)).length;
+
+    const result = modelPlugin.truncateMessagesToTargetLength(messages, targetTokenLength);
+    t.deepEqual(result, messages.slice(1));
+});
+
+test('truncateMessagesToTargetLength: should skip system messages', (t) => {
+    const messages = [
+      generateMessage('system', 'System message 1'),
+      generateMessage('user', 'Hello, how are you?'),
+      generateMessage('assistant', 'I am doing well, thank you!'),
+    ];
+    const targetTokenLength = encode(modelPlugin.messagesToChatML([messages[0], ...messages.slice(2)], false)).length;
+  
+    const result = modelPlugin.truncateMessagesToTargetLength(messages, targetTokenLength);
+    t.deepEqual(result, [messages[0], ...messages.slice(2)]);
+});
+
+test('truncateMessagesToTargetLength: should truncate messages to fit target length', (t) => {
+  const messages = [
+    generateMessage('user', 'Hello, how are you?'),
+    generateMessage('assistant', 'I am doing well, thank you!'),
+  ];
+  const targetTokenLength = encode(modelPlugin.messagesToChatML(messages, false)).length - 4;
+
+  const result = modelPlugin.truncateMessagesToTargetLength(messages, targetTokenLength);
+  t.true(result.every((message, index) => message.content.length <= messages[index].content.length));
+  t.true(encode(modelPlugin.messagesToChatML(result, false)).length <= targetTokenLength);
+});
+
+test('truncateMessagesToTargetLength: should remove messages entirely if they need to be empty to fit target length', (t) => {
+  const messages = [
+    generateMessage('user', 'Hello, how are you?'),
+    generateMessage('assistant', 'I am doing well, thank you!'),
+  ];
+  const targetTokenLength = encode(modelPlugin.messagesToChatML(messages.slice(1), false)).length;
+
+  const result = modelPlugin.truncateMessagesToTargetLength(messages, targetTokenLength);
+  t.deepEqual(result, messages.slice(1));
+});
+
+test('truncateMessagesToTargetLength: should return an empty array if target length is 0', (t) => {
+  const messages = [
+    generateMessage('user', 'Hello, how are you?'),
+    generateMessage('assistant', 'I am doing well, thank you!'),
+  ];
+
+  const result = modelPlugin.truncateMessagesToTargetLength(messages, 0);
+  t.deepEqual(result, []);
+});
+
+test('truncateMessagesToTargetLength: should not remove system messages even if they are too long', (t) => {
+  const messages = [
+      generateMessage('user', 'Hello, how are you?'),
+      generateMessage('system', 'System message content that is very long and exceeds the target token length'),
+      generateMessage('assistant', 'I am fine, thank you.'),
+  ];
+
+  const targetTokenLength = 20;
+  const result = modelPlugin.truncateMessagesToTargetLength(messages, targetTokenLength);
+
+  const systemMessage = result.find((message) => message.role === 'system');
+  t.truthy(systemMessage, 'System message should not be removed');
+  t.is(
+      systemMessage.content,
+      'System message content that is very long and exceeds the target token length',
+      'System message content should not be altered'
+  );
+});

package/lib/fileChunker.js

@@ -1,147 +0,0 @@
-import fs from 'fs';
-import { path as ffmpegPath } from '@ffmpeg-installer/ffmpeg';
-import ffmpeg from 'fluent-ffmpeg';
-ffmpeg.setFfmpegPath(ffmpegPath);
-import path from 'path';
-import { v4 as uuidv4 } from 'uuid';
-import os from 'os';
-import util from 'util';
-import { pipeline } from 'stream';
-
-const ffmpegProbe = util.promisify(ffmpeg.ffprobe);
-
-const cPipeline = util.promisify(pipeline);
-
-import ytdl from 'ytdl-core';
-
-
-async function processChunk(inputPath, outputFileName, start, duration) {
-    return new Promise((resolve, reject) => {
-        ffmpeg(inputPath)
-            .seekInput(start)
-            .duration(duration)
-            .on('start', (cmd) => {
-                console.log(`Started FFmpeg with command: ${cmd}`);
-            })
-            .on('error', (err) => {
-                console.error(`Error occurred while processing chunk:`, err);
-                reject(err);
-            })
-            .on('end', () => {
-                console.log(`Finished processing chunk`);
-                resolve(outputFileName);
-            })
-            .save(outputFileName);
-    });
-}
-
-const generateUniqueFolderName = () => {
-    const uniqueFolderName = uuidv4();
-    const tempFolderPath = os.tmpdir(); // Get the system's temporary folder
-    const uniqueOutputPath = path.join(tempFolderPath, uniqueFolderName);
-    return uniqueOutputPath;
-}
-
-async function splitMediaFile(inputPath, chunkDurationInSeconds = 600) {
-    try {
-        const metadata = await ffmpegProbe(inputPath);
-        const duration = metadata.format.duration;
-        const numChunks = Math.ceil((duration - 1) / chunkDurationInSeconds);
-
-        const chunkPromises = [];
-
-        const uniqueOutputPath = generateUniqueFolderName();
-
-        // Create unique folder
-        fs.mkdirSync(uniqueOutputPath, { recursive: true });
-
-
-        for (let i = 0; i < numChunks; i++) {
-            const outputFileName = path.join(
-                uniqueOutputPath,
-                `chunk-${i + 1}-${path.basename(inputPath)}`
-            );
-
-            const chunkPromise = processChunk(
-                inputPath,
-                outputFileName,
-                i * chunkDurationInSeconds,
-                chunkDurationInSeconds
-            );
-
-            chunkPromises.push(chunkPromise);
-        }
-
-        // const chunkedFiles = await Promise.all(chunkPromises);
-        // console.log('All chunks processed. Chunked file names:', chunkedFiles);
-        // return { chunks: chunkedFiles, folder: uniqueOutputPath }
-        return { chunkPromises, uniqueOutputPath }
-    } catch (err) {
-        console.error('Error occurred during the splitting process:', err);
-    }
-}
-
-async function deleteTempPath(path) {
-    try {
-        if (!path) return;
-        const stats = fs.statSync(path);
-        if (stats.isFile()) {
-            fs.unlinkSync(path);
-            console.log(`Temporary file ${path} deleted successfully.`);
-        } else if (stats.isDirectory()) {
-            fs.rmdirSync(path, { recursive: true });
-            console.log(`Temporary folder ${path} and its contents deleted successfully.`);
-        }
-    } catch (err) {
-        console.error('Error occurred while deleting the temporary path:', err);
-    }
-}
-
-
-function isValidYoutubeUrl(url) {
-    const regex = /^(https?:\/\/)?(www\.)?(youtube\.com|youtu\.?be)\/.+$/;
-    return regex.test(url);
-}
-
-function convertYoutubeToMp3Stream(video) {
-    // Configure ffmpeg to convert the video to mp3
-    const mp3Stream = ffmpeg(video)
-        .withAudioCodec('libmp3lame')
-        .toFormat('mp3')
-        .on('error', (err) => {
-            console.error(`An error occurred during conversion: ${err.message}`);
-        });
-
-    return mp3Stream;
-}
-
-async function pipeStreamToFile(stream, filePath) {
-    try {
-        await cPipeline(stream, fs.createWriteStream(filePath));
-        console.log('Stream piped to file successfully.');
-    } catch (error) {
-        console.error(`Error piping stream to file: ${error.message}`);
-    }
-}
-
-
-const processYoutubeUrl = async (url) => {
-    const info = await ytdl.getInfo(url);
-    const audioFormat = ytdl.chooseFormat(info.formats, { quality: 'highestaudio' });
-
-    if (!audioFormat) {
-        throw new Error('No suitable audio format found');
-    }
-
-    const stream = ytdl.downloadFromInfo(info, { format: audioFormat });
-    // const stream = ytdl(url, { filter: 'audioonly' })
-
-    const mp3Stream = convertYoutubeToMp3Stream(stream);
-    const outputFileName = path.join(os.tmpdir(), `${uuidv4()}.mp3`);
-    await pipeStreamToFile(mp3Stream, outputFileName); // You can also pipe the stream to a file
-    return outputFileName;
-}
-
-export {
-    splitMediaFile, deleteTempPath, processYoutubeUrl, isValidYoutubeUrl
-};