@aj-archipelago/cortex 1.0.2 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.eslintrc +1 -1
- package/README.md +8 -6
- package/config.js +7 -2
- package/graphql/parser.js +6 -0
- package/graphql/pathwayPrompter.js +2 -17
- package/graphql/pathwayResolver.js +10 -8
- package/graphql/pathwayResponseParser.js +13 -4
- package/graphql/plugins/modelPlugin.js +27 -18
- package/graphql/plugins/openAiCompletionPlugin.js +29 -12
- package/graphql/plugins/openAiWhisperPlugin.js +112 -19
- package/helper_apps/MediaFileChunker/blobHandler.js +150 -0
- package/helper_apps/MediaFileChunker/fileChunker.js +123 -0
- package/helper_apps/MediaFileChunker/function.json +20 -0
- package/helper_apps/MediaFileChunker/helper.js +33 -0
- package/helper_apps/MediaFileChunker/index.js +116 -0
- package/helper_apps/MediaFileChunker/localFileHandler.js +36 -0
- package/helper_apps/MediaFileChunker/package-lock.json +2919 -0
- package/helper_apps/MediaFileChunker/package.json +22 -0
- package/helper_apps/MediaFileChunker/redis.js +32 -0
- package/helper_apps/MediaFileChunker/start.js +27 -0
- package/lib/handleBars.js +26 -0
- package/lib/pathwayTools.js +15 -0
- package/lib/redisSubscription.js +51 -0
- package/lib/request.js +4 -4
- package/package.json +5 -6
- package/pathways/transcribe.js +2 -1
- package/tests/config.test.js +69 -0
- package/tests/handleBars.test.js +43 -0
- package/tests/mocks.js +39 -0
- package/tests/modelPlugin.test.js +129 -0
- package/tests/pathwayResolver.test.js +77 -0
- package/tests/truncateMessages.test.js +99 -0
- package/lib/fileChunker.js +0 -147
package/.eslintrc
CHANGED
package/README.md
CHANGED
|
@@ -4,13 +4,15 @@ Cortex simplifies and accelerates the process of creating applications that harn
|
|
|
4
4
|
Modern AI models are transformational, but a number of complexities emerge when developers start using them to deliver application-ready functions. Most models require precisely formatted, carefully engineered and sequenced prompts to produce consistent results, and the responses are typically largely unstructured text without validation or formatting. Additionally, these models are evolving rapidly, are typically costly and slow to query and implement hard request size and rate restrictions that need to be carefully navigated for optimum throughput. Cortex offers a solution to these problems and provides a simple and extensible package for interacting with NL AI models.
|
|
5
5
|
|
|
6
6
|
## Okay, but what can I really do with this thing?
|
|
7
|
-
|
|
8
|
-
* Create custom chat agents with memory and personalization and then expose
|
|
9
|
-
*
|
|
10
|
-
* Create powerful AI editing tools (copy editing, paraphrasing, summarization, etc.) tools for your company and then integrate them with your existing workflow tools without having to build all the LLM-handling logic into those tools.
|
|
7
|
+
Just about anything! It's kind of an LLM swiss army knife. Here are some ideas:
|
|
8
|
+
* Create custom chat agents with memory and personalization and then expose them through a bunch of different UIs (custom chat portals, Slack, Microsoft Teams, etc. - anything that can be extended and speak to a REST or GraphQL endpoint)
|
|
9
|
+
* Spin up LLM powered automatons with their prompting logic and AI API handling logic all centrally encapsulated.
|
|
11
10
|
* Make LLM chains and agents from LangChain.js available via scalable REST or GraphQL endpoints.
|
|
12
11
|
* Put a REST or GraphQL front end on your locally-run models (e.g. llama.cpp) and use them in concert with other tools.
|
|
13
|
-
*
|
|
12
|
+
* Create modular custom coding assistants (code generation, code reviews, test writing, AI pair programming) and easily integrate them with your existing editing tools.
|
|
13
|
+
* Create powerful AI editing tools (copy editing, paraphrasing, summarization, etc.) for your company and then integrate them with your existing workflow tools without having to build all the LLM-handling logic into those tools.
|
|
14
|
+
* Create cached endpoints for functions with repeated calls so the results return instantly and you don't run up LLM token charges.
|
|
15
|
+
* Route all of your company's LLM access through a single API layer to optimize and monitor usage and centrally control rate limiting and which models are being used.
|
|
14
16
|
|
|
15
17
|
## Features
|
|
16
18
|
|
|
@@ -34,7 +36,7 @@ npm install
|
|
|
34
36
|
export OPENAI_API_KEY=<your key>
|
|
35
37
|
npm start
|
|
36
38
|
```
|
|
37
|
-
Yup, that's it, at least in the simplest possible case. That will get you access to all of the built in pathways.
|
|
39
|
+
Yup, that's it, at least in the simplest possible case. That will get you access to all of the built in pathways. If you prefer to use npm instead instead of cloning, we have an npm package too: [@aj-archipelago/cortex](https://www.npmjs.com/package/@aj-archipelago/cortex)
|
|
38
40
|
## Connecting Applications to Cortex
|
|
39
41
|
Cortex speaks GraphQL and by default it enables the GraphQL playground. If you're just using default options, that's at [http://localhost:4000/graphql](http://localhost:4000/graphql). From there you can begin making requests and test out the pathways (listed under Query) to your heart's content. If GraphQL isn't your thing or if you have a client that would rather have REST that's fine - Cortex speaks REST as well.
|
|
40
42
|
|
package/config.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import path from 'path';
|
|
2
2
|
const __dirname = path.dirname(new URL(import.meta.url).pathname);
|
|
3
3
|
import convict from 'convict';
|
|
4
|
-
import
|
|
4
|
+
import HandleBars from './lib/handleBars.js';
|
|
5
5
|
import fs from 'fs';
|
|
6
6
|
|
|
7
7
|
// Schema for config
|
|
@@ -110,6 +110,11 @@ var config = convict({
|
|
|
110
110
|
default: null,
|
|
111
111
|
env: 'CORTEX_CONFIG_FILE'
|
|
112
112
|
},
|
|
113
|
+
whisperMediaApiUrl: {
|
|
114
|
+
format: String,
|
|
115
|
+
default: 'null',
|
|
116
|
+
env: 'WHISPER_MEDIA_API_URL'
|
|
117
|
+
},
|
|
113
118
|
});
|
|
114
119
|
|
|
115
120
|
// Read in environment variables and set up service configuration
|
|
@@ -167,7 +172,7 @@ const buildModels = (config) => {
|
|
|
167
172
|
|
|
168
173
|
for (const [key, model] of Object.entries(models)) {
|
|
169
174
|
// Compile handlebars templates for models
|
|
170
|
-
models[key] = JSON.parse(
|
|
175
|
+
models[key] = JSON.parse(HandleBars.compile(JSON.stringify(model))({ ...config.getEnv(), ...config.getProperties() }))
|
|
171
176
|
}
|
|
172
177
|
|
|
173
178
|
// Add constructed models to config
|
package/graphql/parser.js
CHANGED
|
@@ -32,8 +32,14 @@ const parseNumberedObjectList = (text, format) => {
|
|
|
32
32
|
return result;
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
+
// parse a comma-separated list text format into list
|
|
36
|
+
const parseCommaSeparatedList = (str) => {
|
|
37
|
+
return str.split(',').map(s => s.trim()).filter(s => s.length);
|
|
38
|
+
}
|
|
39
|
+
|
|
35
40
|
export {
|
|
36
41
|
regexParser,
|
|
37
42
|
parseNumberedList,
|
|
38
43
|
parseNumberedObjectList,
|
|
44
|
+
parseCommaSeparatedList,
|
|
39
45
|
};
|
|
@@ -4,21 +4,6 @@ import OpenAICompletionPlugin from './plugins/openAiCompletionPlugin.js';
|
|
|
4
4
|
import AzureTranslatePlugin from './plugins/azureTranslatePlugin.js';
|
|
5
5
|
import OpenAIWhisperPlugin from './plugins/openAiWhisperPlugin.js';
|
|
6
6
|
import LocalModelPlugin from './plugins/localModelPlugin.js';
|
|
7
|
-
import handlebars from 'handlebars';
|
|
8
|
-
|
|
9
|
-
// register functions that can be called directly in the prompt markdown
|
|
10
|
-
handlebars.registerHelper('stripHTML', function (value) {
|
|
11
|
-
return value.replace(/<[^>]*>/g, '');
|
|
12
|
-
});
|
|
13
|
-
|
|
14
|
-
handlebars.registerHelper('now', function () {
|
|
15
|
-
return new Date().toISOString();
|
|
16
|
-
});
|
|
17
|
-
|
|
18
|
-
handlebars.registerHelper('toJSON', function (object) {
|
|
19
|
-
return JSON.stringify(object);
|
|
20
|
-
});
|
|
21
|
-
|
|
22
7
|
|
|
23
8
|
class PathwayPrompter {
|
|
24
9
|
constructor({ config, pathway }) {
|
|
@@ -27,7 +12,7 @@ class PathwayPrompter {
|
|
|
27
12
|
const model = config.get('models')[modelName];
|
|
28
13
|
|
|
29
14
|
if (!model) {
|
|
30
|
-
throw new
|
|
15
|
+
throw new Error(`Model ${modelName} not found in config`);
|
|
31
16
|
}
|
|
32
17
|
|
|
33
18
|
let plugin;
|
|
@@ -49,7 +34,7 @@ class PathwayPrompter {
|
|
|
49
34
|
plugin = new LocalModelPlugin(config, pathway);
|
|
50
35
|
break;
|
|
51
36
|
default:
|
|
52
|
-
throw new
|
|
37
|
+
throw new Error(`Unsupported model type: ${model.type}`);
|
|
53
38
|
}
|
|
54
39
|
|
|
55
40
|
this.plugin = plugin;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { PathwayPrompter } from './pathwayPrompter.js';
|
|
2
|
+
// eslint-disable-next-line import/no-extraneous-dependencies
|
|
2
3
|
import { v4 as uuidv4 } from 'uuid';
|
|
3
4
|
import pubsub from './pubsub.js';
|
|
4
5
|
import { encode } from 'gpt-3-encoder';
|
|
@@ -7,11 +8,7 @@ import { PathwayResponseParser } from './pathwayResponseParser.js';
|
|
|
7
8
|
import { Prompt } from './prompt.js';
|
|
8
9
|
import { getv, setv } from '../lib/keyValueStorageClient.js';
|
|
9
10
|
import { requestState } from './requestState.js';
|
|
10
|
-
|
|
11
|
-
const callPathway = async (config, pathwayName, args, requestState, { text, ...parameters }) => {
|
|
12
|
-
const pathwayResolver = new PathwayResolver({ config, pathway: config.get(`pathways.${pathwayName}`), args, requestState });
|
|
13
|
-
return await pathwayResolver.resolve({ text, ...parameters });
|
|
14
|
-
}
|
|
11
|
+
import { callPathway } from '../lib/pathwayTools.js';
|
|
15
12
|
|
|
16
13
|
class PathwayResolver {
|
|
17
14
|
constructor({ config, pathway, args }) {
|
|
@@ -139,6 +136,12 @@ class PathwayResolver {
|
|
|
139
136
|
return this.responseParser.parse(data);
|
|
140
137
|
}
|
|
141
138
|
|
|
139
|
+
// Add a warning and log it
|
|
140
|
+
logWarning(warning) {
|
|
141
|
+
this.warnings.push(warning);
|
|
142
|
+
console.warn(warning);
|
|
143
|
+
}
|
|
144
|
+
|
|
142
145
|
// Here we choose how to handle long input - either summarize or chunk
|
|
143
146
|
processInputText(text) {
|
|
144
147
|
let chunkTokenLength = 0;
|
|
@@ -151,8 +154,7 @@ class PathwayResolver {
|
|
|
151
154
|
if (!this.useInputChunking || encoded.length <= chunkTokenLength) { // no chunking, return as is
|
|
152
155
|
if (encoded.length > 0 && encoded.length >= chunkTokenLength) {
|
|
153
156
|
const warnText = `Truncating long input text. Text length: ${text.length}`;
|
|
154
|
-
this.
|
|
155
|
-
console.warn(warnText);
|
|
157
|
+
this.logWarning(warnText);
|
|
156
158
|
text = this.truncate(text, chunkTokenLength);
|
|
157
159
|
}
|
|
158
160
|
return [text];
|
|
@@ -171,7 +173,7 @@ class PathwayResolver {
|
|
|
171
173
|
|
|
172
174
|
async summarizeIfEnabled({ text, ...parameters }) {
|
|
173
175
|
if (this.pathway.useInputSummarization) {
|
|
174
|
-
return await callPathway(this.config, 'summary', this.args,
|
|
176
|
+
return await callPathway(this.config, 'summary', { ...this.args, ...parameters, targetLength: 0});
|
|
175
177
|
}
|
|
176
178
|
return text;
|
|
177
179
|
}
|
|
@@ -1,20 +1,29 @@
|
|
|
1
|
-
import { parseNumberedList, parseNumberedObjectList } from './parser.js';
|
|
1
|
+
import { parseNumberedList, parseNumberedObjectList, parseCommaSeparatedList } from './parser.js';
|
|
2
2
|
|
|
3
3
|
class PathwayResponseParser {
|
|
4
4
|
constructor(pathway) {
|
|
5
5
|
this.pathway = pathway;
|
|
6
6
|
}
|
|
7
7
|
|
|
8
|
+
isCommaSeparatedList(data) {
|
|
9
|
+
const commaSeparatedPattern = /^([^,\n]+,)+[^,\n]+$/;
|
|
10
|
+
return commaSeparatedPattern.test(data.trim());
|
|
11
|
+
}
|
|
12
|
+
|
|
8
13
|
parse(data) {
|
|
9
14
|
if (this.pathway.parser) {
|
|
10
15
|
return this.pathway.parser(data);
|
|
11
16
|
}
|
|
12
17
|
|
|
13
18
|
if (this.pathway.list) {
|
|
14
|
-
if (this.
|
|
15
|
-
return
|
|
19
|
+
if (this.isCommaSeparatedList(data)) {
|
|
20
|
+
return parseCommaSeparatedList(data);
|
|
21
|
+
} else {
|
|
22
|
+
if (this.pathway.format) {
|
|
23
|
+
return parseNumberedObjectList(data, this.pathway.format);
|
|
24
|
+
}
|
|
25
|
+
return parseNumberedList(data);
|
|
16
26
|
}
|
|
17
|
-
return parseNumberedList(data)
|
|
18
27
|
}
|
|
19
28
|
|
|
20
29
|
return data;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// ModelPlugin.js
|
|
2
|
-
import
|
|
2
|
+
import HandleBars from '../../lib/handleBars.js';
|
|
3
3
|
|
|
4
4
|
import { request } from '../../lib/request.js';
|
|
5
5
|
import { encode } from 'gpt-3-encoder';
|
|
@@ -58,7 +58,7 @@ class ModelPlugin {
|
|
|
58
58
|
|
|
59
59
|
// Remove and/or truncate messages until the target token length is reached
|
|
60
60
|
let index = 0;
|
|
61
|
-
while (totalTokenLength > targetTokenLength) {
|
|
61
|
+
while ((totalTokenLength > targetTokenLength) && (index < tokenLengths.length)) {
|
|
62
62
|
const message = tokenLengths[index].message;
|
|
63
63
|
|
|
64
64
|
// Skip system messages
|
|
@@ -79,19 +79,28 @@ class ModelPlugin {
|
|
|
79
79
|
const otherMessageTokens = totalTokenLength - currentTokenLength;
|
|
80
80
|
const tokensToKeep = targetTokenLength - (otherMessageTokens + emptyContentLength);
|
|
81
81
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
82
|
+
if (tokensToKeep <= 0) {
|
|
83
|
+
// If the message needs to be empty to make the target, remove it entirely
|
|
84
|
+
totalTokenLength -= currentTokenLength;
|
|
85
|
+
tokenLengths.splice(index, 1);
|
|
86
|
+
} else {
|
|
87
|
+
// Otherwise, update the message and token length
|
|
88
|
+
const truncatedContent = getFirstNToken(message.content, tokensToKeep);
|
|
89
|
+
const truncatedMessage = { ...message, content: truncatedContent };
|
|
89
90
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
91
|
+
tokenLengths[index] = {
|
|
92
|
+
message: truncatedMessage,
|
|
93
|
+
tokenLength: encode(this.messagesToChatML([ truncatedMessage ], false)).length
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// calculate the length again to keep us honest
|
|
97
|
+
totalTokenLength = tokenLengths.reduce(
|
|
98
|
+
(sum, { tokenLength }) => sum + tokenLength,
|
|
99
|
+
0
|
|
100
|
+
);
|
|
101
|
+
|
|
102
|
+
index++;
|
|
103
|
+
}
|
|
95
104
|
}
|
|
96
105
|
}
|
|
97
106
|
|
|
@@ -118,7 +127,7 @@ class ModelPlugin {
|
|
|
118
127
|
getCompiledPrompt(text, parameters, prompt) {
|
|
119
128
|
const combinedParameters = { ...this.promptParameters, ...parameters };
|
|
120
129
|
const modelPrompt = this.getModelPrompt(prompt, parameters);
|
|
121
|
-
const modelPromptText = modelPrompt.prompt ?
|
|
130
|
+
const modelPromptText = modelPrompt.prompt ? HandleBars.compile(modelPrompt.prompt)({ ...combinedParameters, text }) : '';
|
|
122
131
|
const modelPromptMessages = this.getModelPromptMessages(modelPrompt, combinedParameters, text);
|
|
123
132
|
const modelPromptMessagesML = this.messagesToChatML(modelPromptMessages);
|
|
124
133
|
|
|
@@ -135,7 +144,7 @@ class ModelPlugin {
|
|
|
135
144
|
|
|
136
145
|
getPromptTokenRatio() {
|
|
137
146
|
// TODO: Is this the right order of precedence? inputParameters should maybe be second?
|
|
138
|
-
return this.promptParameters.inputParameters
|
|
147
|
+
return this.promptParameters.inputParameters?.tokenRatio ?? this.promptParameters.tokenRatio ?? DEFAULT_PROMPT_TOKEN_RATIO;
|
|
139
148
|
}
|
|
140
149
|
|
|
141
150
|
|
|
@@ -155,7 +164,7 @@ class ModelPlugin {
|
|
|
155
164
|
// First run handlebars compile on the pathway messages
|
|
156
165
|
const compiledMessages = modelPrompt.messages.map((message) => {
|
|
157
166
|
if (message.content) {
|
|
158
|
-
const compileText =
|
|
167
|
+
const compileText = HandleBars.compile(message.content);
|
|
159
168
|
return {
|
|
160
169
|
role: message.role,
|
|
161
170
|
content: compileText({ ...combinedParameters, text }),
|
|
@@ -184,7 +193,7 @@ class ModelPlugin {
|
|
|
184
193
|
}
|
|
185
194
|
|
|
186
195
|
requestUrl() {
|
|
187
|
-
const generateUrl =
|
|
196
|
+
const generateUrl = HandleBars.compile(this.model.url);
|
|
188
197
|
return generateUrl({ ...this.model, ...this.environmentVariables, ...this.config });
|
|
189
198
|
}
|
|
190
199
|
|
|
@@ -1,15 +1,27 @@
|
|
|
1
1
|
// OpenAICompletionPlugin.js
|
|
2
|
+
|
|
2
3
|
import ModelPlugin from './modelPlugin.js';
|
|
3
4
|
|
|
4
5
|
import { encode } from 'gpt-3-encoder';
|
|
5
6
|
|
|
7
|
+
// Helper function to truncate the prompt if it is too long
|
|
8
|
+
const truncatePromptIfNecessary = (text, textTokenCount, modelMaxTokenCount, targetTextTokenCount, pathwayResolver) => {
|
|
9
|
+
const maxAllowedTextTokenCount = textTokenCount + ((modelMaxTokenCount - targetTextTokenCount) * 0.5);
|
|
10
|
+
|
|
11
|
+
if (textTokenCount > maxAllowedTextTokenCount) {
|
|
12
|
+
pathwayResolver.logWarning(`Prompt is too long at ${textTokenCount} tokens (this target token length for this pathway is ${targetTextTokenCount} tokens because the response is expected to take up the rest of the model's max tokens (${modelMaxTokenCount}). Prompt will be truncated.`);
|
|
13
|
+
return pathwayResolver.truncate(text, maxAllowedTextTokenCount);
|
|
14
|
+
}
|
|
15
|
+
return text;
|
|
16
|
+
}
|
|
17
|
+
|
|
6
18
|
class OpenAICompletionPlugin extends ModelPlugin {
|
|
7
19
|
constructor(config, pathway) {
|
|
8
20
|
super(config, pathway);
|
|
9
21
|
}
|
|
10
22
|
|
|
11
23
|
// Set up parameters specific to the OpenAI Completion API
|
|
12
|
-
getRequestParameters(text, parameters, prompt) {
|
|
24
|
+
getRequestParameters(text, parameters, prompt, pathwayResolver) {
|
|
13
25
|
let { modelPromptMessages, modelPromptText, tokenLength } = this.getCompiledPrompt(text, parameters, prompt);
|
|
14
26
|
const { stream } = parameters;
|
|
15
27
|
let modelPromptMessagesML = '';
|
|
@@ -23,12 +35,14 @@ class OpenAICompletionPlugin extends ModelPlugin {
|
|
|
23
35
|
const requestMessages = this.truncateMessagesToTargetLength(modelPromptMessages, (modelTargetTokenLength - addAssistantTokens));
|
|
24
36
|
modelPromptMessagesML = this.messagesToChatML(requestMessages);
|
|
25
37
|
tokenLength = encode(modelPromptMessagesML).length;
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
}
|
|
30
|
-
|
|
38
|
+
|
|
39
|
+
modelPromptMessagesML = truncatePromptIfNecessary(modelPromptMessagesML, tokenLength, this.getModelMaxTokenLength(), modelTargetTokenLength, pathwayResolver);
|
|
40
|
+
|
|
31
41
|
const max_tokens = this.getModelMaxTokenLength() - tokenLength;
|
|
42
|
+
|
|
43
|
+
if (max_tokens < 0) {
|
|
44
|
+
throw new Error(`Prompt is too long to successfully call the model at ${tokenLength} tokens. The model will not be called.`);
|
|
45
|
+
}
|
|
32
46
|
|
|
33
47
|
requestParameters = {
|
|
34
48
|
prompt: modelPromptMessagesML,
|
|
@@ -41,11 +55,14 @@ class OpenAICompletionPlugin extends ModelPlugin {
|
|
|
41
55
|
stream
|
|
42
56
|
};
|
|
43
57
|
} else {
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
58
|
+
|
|
59
|
+
modelPromptText = truncatePromptIfNecessary(modelPromptText, tokenLength, this.getModelMaxTokenLength(), modelTargetTokenLength, pathwayResolver);
|
|
60
|
+
|
|
48
61
|
const max_tokens = this.getModelMaxTokenLength() - tokenLength;
|
|
62
|
+
|
|
63
|
+
if (max_tokens < 0) {
|
|
64
|
+
throw new Error(`Prompt is too long to successfully call the model at ${tokenLength} tokens. The model will not be called.`);
|
|
65
|
+
}
|
|
49
66
|
|
|
50
67
|
requestParameters = {
|
|
51
68
|
prompt: modelPromptText,
|
|
@@ -59,9 +76,9 @@ class OpenAICompletionPlugin extends ModelPlugin {
|
|
|
59
76
|
}
|
|
60
77
|
|
|
61
78
|
// Execute the request to the OpenAI Completion API
|
|
62
|
-
async execute(text, parameters, prompt) {
|
|
79
|
+
async execute(text, parameters, prompt, pathwayResolver) {
|
|
63
80
|
const url = this.requestUrl(text);
|
|
64
|
-
const requestParameters = this.getRequestParameters(text, parameters, prompt);
|
|
81
|
+
const requestParameters = this.getRequestParameters(text, parameters, prompt, pathwayResolver);
|
|
65
82
|
|
|
66
83
|
const data = { ...(this.model.params || {}), ...requestParameters };
|
|
67
84
|
const params = {};
|
|
@@ -1,16 +1,90 @@
|
|
|
1
1
|
// openAiWhisperPlugin.js
|
|
2
2
|
import ModelPlugin from './modelPlugin.js';
|
|
3
|
-
|
|
4
3
|
import FormData from 'form-data';
|
|
5
4
|
import fs from 'fs';
|
|
6
|
-
import { splitMediaFile, isValidYoutubeUrl, processYoutubeUrl, deleteTempPath } from '../../lib/fileChunker.js';
|
|
7
5
|
import pubsub from '../pubsub.js';
|
|
6
|
+
import { axios } from '../../lib/request.js';
|
|
7
|
+
import stream from 'stream';
|
|
8
|
+
import os from 'os';
|
|
9
|
+
import path from 'path';
|
|
10
|
+
import { v4 as uuidv4 } from 'uuid';
|
|
11
|
+
import { config } from '../../config.js';
|
|
12
|
+
import { deleteTempPath } from '../../helper_apps/MediaFileChunker/helper.js';
|
|
13
|
+
import http from 'http';
|
|
14
|
+
import https from 'https';
|
|
15
|
+
import url from 'url';
|
|
16
|
+
import { promisify } from 'util';
|
|
17
|
+
const pipeline = promisify(stream.pipeline);
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
const API_URL = config.get('whisperMediaApiUrl');
|
|
21
|
+
|
|
22
|
+
function generateUniqueFilename(extension) {
|
|
23
|
+
return `${uuidv4()}.${extension}`;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const downloadFile = async (fileUrl) => {
|
|
27
|
+
const fileExtension = path.extname(fileUrl).slice(1);
|
|
28
|
+
const uniqueFilename = generateUniqueFilename(fileExtension);
|
|
29
|
+
const tempDir = os.tmpdir();
|
|
30
|
+
const localFilePath = `${tempDir}/${uniqueFilename}`;
|
|
31
|
+
|
|
32
|
+
// eslint-disable-next-line no-async-promise-executor
|
|
33
|
+
return new Promise(async (resolve, reject) => {
|
|
34
|
+
try {
|
|
35
|
+
const parsedUrl = url.parse(fileUrl);
|
|
36
|
+
const protocol = parsedUrl.protocol === 'https:' ? https : http;
|
|
37
|
+
|
|
38
|
+
const response = await new Promise((resolve, reject) => {
|
|
39
|
+
protocol.get(parsedUrl, (res) => {
|
|
40
|
+
if (res.statusCode === 200) {
|
|
41
|
+
resolve(res);
|
|
42
|
+
} else {
|
|
43
|
+
reject(new Error(`HTTP request failed with status code ${res.statusCode}`));
|
|
44
|
+
}
|
|
45
|
+
}).on('error', reject);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
await pipeline(response, fs.createWriteStream(localFilePath));
|
|
49
|
+
console.log(`Downloaded file to ${localFilePath}`);
|
|
50
|
+
resolve(localFilePath);
|
|
51
|
+
} catch (error) {
|
|
52
|
+
fs.unlink(localFilePath, () => {
|
|
53
|
+
reject(error);
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
});
|
|
57
|
+
};
|
|
8
58
|
|
|
9
59
|
class OpenAIWhisperPlugin extends ModelPlugin {
|
|
10
60
|
constructor(config, pathway) {
|
|
11
61
|
super(config, pathway);
|
|
12
62
|
}
|
|
13
63
|
|
|
64
|
+
async getMediaChunks(file, requestId) {
|
|
65
|
+
try {
|
|
66
|
+
if (API_URL) {
|
|
67
|
+
//call helper api and get list of file uris
|
|
68
|
+
const res = await axios.get(API_URL, { params: { uri: file, requestId } });
|
|
69
|
+
return res.data;
|
|
70
|
+
} else {
|
|
71
|
+
console.log(`No API_URL set, returning file as chunk`);
|
|
72
|
+
return [file];
|
|
73
|
+
}
|
|
74
|
+
} catch (err) {
|
|
75
|
+
console.log(`Error getting media chunks list from api:`, err);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
async markCompletedForCleanUp(requestId) {
|
|
80
|
+
if (API_URL) {
|
|
81
|
+
//call helper api to mark processing as completed
|
|
82
|
+
const res = await axios.delete(API_URL, { params: { requestId } });
|
|
83
|
+
console.log(`Marked request ${requestId} as completed:`, res.data);
|
|
84
|
+
return res.data;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
14
88
|
// Execute the request to the OpenAI Whisper API
|
|
15
89
|
async execute(text, parameters, prompt, pathwayResolver) {
|
|
16
90
|
const url = this.requestUrl(text);
|
|
@@ -19,11 +93,12 @@ class OpenAIWhisperPlugin extends ModelPlugin {
|
|
|
19
93
|
|
|
20
94
|
const processChunk = async (chunk) => {
|
|
21
95
|
try {
|
|
96
|
+
const { language } = parameters;
|
|
22
97
|
const formData = new FormData();
|
|
23
98
|
formData.append('file', fs.createReadStream(chunk));
|
|
24
99
|
formData.append('model', this.model.params.model);
|
|
25
100
|
formData.append('response_format', 'text');
|
|
26
|
-
|
|
101
|
+
language && formData.append('language', language);
|
|
27
102
|
modelPromptText && formData.append('prompt', modelPromptText);
|
|
28
103
|
|
|
29
104
|
return this.executeRequest(url, formData, params, { ...this.model.headers, ...formData.getHeaders() });
|
|
@@ -34,14 +109,13 @@ class OpenAIWhisperPlugin extends ModelPlugin {
|
|
|
34
109
|
|
|
35
110
|
let result = ``;
|
|
36
111
|
let { file } = parameters;
|
|
37
|
-
let folder;
|
|
38
|
-
const isYoutubeUrl = isValidYoutubeUrl(file);
|
|
39
112
|
let totalCount = 0;
|
|
40
113
|
let completedCount = 0;
|
|
41
114
|
const { requestId } = pathwayResolver;
|
|
42
115
|
|
|
43
116
|
const sendProgress = () => {
|
|
44
117
|
completedCount++;
|
|
118
|
+
if (completedCount >= totalCount) return;
|
|
45
119
|
pubsub.publish('REQUEST_PROGRESS', {
|
|
46
120
|
requestProgress: {
|
|
47
121
|
requestId,
|
|
@@ -51,24 +125,23 @@ class OpenAIWhisperPlugin extends ModelPlugin {
|
|
|
51
125
|
});
|
|
52
126
|
}
|
|
53
127
|
|
|
128
|
+
let chunks = []; // array of local file paths
|
|
54
129
|
try {
|
|
55
|
-
if (isYoutubeUrl) {
|
|
56
|
-
// totalCount += 1; // extra 1 step for youtube download
|
|
57
|
-
file = await processYoutubeUrl(file);
|
|
58
|
-
}
|
|
59
130
|
|
|
60
|
-
const
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
131
|
+
const uris = await this.getMediaChunks(file, requestId); // array of remote file uris
|
|
132
|
+
if (!uris || !uris.length) {
|
|
133
|
+
throw new Error(`Error in getting chunks from media helper for file ${file}`);
|
|
134
|
+
}
|
|
135
|
+
totalCount = uris.length * 4; // 4 steps for each chunk (download and upload)
|
|
136
|
+
API_URL && (completedCount = uris.length); // api progress is already calculated
|
|
64
137
|
|
|
65
138
|
// sequential download of chunks
|
|
66
|
-
const
|
|
67
|
-
|
|
139
|
+
for (const uri of uris) {
|
|
140
|
+
chunks.push(await downloadFile(uri));
|
|
68
141
|
sendProgress();
|
|
69
|
-
chunks.push(await chunkPromise);
|
|
70
142
|
}
|
|
71
143
|
|
|
144
|
+
|
|
72
145
|
// sequential processing of chunks
|
|
73
146
|
for (const chunk of chunks) {
|
|
74
147
|
result += await processChunk(chunk);
|
|
@@ -80,9 +153,29 @@ class OpenAIWhisperPlugin extends ModelPlugin {
|
|
|
80
153
|
|
|
81
154
|
} catch (error) {
|
|
82
155
|
console.error("An error occurred:", error);
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
|
|
156
|
+
}
|
|
157
|
+
finally {
|
|
158
|
+
// isYoutubeUrl && (await deleteTempPath(file));
|
|
159
|
+
// folder && (await deleteTempPath(folder));
|
|
160
|
+
try {
|
|
161
|
+
for (const chunk of chunks) {
|
|
162
|
+
await deleteTempPath(chunk);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
await this.markCompletedForCleanUp(requestId);
|
|
166
|
+
|
|
167
|
+
//check cleanup for whisper temp uploaded files url
|
|
168
|
+
const regex = /whispertempfiles\/([a-z0-9-]+)/;
|
|
169
|
+
const match = file.match(regex);
|
|
170
|
+
if (match && match[1]) {
|
|
171
|
+
const extractedValue = match[1];
|
|
172
|
+
await this.markCompletedForCleanUp(extractedValue);
|
|
173
|
+
console.log(`Cleaned temp whisper file ${file} with request id ${extractedValue}`);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
} catch (error) {
|
|
177
|
+
console.error("An error occurred while deleting:", error);
|
|
178
|
+
}
|
|
86
179
|
}
|
|
87
180
|
return result;
|
|
88
181
|
}
|