@derogab/llm-proxy 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +115 -1
- package/dist/{index.js → cjs/index.js} +76 -8
- package/dist/cjs/index.js.map +1 -0
- package/dist/esm/index.js +197 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/types/index.d.ts.map +1 -0
- package/package.json +25 -6
- package/.github/CODEOWNERS +0 -1
- package/.github/dependabot.yml +0 -27
- package/.github/workflows/release.yml +0 -74
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/src/index.ts +0 -108
- package/tsconfig.json +0 -44
- /package/dist/{index.d.ts → types/index.d.ts} +0 -0
package/README.md
CHANGED
|
@@ -1,2 +1,116 @@
|
|
|
1
1
|
# llm-proxy
|
|
2
|
-
|
|
2
|
+
A simple and lightweight proxy for seamless integration with multiple LLM providers including OpenAI, Ollama, Cloudflare AI, and Llama.cpp.
|
|
3
|
+
|
|
4
|
+
## Features
|
|
5
|
+
|
|
6
|
+
- **Multi-provider support**: Switch between OpenAI, Ollama, Cloudflare AI, and Llama.cpp with environment variables.
|
|
7
|
+
- **TypeScript support**: Full TypeScript definitions included.
|
|
8
|
+
- **Simple API**: Single function interface for all providers.
|
|
9
|
+
- **Automatic provider detection**: Automatically selects the best available provider based on environment variables.
|
|
10
|
+
|
|
11
|
+
## Installation
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
npm install @derogab/llm-proxy
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
## Quick Start
|
|
18
|
+
|
|
19
|
+
```typescript
|
|
20
|
+
import { generate } from '@derogab/llm-proxy';
|
|
21
|
+
|
|
22
|
+
const messages = [
|
|
23
|
+
{ role: 'user', content: 'Hello, how are you?' }
|
|
24
|
+
];
|
|
25
|
+
|
|
26
|
+
const response = await generate(messages);
|
|
27
|
+
console.log(response.content);
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
## Configuration
|
|
31
|
+
|
|
32
|
+
The package automatically detects which LLM provider to use based on your environment variables.
|
|
33
|
+
Configure one or more providers:
|
|
34
|
+
|
|
35
|
+
### OpenAI
|
|
36
|
+
```bash
|
|
37
|
+
OPENAI_API_KEY=your_openai_api_key # Required
|
|
38
|
+
OPENAI_BASE_URL=https://api.openai.com/v1 # Optional
|
|
39
|
+
OPENAI_MODEL=gpt-4o-mini # Optional, defaults to gpt-4o-mini
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
### Cloudflare AI
|
|
43
|
+
```bash
|
|
44
|
+
CLOUDFLARE_ACCOUNT_ID=your_account_id # Required
|
|
45
|
+
CLOUDFLARE_AUTH_KEY=your_auth_key # Required
|
|
46
|
+
CLOUDFLARE_MODEL=your_model_name # Required
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
### Ollama (Local)
|
|
50
|
+
```bash
|
|
51
|
+
OLLAMA_URI=http://localhost:11434 # Optional, defaults to http://localhost:11434
|
|
52
|
+
OLLAMA_MODEL=llama3.1 # Optional, defaults to llama3.1
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
### Llama.cpp (Local)
|
|
56
|
+
```bash
|
|
57
|
+
LLAMA_CPP_MODEL_PATH=/path/to/your/model.gguf # Required, path to your GGUF model file
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## API Reference
|
|
61
|
+
|
|
62
|
+
### `generate(messages: MessageInputParam[]): Promise<MessageInputParam>`
|
|
63
|
+
|
|
64
|
+
Generates a response from the configured LLM provider.
|
|
65
|
+
|
|
66
|
+
**Parameters:**
|
|
67
|
+
- `messages`: Array of message objects with `role` and `content` properties
|
|
68
|
+
|
|
69
|
+
**Returns:**
|
|
70
|
+
- Promise that resolves to a message object with `role` and `content` properties
|
|
71
|
+
|
|
72
|
+
**Message Format:**
|
|
73
|
+
```typescript
|
|
74
|
+
type MessageInputParam = {
|
|
75
|
+
role: 'user' | 'assistant' | 'system';
|
|
76
|
+
content: string;
|
|
77
|
+
};
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
## Provider Priority
|
|
81
|
+
|
|
82
|
+
The package selects providers in the following order:
|
|
83
|
+
1. **OpenAI** (if `OPENAI_API_KEY` is set)
|
|
84
|
+
2. **Cloudflare AI** (if `CLOUDFLARE_ACCOUNT_ID`, `CLOUDFLARE_AUTH_KEY`, and `CLOUDFLARE_MODEL` are set)
|
|
85
|
+
3. **Ollama** (if `OLLAMA_URI` is set)
|
|
86
|
+
4. **Llama.cpp** (if `LLAMA_CPP_MODEL_PATH` is set)
|
|
87
|
+
|
|
88
|
+
If no providers are configured, the function throws an error.
|
|
89
|
+
|
|
90
|
+
## Development
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
# Install dependencies
|
|
94
|
+
npm install
|
|
95
|
+
|
|
96
|
+
# Build the package
|
|
97
|
+
npm run build
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
## Credits
|
|
101
|
+
_LLM Proxy_ is made with ♥ by [derogab](https://github.com/derogab) and it's released under the [MIT license](./LICENSE).
|
|
102
|
+
|
|
103
|
+
## Contributors
|
|
104
|
+
|
|
105
|
+
<a href="https://github.com/derogab/llm-proxy/graphs/contributors">
|
|
106
|
+
<img src="https://contrib.rocks/image?repo=derogab/llm-proxy" />
|
|
107
|
+
</a>
|
|
108
|
+
|
|
109
|
+
## Tip
|
|
110
|
+
If you like this project or directly benefit from it, please consider buying me a coffee:
|
|
111
|
+
🔗 `bc1qd0qatgz8h62uvnr74utwncc6j5ckfz2v2g4lef`
|
|
112
|
+
⚡️ `derogab@sats.mobi`
|
|
113
|
+
💶 [Sponsor on GitHub](https://github.com/sponsors/derogab)
|
|
114
|
+
|
|
115
|
+
## Stargazers over time
|
|
116
|
+
[](https://starchart.cc/derogab/llm-proxy)
|
|
@@ -1,8 +1,12 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.generate = generate;
|
|
1
4
|
// Dependencies.
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
5
|
+
const axios_1 = require("axios");
|
|
6
|
+
const dotenv = require("dotenv");
|
|
7
|
+
const node_llama_cpp_1 = require("node-llama-cpp");
|
|
8
|
+
const ollama_1 = require("ollama");
|
|
9
|
+
const openai_1 = require("openai");
|
|
6
10
|
// Configs.
|
|
7
11
|
dotenv.config();
|
|
8
12
|
/**
|
|
@@ -13,7 +17,7 @@ dotenv.config();
|
|
|
13
17
|
*/
|
|
14
18
|
async function generate_openai(messages) {
|
|
15
19
|
// Create a new instance of the OpenAI class.
|
|
16
|
-
const openai = new
|
|
20
|
+
const openai = new openai_1.default({ apiKey: process.env.OPENAI_API_KEY, baseURL: process.env.OPENAI_BASE_URL });
|
|
17
21
|
// Call the OpenAI API.
|
|
18
22
|
const chatCompletion = await openai.chat.completions.create({
|
|
19
23
|
messages: messages,
|
|
@@ -30,7 +34,7 @@ async function generate_openai(messages) {
|
|
|
30
34
|
*/
|
|
31
35
|
async function generate_ollama(messages) {
|
|
32
36
|
// Create a new instance of the OpenAI class.
|
|
33
|
-
const ollama = new Ollama({ host: process.env.OLLAMA_URI || 'http://localhost:11434' });
|
|
37
|
+
const ollama = new ollama_1.Ollama({ host: process.env.OLLAMA_URI || 'http://localhost:11434' });
|
|
34
38
|
// Call the Ollama API.
|
|
35
39
|
const response = await ollama.chat({
|
|
36
40
|
model: process.env.OLLAMA_MODEL || 'llama3.1',
|
|
@@ -39,6 +43,66 @@ async function generate_ollama(messages) {
|
|
|
39
43
|
// Return the response.
|
|
40
44
|
return response['message'];
|
|
41
45
|
}
|
|
46
|
+
/**
|
|
47
|
+
* Convert messages to chat history.
|
|
48
|
+
*
|
|
49
|
+
* Llama.cpp expects the chat history in a custom format.
|
|
50
|
+
* Convert the default messages format to the Llama.cpp format.
|
|
51
|
+
*
|
|
52
|
+
* @param messages the messages to be sent to Llama.cpp.
|
|
53
|
+
* @returns the same messages in the Llama.cpp format.
|
|
54
|
+
*/
|
|
55
|
+
function convert_messages_to_chat_history(messages) {
|
|
56
|
+
// Init chat history.
|
|
57
|
+
const chat_history = [];
|
|
58
|
+
// Loop through messages.
|
|
59
|
+
for (const message of messages) {
|
|
60
|
+
if (message.role === 'system' || message.role === 'user') {
|
|
61
|
+
chat_history.push({
|
|
62
|
+
type: message.role,
|
|
63
|
+
text: message.content
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
else if (message.role === 'assistant') {
|
|
67
|
+
chat_history.push({
|
|
68
|
+
type: "model",
|
|
69
|
+
response: [message.content]
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
// Return the chat history.
|
|
74
|
+
return chat_history;
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Generate a response using Llama.cpp Local Model.
|
|
78
|
+
*
|
|
79
|
+
* @param messages the messages to be sent to Llama.cpp.
|
|
80
|
+
* @returns the response string.
|
|
81
|
+
*/
|
|
82
|
+
async function generate_llama_cpp(messages) {
|
|
83
|
+
// Create a new instance of the Llama.cpp class.
|
|
84
|
+
const llama = await (0, node_llama_cpp_1.getLlama)();
|
|
85
|
+
// Set model to use.
|
|
86
|
+
const modelPath = process.env.LLAMA_CPP_MODEL_PATH;
|
|
87
|
+
if (!modelPath) {
|
|
88
|
+
throw new Error('LLAMA_CPP_MODEL_PATH is not set.');
|
|
89
|
+
}
|
|
90
|
+
const model = await llama.loadModel({
|
|
91
|
+
modelPath: modelPath,
|
|
92
|
+
});
|
|
93
|
+
// Import history into the context.
|
|
94
|
+
const context = await model.createContext();
|
|
95
|
+
const session = new node_llama_cpp_1.LlamaChatSession({
|
|
96
|
+
contextSequence: context.getSequence()
|
|
97
|
+
});
|
|
98
|
+
if (messages.length > 1)
|
|
99
|
+
session.setChatHistory(convert_messages_to_chat_history(messages.slice(0, -1)));
|
|
100
|
+
// Generate and return the response.
|
|
101
|
+
return {
|
|
102
|
+
role: 'assistant',
|
|
103
|
+
content: await session.prompt(messages[messages.length - 1]?.content || ''),
|
|
104
|
+
};
|
|
105
|
+
}
|
|
42
106
|
/**
|
|
43
107
|
* Generate a response using Cloudflare AI API.
|
|
44
108
|
*
|
|
@@ -49,7 +113,7 @@ async function generate_cloudflare(messages) {
|
|
|
49
113
|
// Generate API URL based on the environment variables.
|
|
50
114
|
const model_url = 'https://api.cloudflare.com/client/v4/accounts/' + process.env.CLOUDFLARE_ACCOUNT_ID + '/ai/run/' + process.env.CLOUDFLARE_MODEL;
|
|
51
115
|
// Call the Cloudflare AI API.
|
|
52
|
-
const response = await
|
|
116
|
+
const response = await (0, axios_1.default)({
|
|
53
117
|
method: 'post',
|
|
54
118
|
url: model_url,
|
|
55
119
|
headers: {
|
|
@@ -71,7 +135,7 @@ async function generate_cloudflare(messages) {
|
|
|
71
135
|
* @param messages the messages to be sent to the LLM.
|
|
72
136
|
* @returns the response string.
|
|
73
137
|
*/
|
|
74
|
-
|
|
138
|
+
async function generate(messages) {
|
|
75
139
|
// Check what LLM to use, based on the environment variables.
|
|
76
140
|
if (process.env.OPENAI_API_KEY) {
|
|
77
141
|
// If openai key is available, use openai.
|
|
@@ -85,6 +149,10 @@ export async function generate(messages) {
|
|
|
85
149
|
// If ollama is available, use ollama.
|
|
86
150
|
return await generate_ollama(messages);
|
|
87
151
|
}
|
|
152
|
+
else if (process.env.LLAMA_CPP_MODEL_PATH) {
|
|
153
|
+
// If llama_cpp is available, use llama_cpp.
|
|
154
|
+
return await generate_llama_cpp(messages);
|
|
155
|
+
}
|
|
88
156
|
else {
|
|
89
157
|
// Throw an error if no LLM is available.
|
|
90
158
|
throw new Error('No available LLM found.');
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":";;AAwJA,4BAsBC;AA9KD,gBAAgB;AAChB,iCAA0B;AAC1B,iCAAiC;AACjC,mDAA4D;AAC5D,mCAAgC;AAChC,mCAA4B;AAc5B,WAAW;AACX,MAAM,CAAC,MAAM,EAAE,CAAC;AAEhB;;;;;GAKG;AACH,KAAK,UAAU,eAAe,CAAC,QAAsC;IACnE,6CAA6C;IAC7C,MAAM,MAAM,GAAG,IAAI,gBAAM,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC,CAAC;IACxG,uBAAuB;IACvB,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;QAC1D,QAAQ,EAAE,QAAQ;QAClB,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,aAAa;KACjD,CAAC,CAAC;IACH,uBAAuB;IACvB,OAAO,cAAc,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,OAAqC,CAAC;AAC3E,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,eAAe,CAAC,QAAmB;IAChD,6CAA6C;IAC7C,MAAM,MAAM,GAAG,IAAI,eAAM,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,wBAAwB,EAAE,CAAC,CAAC;IACxF,uBAAuB;IACvB,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC;QACjC,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,UAAU;QAC7C,QAAQ,EAAE,QAAQ;KACnB,CAAC,CAAC;IACH,uBAAuB;IACvB,OAAO,QAAQ,CAAC,SAAS,CAAC,CAAC;AAC7B,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,gCAAgC,CAAC,QAAmB;IAC3D,qBAAqB;IACrB,MAAM,YAAY,GAAsB,EAAE,CAAC;IAC3C,yBAAyB;IACzB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,OAAO,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACzD,YAAY,CAAC,IAAI,CAAC;gBAChB,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,IAAI,EAAE,OAAO,CAAC,OAAO;aACtB,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YACxC,YAAY,CAAC,IAAI,CAAC;gBAChB,IAAI,EAAE,OAAO;gBACb,QAAQ,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC;aAC5B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,2BAA2B;IAC3B,OAAO,YAAY,CAAC;AACtB,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,kBAAkB,CAAC,QAAmB;IACnD,gDAAgD;IAChD,MAAM,KAAK,GAAG,MAAM,IAAA,yBAAQ,GAAE,CAAC;IAC/B,oBAAoB;IACpB,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC;IACnD,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACtD,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC;QAClC,SAAS,EAAE,SAAS;KACrB,CAAC,CAAC;IACH,mCAAmC;IACnC,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,aAAa,EAAE,CAAC;IAC5C,MAAM,OAAO,GAAG,IAAI,iCAAgB,CAAC;QACnC,eAAe,EAAE,OAAO,CAAC,WAAW,EAAE;KACvC,CAAC,CAAC;IACH,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,cAAc,CAAC,gCAAgC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEzG,oCAAoC;IACpC,OAAO;QACL,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,OAAO,IAAI,EAAE,CAAC;KAC5E,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,mBAAmB,CAAC,QAA6B;IAC9D,uDAAuD;IACvD,MAAM,SAAS,GAAG,gDAAgD,GAAG,OAAO,CAAC,GAAG,CAAC,qBAAqB,GAAG,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;IACnJ,8BAA8B;IAC9B,MAAM,QAAQ,GAAG,MAAM,IAAA,eAAK,EAAC;QAC3B,MAAM,EAAE,MAAM;QACd,GAAG,EAAE,SAAS;QACd,OAAO,EAAE;YACP,eAAe,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB;YAC5D,cAAc,EAAG,kBAAkB;SACpC;QACD,IAAI,EAAE;YACJ,QAAQ,EAAE,QAAQ;SACnB;KACF,CAAC,CAAC;IACH,gCAAgC;IAChC,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;IACvE,uBAAuB;IACvB,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;AAC7C,CAAC;AAED;;;;;GAKG;AACI,KAAK,UAAU,QAAQ,CAAC,QAA6B;IAC1D,6DAA6D;IAC7D,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC;QAC/B,0CAA0C;QAC1C,OAAO,MAAM,eAAe,CAAC,QAAwC,CAAC,CAAC;IAEzE,CAAC;SAAM,IAAI,OAAO,CAAC,GAAG,CAAC,qBAAqB,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QAChH,oDAAoD;QACpD,OAAO,MAAM,mBAAmB,CAAC,QAA+B,CAAC,CAAC;IAEpE,CAAC;SAAM,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC;QAClC,sCAAsC;QACtC,OAAO,MAAM,eAAe,CAAC,QAAqB,CAAC,CAAC;IAEtD,CAAC;SAAM,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,CAAC;QAC5C,4CAA4C;QAC5C,OAAO,MAAM,kBAAkB,CAAC,QAAqB,CAAC,CAAC;IAEzD,CAAC;SAAM,CAAC;QACN,yCAAyC;QACzC,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAC7C,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
36
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
|
+
};
|
|
38
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
+
exports.generate = generate;
|
|
40
|
+
// Dependencies.
|
|
41
|
+
const axios_1 = __importDefault(require("axios"));
|
|
42
|
+
const dotenv = __importStar(require("dotenv"));
|
|
43
|
+
const node_llama_cpp_1 = require("node-llama-cpp");
|
|
44
|
+
const ollama_1 = require("ollama");
|
|
45
|
+
const openai_1 = __importDefault(require("openai"));
|
|
46
|
+
// Configs.
|
|
47
|
+
dotenv.config();
|
|
48
|
+
/**
|
|
49
|
+
* Generate a response from the OpenAI API.
|
|
50
|
+
*
|
|
51
|
+
* @param messages the messages to be sent to the OpenAI API.
|
|
52
|
+
* @returns the response string from the OpenAI API.
|
|
53
|
+
*/
|
|
54
|
+
async function generate_openai(messages) {
|
|
55
|
+
// Create a new instance of the OpenAI class.
|
|
56
|
+
const openai = new openai_1.default({ apiKey: process.env.OPENAI_API_KEY, baseURL: process.env.OPENAI_BASE_URL });
|
|
57
|
+
// Call the OpenAI API.
|
|
58
|
+
const chatCompletion = await openai.chat.completions.create({
|
|
59
|
+
messages: messages,
|
|
60
|
+
model: process.env.OPENAI_MODEL || 'gpt-4o-mini',
|
|
61
|
+
});
|
|
62
|
+
// Return the response.
|
|
63
|
+
return chatCompletion?.choices[0]?.message;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Generate a response using Ollama Local API.
|
|
67
|
+
*
|
|
68
|
+
* @param messages the messages to be sent to Ollama.
|
|
69
|
+
* @returns the response string.
|
|
70
|
+
*/
|
|
71
|
+
async function generate_ollama(messages) {
|
|
72
|
+
// Create a new instance of the OpenAI class.
|
|
73
|
+
const ollama = new ollama_1.Ollama({ host: process.env.OLLAMA_URI || 'http://localhost:11434' });
|
|
74
|
+
// Call the Ollama API.
|
|
75
|
+
const response = await ollama.chat({
|
|
76
|
+
model: process.env.OLLAMA_MODEL || 'llama3.1',
|
|
77
|
+
messages: messages,
|
|
78
|
+
});
|
|
79
|
+
// Return the response.
|
|
80
|
+
return response['message'];
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Convert messages to chat history.
|
|
84
|
+
*
|
|
85
|
+
* Llama.cpp expects the chat history in a custom format.
|
|
86
|
+
* Convert the default messages format to the Llama.cpp format.
|
|
87
|
+
*
|
|
88
|
+
* @param messages the messages to be sent to Llama.cpp.
|
|
89
|
+
* @returns the same messages in the Llama.cpp format.
|
|
90
|
+
*/
|
|
91
|
+
function convert_messages_to_chat_history(messages) {
|
|
92
|
+
// Init chat history.
|
|
93
|
+
const chat_history = [];
|
|
94
|
+
// Loop through messages.
|
|
95
|
+
for (const message of messages) {
|
|
96
|
+
if (message.role === 'system' || message.role === 'user') {
|
|
97
|
+
chat_history.push({
|
|
98
|
+
type: message.role,
|
|
99
|
+
text: message.content
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
else if (message.role === 'assistant') {
|
|
103
|
+
chat_history.push({
|
|
104
|
+
type: "model",
|
|
105
|
+
response: [message.content]
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
// Return the chat history.
|
|
110
|
+
return chat_history;
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Generate a response using Llama.cpp Local Model.
|
|
114
|
+
*
|
|
115
|
+
* @param messages the messages to be sent to Llama.cpp.
|
|
116
|
+
* @returns the response string.
|
|
117
|
+
*/
|
|
118
|
+
async function generate_llama_cpp(messages) {
|
|
119
|
+
// Create a new instance of the Llama.cpp class.
|
|
120
|
+
const llama = await (0, node_llama_cpp_1.getLlama)();
|
|
121
|
+
// Set model to use.
|
|
122
|
+
const modelPath = process.env.LLAMA_CPP_MODEL_PATH;
|
|
123
|
+
if (!modelPath) {
|
|
124
|
+
throw new Error('LLAMA_CPP_MODEL_PATH is not set.');
|
|
125
|
+
}
|
|
126
|
+
const model = await llama.loadModel({
|
|
127
|
+
modelPath: modelPath,
|
|
128
|
+
});
|
|
129
|
+
// Import history into the context.
|
|
130
|
+
const context = await model.createContext();
|
|
131
|
+
const session = new node_llama_cpp_1.LlamaChatSession({
|
|
132
|
+
contextSequence: context.getSequence()
|
|
133
|
+
});
|
|
134
|
+
if (messages.length > 1)
|
|
135
|
+
session.setChatHistory(convert_messages_to_chat_history(messages.slice(0, -1)));
|
|
136
|
+
// Generate and return the response.
|
|
137
|
+
return {
|
|
138
|
+
role: 'assistant',
|
|
139
|
+
content: await session.prompt(messages[messages.length - 1]?.content || ''),
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Generate a response using Cloudflare AI API.
|
|
144
|
+
*
|
|
145
|
+
* @param messages the messages to be sent to Cloudflare AI.
|
|
146
|
+
* @returns the response string.
|
|
147
|
+
*/
|
|
148
|
+
async function generate_cloudflare(messages) {
|
|
149
|
+
// Generate API URL based on the environment variables.
|
|
150
|
+
const model_url = 'https://api.cloudflare.com/client/v4/accounts/' + process.env.CLOUDFLARE_ACCOUNT_ID + '/ai/run/' + process.env.CLOUDFLARE_MODEL;
|
|
151
|
+
// Call the Cloudflare AI API.
|
|
152
|
+
const response = await (0, axios_1.default)({
|
|
153
|
+
method: 'post',
|
|
154
|
+
url: model_url,
|
|
155
|
+
headers: {
|
|
156
|
+
'Authorization': 'Bearer ' + process.env.CLOUDFLARE_AUTH_KEY,
|
|
157
|
+
'Content-Type': 'application/json',
|
|
158
|
+
},
|
|
159
|
+
data: {
|
|
160
|
+
messages: messages,
|
|
161
|
+
},
|
|
162
|
+
});
|
|
163
|
+
// Extract the response message.
|
|
164
|
+
const msg = response.data.success ? response.data.result.response : '';
|
|
165
|
+
// Return the response.
|
|
166
|
+
return { role: 'assistant', content: msg };
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* Generate a response using an LLM.
|
|
170
|
+
*
|
|
171
|
+
* @param messages the messages to be sent to the LLM.
|
|
172
|
+
* @returns the response string.
|
|
173
|
+
*/
|
|
174
|
+
async function generate(messages) {
|
|
175
|
+
// Check what LLM to use, based on the environment variables.
|
|
176
|
+
if (process.env.OPENAI_API_KEY) {
|
|
177
|
+
// If openai key is available, use openai.
|
|
178
|
+
return await generate_openai(messages);
|
|
179
|
+
}
|
|
180
|
+
else if (process.env.CLOUDFLARE_ACCOUNT_ID && process.env.CLOUDFLARE_AUTH_KEY && process.env.CLOUDFLARE_MODEL) {
|
|
181
|
+
// If cloudflare keys are available, use cloudflare.
|
|
182
|
+
return await generate_cloudflare(messages);
|
|
183
|
+
}
|
|
184
|
+
else if (process.env.OLLAMA_URI) {
|
|
185
|
+
// If ollama is available, use ollama.
|
|
186
|
+
return await generate_ollama(messages);
|
|
187
|
+
}
|
|
188
|
+
else if (process.env.LLAMA_CPP_MODEL_PATH) {
|
|
189
|
+
// If llama_cpp is available, use llama_cpp.
|
|
190
|
+
return await generate_llama_cpp(messages);
|
|
191
|
+
}
|
|
192
|
+
else {
|
|
193
|
+
// Throw an error if no LLM is available.
|
|
194
|
+
throw new Error('No available LLM found.');
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwJA,4BAsBC;AA9KD,gBAAgB;AAChB,kDAA0B;AAC1B,+CAAiC;AACjC,mDAA4D;AAC5D,mCAAgC;AAChC,oDAA4B;AAc5B,WAAW;AACX,MAAM,CAAC,MAAM,EAAE,CAAC;AAEhB;;;;;GAKG;AACH,KAAK,UAAU,eAAe,CAAC,QAAsC;IACnE,6CAA6C;IAC7C,MAAM,MAAM,GAAG,IAAI,gBAAM,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC,CAAC;IACxG,uBAAuB;IACvB,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;QAC1D,QAAQ,EAAE,QAAQ;QAClB,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,aAAa;KACjD,CAAC,CAAC;IACH,uBAAuB;IACvB,OAAO,cAAc,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,OAAqC,CAAC;AAC3E,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,eAAe,CAAC,QAAmB;IAChD,6CAA6C;IAC7C,MAAM,MAAM,GAAG,IAAI,eAAM,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,wBAAwB,EAAE,CAAC,CAAC;IACxF,uBAAuB;IACvB,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC;QACjC,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,UAAU;QAC7C,QAAQ,EAAE,QAAQ;KACnB,CAAC,CAAC;IACH,uBAAuB;IACvB,OAAO,QAAQ,CAAC,SAAS,CAAC,CAAC;AAC7B,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,gCAAgC,CAAC,QAAmB;IAC3D,qBAAqB;IACrB,MAAM,YAAY,GAAsB,EAAE,CAAC;IAC3C,yBAAyB;IACzB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,OAAO,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACzD,YAAY,CAAC,IAAI,CAAC;gBAChB,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,IAAI,EAAE,OAAO,CAAC,OAAO;aACtB,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YACxC,YAAY,CAAC,IAAI,CAAC;gBAChB,IAAI,EAAE,OAAO;gBACb,QAAQ,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC;aAC5B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,2BAA2B;IAC3B,OAAO,YAAY,CAAC;AACtB,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,kBAAkB,CAAC,QAAmB;IACnD,gDAAgD;IAChD,MAAM,KAAK,GAAG,MAAM,IAAA,yBAAQ,GAAE,CAAC;IAC/B,oBAAoB;IACpB,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC;IACnD,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACtD,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC;QAClC,SAAS,EAAE,SAAS;KACrB,CAAC,CAAC;IACH,mCAAmC;IACnC,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,aAAa,EAAE,CAAC;IAC5C,MAAM,OAAO,GAAG,IAAI,iCAAgB,CAAC;QACnC,eAAe,EAAE,OAAO,CAAC,WAAW,EAAE;KACvC,CAAC,CAAC;IACH,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,cAAc,CAAC,gCAAgC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEzG,oCAAoC;IACpC,OAAO;QACL,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,OAAO,IAAI,EAAE,CAAC;KAC5E,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,mBAAmB,CAAC,QAA6B;IAC9D,uDAAuD;IACvD,MAAM,SAAS,GAAG,gDAAgD,GAAG,OAAO,CAAC,GAAG,CAAC,qBAAqB,GAAG,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;IACnJ,8BAA8B;IAC9B,MAAM,QAAQ,GAAG,MAAM,IAAA,eAAK,EAAC;QAC3B,MAAM,EAAE,MAAM;QACd,GAAG,EAAE,SAAS;QACd,OAAO,EAAE;YACP,eAAe,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB;YAC5D,cAAc,EAAG,kBAAkB;SACpC;QACD,IAAI,EAAE;YACJ,QAAQ,EAAE,QAAQ;SACnB;KACF,CAAC,CAAC;IACH,gCAAgC;IAChC,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;IACvE,uBAAuB;IACvB,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;AAC7C,CAAC;AAED;;;;;GAKG;AACI,KAAK,UAAU,QAAQ,CAAC,QAA6B;IAC1D,6DAA6D;IAC7D,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC;QAC/B,0CAA0C;QAC1C,OAAO,MAAM,eAAe,CAAC,QAAwC,CAAC,CAAC;IAEzE,CAAC;SAAM,IAAI,OAAO,CAAC,GAAG,CAAC,qBAAqB,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QAChH,oDAAoD;QACpD,OAAO,MAAM,mBAAmB,CAAC,QAA+B,CAAC,CAAC;IAEpE,CAAC;SAAM,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC;QAClC,sCAAsC;QACtC,OAAO,MAAM,eAAe,CAAC,QAAqB,CAAC,CAAC;IAEtD,CAAC;SAAM,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,CAAC;QAC5C,4CAA4C;QAC5C,OAAO,MAAM,kBAAkB,CAAC,QAAqB,CAAC,CAAC;IAEzD,CAAC;SAAM,CAAC;QACN,yCAAyC;QACzC,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAC7C,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,kBAAkB,CAAC;AAEnE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAC;AAEtC,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG,0BAA0B,GAAG,OAAO,GAAG,iBAAiB,CAAC;AAiIzF;;;;;GAKG;AACH,wBAAsB,QAAQ,CAAC,QAAQ,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAsBxF"}
|
package/package.json
CHANGED
|
@@ -1,19 +1,37 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@derogab/llm-proxy",
|
|
3
|
-
"description": "
|
|
4
|
-
"version": "0.
|
|
3
|
+
"description": "A simple and lightweight proxy for seamless integration with multiple LLM providers including OpenAI, Ollama, and Cloudflare AI",
|
|
4
|
+
"version": "0.3.1",
|
|
5
5
|
"author": "derogab",
|
|
6
6
|
"license": "MIT",
|
|
7
7
|
"repository": {
|
|
8
8
|
"type": "git",
|
|
9
9
|
"url": "https://github.com/derogab/llm-proxy.git"
|
|
10
10
|
},
|
|
11
|
-
"main": "dist/index.js",
|
|
12
|
-
"
|
|
13
|
-
"
|
|
11
|
+
"main": "./dist/cjs/index.js",
|
|
12
|
+
"module": "./dist/esm/index.js",
|
|
13
|
+
"types": "./dist/types/index.d.ts",
|
|
14
|
+
"exports": {
|
|
15
|
+
".": {
|
|
16
|
+
"import": {
|
|
17
|
+
"types": "./dist/types/index.d.ts",
|
|
18
|
+
"default": "./dist/esm/index.js"
|
|
19
|
+
},
|
|
20
|
+
"require": {
|
|
21
|
+
"types": "./dist/types/index.d.ts",
|
|
22
|
+
"default": "./dist/cjs/index.js"
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
},
|
|
14
26
|
"scripts": {
|
|
15
|
-
"build": "
|
|
27
|
+
"build": "npm run build:cjs && npm run build:esm && npm run build:types",
|
|
28
|
+
"build:cjs": "tsc -p tsconfig.cjs.json",
|
|
29
|
+
"build:esm": "tsc -p tsconfig.esm.json",
|
|
30
|
+
"build:types": "tsc -p tsconfig.types.json"
|
|
16
31
|
},
|
|
32
|
+
"files": [
|
|
33
|
+
"dist"
|
|
34
|
+
],
|
|
17
35
|
"keywords": [
|
|
18
36
|
"LLM",
|
|
19
37
|
"proxy",
|
|
@@ -26,6 +44,7 @@
|
|
|
26
44
|
"dependencies": {
|
|
27
45
|
"axios": "1.11.0",
|
|
28
46
|
"dotenv": "17.2.1",
|
|
47
|
+
"node-llama-cpp": "3.13.0",
|
|
29
48
|
"ollama": "0.5.17",
|
|
30
49
|
"openai": "5.16.0"
|
|
31
50
|
}
|
package/.github/CODEOWNERS
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
* @derogab
|
package/.github/dependabot.yml
DELETED
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
# Dependabot Configuration:
|
|
2
|
-
# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
|
|
3
|
-
|
|
4
|
-
version: 2
|
|
5
|
-
updates:
|
|
6
|
-
|
|
7
|
-
# Maintain dependencies for GitHub Actions
|
|
8
|
-
- package-ecosystem: 'github-actions'
|
|
9
|
-
directory: '/'
|
|
10
|
-
target-branch: 'master'
|
|
11
|
-
schedule:
|
|
12
|
-
interval: 'monthly'
|
|
13
|
-
assignees:
|
|
14
|
-
- 'derogab'
|
|
15
|
-
labels:
|
|
16
|
-
- 'dependencies'
|
|
17
|
-
|
|
18
|
-
# Maintain dependencies for NPM
|
|
19
|
-
- package-ecosystem: 'npm'
|
|
20
|
-
directory: '/'
|
|
21
|
-
target-branch: 'master'
|
|
22
|
-
schedule:
|
|
23
|
-
interval: 'monthly'
|
|
24
|
-
assignees:
|
|
25
|
-
- 'derogab'
|
|
26
|
-
labels:
|
|
27
|
-
- 'dependencies'
|
|
@@ -1,74 +0,0 @@
|
|
|
1
|
-
name: Release and publish package to NPM
|
|
2
|
-
|
|
3
|
-
on:
|
|
4
|
-
push:
|
|
5
|
-
# Publish `v1.2.3` tags as releases.
|
|
6
|
-
tags:
|
|
7
|
-
- v*
|
|
8
|
-
|
|
9
|
-
jobs:
|
|
10
|
-
# Release the TAG to GitHub.
|
|
11
|
-
release:
|
|
12
|
-
name: Release pushed tag
|
|
13
|
-
if: startsWith(github.ref, 'refs/tags/')
|
|
14
|
-
permissions:
|
|
15
|
-
contents: write
|
|
16
|
-
runs-on: ubuntu-latest
|
|
17
|
-
steps:
|
|
18
|
-
- name: Create release
|
|
19
|
-
env:
|
|
20
|
-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
21
|
-
tag: ${{ github.ref_name }}
|
|
22
|
-
run: |
|
|
23
|
-
gh release create "$tag" \
|
|
24
|
-
--repo="$GITHUB_REPOSITORY" \
|
|
25
|
-
--title="v${tag#v}" \
|
|
26
|
-
--generate-notes
|
|
27
|
-
# Publish the package to NPM.
|
|
28
|
-
publish:
|
|
29
|
-
name: Publish Package
|
|
30
|
-
needs: release
|
|
31
|
-
runs-on: ubuntu-latest
|
|
32
|
-
permissions:
|
|
33
|
-
contents: read
|
|
34
|
-
id-token: write
|
|
35
|
-
steps:
|
|
36
|
-
- name: Checkout
|
|
37
|
-
uses: actions/checkout@v5
|
|
38
|
-
- name: Setup Node
|
|
39
|
-
uses: actions/setup-node@v4
|
|
40
|
-
with:
|
|
41
|
-
node-version: '20.x'
|
|
42
|
-
cache: 'npm'
|
|
43
|
-
registry-url: 'https://registry.npmjs.org'
|
|
44
|
-
always-auth: true
|
|
45
|
-
- name: Install dependencies (clean)
|
|
46
|
-
run: npm ci
|
|
47
|
-
- name: Type check
|
|
48
|
-
run: npx tsc -p tsconfig.json --noEmit
|
|
49
|
-
- name: Run tests
|
|
50
|
-
run: npm test --if-present
|
|
51
|
-
- name: Build
|
|
52
|
-
run: |
|
|
53
|
-
if npm run | grep -q "build"; then
|
|
54
|
-
npm run build
|
|
55
|
-
else
|
|
56
|
-
# Fall back to a standard TS build if no script is defined
|
|
57
|
-
npx tsc -p tsconfig.json
|
|
58
|
-
fi
|
|
59
|
-
- name: Verify tag matches package.json version
|
|
60
|
-
run: |
|
|
61
|
-
PKG_VERSION="$(node -p "require('./package.json').version")"
|
|
62
|
-
TAG_VERSION="${GITHUB_REF_NAME#v}" # supports tags like v1.2.3
|
|
63
|
-
echo "package.json: $PKG_VERSION"
|
|
64
|
-
echo "release tag: $TAG_VERSION"
|
|
65
|
-
if [ "$PKG_VERSION" != "$TAG_VERSION" ]; then
|
|
66
|
-
echo "Release tag ($TAG_VERSION) does not match package.json version ($PKG_VERSION)."
|
|
67
|
-
exit 1
|
|
68
|
-
fi
|
|
69
|
-
- name: Show publish contents (dry run)
|
|
70
|
-
run: npm pack --dry-run
|
|
71
|
-
- name: Publish to npm (with provenance)
|
|
72
|
-
env:
|
|
73
|
-
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
|
74
|
-
run: npm publish --provenance --access public
|
package/dist/index.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,kBAAkB,CAAC;AACnE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAC;AAEtC,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG,0BAA0B,GAAG,OAAO,GAAG,iBAAiB,CAAC;AAoEzF;;;;;GAKG;AACH,wBAAsB,QAAQ,CAAC,QAAQ,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAkBxF"}
|
package/dist/index.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,gBAAgB;AAChB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAa5B,WAAW;AACX,MAAM,CAAC,MAAM,EAAE,CAAC;AAEhB;;;;;GAKG;AACH,KAAK,UAAU,eAAe,CAAC,QAAsC;IACnE,6CAA6C;IAC7C,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC,CAAC;IACxG,uBAAuB;IACvB,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;QAC1D,QAAQ,EAAE,QAAQ;QAClB,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,aAAa;KACjD,CAAC,CAAC;IACH,uBAAuB;IACvB,OAAO,cAAc,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,OAAqC,CAAC;AAC3E,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,eAAe,CAAC,QAAmB;IAChD,6CAA6C;IAC7C,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,wBAAwB,EAAE,CAAC,CAAC;IACxF,uBAAuB;IACvB,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC;QACjC,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,UAAU;QAC7C,QAAQ,EAAE,QAAQ;KACnB,CAAC,CAAC;IACH,uBAAuB;IACvB,OAAO,QAAQ,CAAC,SAAS,CAAC,CAAC;AAC7B,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,mBAAmB,CAAC,QAA6B;IAC9D,uDAAuD;IACvD,MAAM,SAAS,GAAG,gDAAgD,GAAG,OAAO,CAAC,GAAG,CAAC,qBAAqB,GAAG,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;IACnJ,8BAA8B;IAC9B,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC;QAC3B,MAAM,EAAE,MAAM;QACd,GAAG,EAAE,SAAS;QACd,OAAO,EAAE;YACP,eAAe,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB;YAC5D,cAAc,EAAG,kBAAkB;SACpC;QACD,IAAI,EAAE;YACJ,QAAQ,EAAE,QAAQ;SACnB;KACF,CAAC,CAAC;IACH,gCAAgC;IAChC,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;IACvE,uBAAuB;IACvB,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;AAC7C,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,QAA6B;IAC1D,6DAA6D;IAC7D,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC;QAC/B,0CAA0C;QAC1C,OAAO,MAAM,eAAe,CAAC,QAAwC,CAAC,CAAC;IAEzE,CAAC;SAAM,IAAI,OAAO,CAAC,GAAG,CAAC,qBAAqB,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QAChH,oDAAoD;QACpD,OAAO,MAAM,mBAAmB,CAAC,QAA+B,CAAC,CAAC;IAEpE,CAAC;SAAM,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC;QAClC,sCAAsC;QACtC,OAAO,MAAM,eAAe,CAAC,QAAqB,CAAC,CAAC;IAEtD,CAAC;SAAM,CAAC;QACN,yCAAyC;QACzC,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAC7C,CAAC;AACH,CAAC"}
|
package/src/index.ts
DELETED
|
@@ -1,108 +0,0 @@
|
|
|
1
|
-
// Dependencies.
|
|
2
|
-
import axios from 'axios';
|
|
3
|
-
import * as dotenv from 'dotenv';
|
|
4
|
-
import { Ollama } from 'ollama';
|
|
5
|
-
import OpenAI from 'openai';
|
|
6
|
-
|
|
7
|
-
// Types.
|
|
8
|
-
import type { ChatCompletionMessageParam } from 'openai/resources';
|
|
9
|
-
import type { Message } from 'ollama';
|
|
10
|
-
|
|
11
|
-
export type CloudflareMessage = {
|
|
12
|
-
role: string;
|
|
13
|
-
content: string;
|
|
14
|
-
};
|
|
15
|
-
|
|
16
|
-
export type MessageInputParam = ChatCompletionMessageParam | Message | CloudflareMessage;
|
|
17
|
-
|
|
18
|
-
// Configs.
|
|
19
|
-
dotenv.config();
|
|
20
|
-
|
|
21
|
-
/**
|
|
22
|
-
* Generate a response from the OpenAI API.
|
|
23
|
-
*
|
|
24
|
-
* @param messages the messages to be sent to the OpenAI API.
|
|
25
|
-
* @returns the response string from the OpenAI API.
|
|
26
|
-
*/
|
|
27
|
-
async function generate_openai(messages: ChatCompletionMessageParam[]): Promise<ChatCompletionMessageParam> {
|
|
28
|
-
// Create a new instance of the OpenAI class.
|
|
29
|
-
const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY, baseURL: process.env.OPENAI_BASE_URL });
|
|
30
|
-
// Call the OpenAI API.
|
|
31
|
-
const chatCompletion = await openai.chat.completions.create({
|
|
32
|
-
messages: messages,
|
|
33
|
-
model: process.env.OPENAI_MODEL || 'gpt-4o-mini',
|
|
34
|
-
});
|
|
35
|
-
// Return the response.
|
|
36
|
-
return chatCompletion?.choices[0]?.message as ChatCompletionMessageParam;
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
/**
|
|
40
|
-
* Generate a response using Ollama Local API.
|
|
41
|
-
*
|
|
42
|
-
* @param messages the messages to be sent to Ollama.
|
|
43
|
-
* @returns the response string.
|
|
44
|
-
*/
|
|
45
|
-
async function generate_ollama(messages: Message[]): Promise<Message> {
|
|
46
|
-
// Create a new instance of the OpenAI class.
|
|
47
|
-
const ollama = new Ollama({ host: process.env.OLLAMA_URI || 'http://localhost:11434' });
|
|
48
|
-
// Call the Ollama API.
|
|
49
|
-
const response = await ollama.chat({
|
|
50
|
-
model: process.env.OLLAMA_MODEL || 'llama3.1',
|
|
51
|
-
messages: messages,
|
|
52
|
-
});
|
|
53
|
-
// Return the response.
|
|
54
|
-
return response['message'];
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
/**
|
|
58
|
-
* Generate a response using Cloudflare AI API.
|
|
59
|
-
*
|
|
60
|
-
* @param messages the messages to be sent to Cloudflare AI.
|
|
61
|
-
* @returns the response string.
|
|
62
|
-
*/
|
|
63
|
-
async function generate_cloudflare(messages: CloudflareMessage[]): Promise<CloudflareMessage> {
|
|
64
|
-
// Generate API URL based on the environment variables.
|
|
65
|
-
const model_url = 'https://api.cloudflare.com/client/v4/accounts/' + process.env.CLOUDFLARE_ACCOUNT_ID + '/ai/run/' + process.env.CLOUDFLARE_MODEL;
|
|
66
|
-
// Call the Cloudflare AI API.
|
|
67
|
-
const response = await axios({
|
|
68
|
-
method: 'post',
|
|
69
|
-
url: model_url,
|
|
70
|
-
headers: {
|
|
71
|
-
'Authorization': 'Bearer ' + process.env.CLOUDFLARE_AUTH_KEY,
|
|
72
|
-
'Content-Type' : 'application/json',
|
|
73
|
-
},
|
|
74
|
-
data: {
|
|
75
|
-
messages: messages,
|
|
76
|
-
},
|
|
77
|
-
});
|
|
78
|
-
// Extract the response message.
|
|
79
|
-
const msg = response.data.success ? response.data.result.response : '';
|
|
80
|
-
// Return the response.
|
|
81
|
-
return { role: 'assistant', content: msg };
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
/**
|
|
85
|
-
* Generate a response using an LLM.
|
|
86
|
-
*
|
|
87
|
-
* @param messages the messages to be sent to the LLM.
|
|
88
|
-
* @returns the response string.
|
|
89
|
-
*/
|
|
90
|
-
export async function generate(messages: MessageInputParam[]): Promise<MessageInputParam> {
|
|
91
|
-
// Check what LLM to use, based on the environment variables.
|
|
92
|
-
if (process.env.OPENAI_API_KEY) {
|
|
93
|
-
// If openai key is available, use openai.
|
|
94
|
-
return await generate_openai(messages as ChatCompletionMessageParam[]);
|
|
95
|
-
|
|
96
|
-
} else if (process.env.CLOUDFLARE_ACCOUNT_ID && process.env.CLOUDFLARE_AUTH_KEY && process.env.CLOUDFLARE_MODEL) {
|
|
97
|
-
// If cloudflare keys are available, use cloudflare.
|
|
98
|
-
return await generate_cloudflare(messages as CloudflareMessage[]);
|
|
99
|
-
|
|
100
|
-
} else if (process.env.OLLAMA_URI) {
|
|
101
|
-
// If ollama is available, use ollama.
|
|
102
|
-
return await generate_ollama(messages as Message[]);
|
|
103
|
-
|
|
104
|
-
} else {
|
|
105
|
-
// Throw an error if no LLM is available.
|
|
106
|
-
throw new Error('No available LLM found.');
|
|
107
|
-
}
|
|
108
|
-
}
|
package/tsconfig.json
DELETED
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
// Visit https://aka.ms/tsconfig to read more about this file
|
|
3
|
-
"compilerOptions": {
|
|
4
|
-
// File Layout
|
|
5
|
-
"rootDir": "./src",
|
|
6
|
-
"outDir": "./dist",
|
|
7
|
-
|
|
8
|
-
// Environment Settings
|
|
9
|
-
// See also https://aka.ms/tsconfig/module
|
|
10
|
-
"module": "nodenext",
|
|
11
|
-
"target": "esnext",
|
|
12
|
-
"types": [],
|
|
13
|
-
// For nodejs:
|
|
14
|
-
// "lib": ["esnext"],
|
|
15
|
-
// "types": ["node"],
|
|
16
|
-
// and npm install -D @types/node
|
|
17
|
-
|
|
18
|
-
// Other Outputs
|
|
19
|
-
"sourceMap": true,
|
|
20
|
-
"declaration": true,
|
|
21
|
-
"declarationMap": true,
|
|
22
|
-
|
|
23
|
-
// Stricter Typechecking Options
|
|
24
|
-
"noUncheckedIndexedAccess": true,
|
|
25
|
-
"exactOptionalPropertyTypes": true,
|
|
26
|
-
|
|
27
|
-
// Style Options
|
|
28
|
-
// "noImplicitReturns": true,
|
|
29
|
-
// "noImplicitOverride": true,
|
|
30
|
-
// "noUnusedLocals": true,
|
|
31
|
-
// "noUnusedParameters": true,
|
|
32
|
-
// "noFallthroughCasesInSwitch": true,
|
|
33
|
-
// "noPropertyAccessFromIndexSignature": true,
|
|
34
|
-
|
|
35
|
-
// Recommended Options
|
|
36
|
-
"strict": true,
|
|
37
|
-
"jsx": "react-jsx",
|
|
38
|
-
"verbatimModuleSyntax": true,
|
|
39
|
-
"isolatedModules": true,
|
|
40
|
-
"noUncheckedSideEffectImports": true,
|
|
41
|
-
"moduleDetection": "force",
|
|
42
|
-
"skipLibCheck": true,
|
|
43
|
-
}
|
|
44
|
-
}
|
|
File without changes
|