@derogab/llm-proxy 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. package/.github/workflows/release.yml +96 -1
  2. package/README.md +115 -1
  3. package/dist/index.d.ts.map +1 -1
  4. package/dist/index.js +66 -1
  5. package/dist/index.js.map +1 -1
  6. package/package.json +3 -2
  7. package/src/index.ts +68 -1
  8. package/.github/workflows/publish.yml +0 -54
package/.github/workflows/release.yml CHANGED
@@ -1,4 +1,4 @@
1
- name: Create Release
1
+ name: Release and publish package to NPM
2
2
 
3
3
  on:
4
4
  push:
@@ -24,3 +24,98 @@ jobs:
24
24
  --repo="$GITHUB_REPOSITORY" \
25
25
  --title="v${tag#v}" \
26
26
  --generate-notes
27
+ # Publish the package.
28
+ publish-npm:
29
+ name: Publish Package on NPM
30
+ needs: release
31
+ runs-on: ubuntu-latest
32
+ permissions:
33
+ contents: read
34
+ id-token: write
35
+ steps:
36
+ - name: Checkout
37
+ uses: actions/checkout@v5
38
+ - name: Setup Node
39
+ uses: actions/setup-node@v4
40
+ with:
41
+ node-version: '20.x'
42
+ cache: 'npm'
43
+ registry-url: 'https://registry.npmjs.org'
44
+ always-auth: true
45
+ - name: Install dependencies (clean)
46
+ run: npm ci
47
+ - name: Type check
48
+ run: npx tsc -p tsconfig.json --noEmit
49
+ - name: Run tests
50
+ run: npm test --if-present
51
+ - name: Build
52
+ run: |
53
+ if npm run | grep -q "build"; then
54
+ npm run build
55
+ else
56
+ # Fall back to a standard TS build if no script is defined
57
+ npx tsc -p tsconfig.json
58
+ fi
59
+ - name: Verify tag matches package.json version
60
+ run: |
61
+ PKG_VERSION="$(node -p "require('./package.json').version")"
62
+ TAG_VERSION="${GITHUB_REF_NAME#v}" # supports tags like v1.2.3
63
+ echo "package.json: $PKG_VERSION"
64
+ echo "release tag: $TAG_VERSION"
65
+ if [ "$PKG_VERSION" != "$TAG_VERSION" ]; then
66
+ echo "Release tag ($TAG_VERSION) does not match package.json version ($PKG_VERSION)."
67
+ exit 1
68
+ fi
69
+ - name: Show publish contents (dry run)
70
+ run: npm pack --dry-run
71
+ - name: Publish to npm (with provenance)
72
+ env:
73
+ NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
74
+ run: npm publish --provenance --access public
75
+ publish-github:
76
+ name: Publish Package on GitHub
77
+ needs: release
78
+ runs-on: ubuntu-latest
79
+ permissions:
80
+ contents: read
81
+ id-token: write
82
+ steps:
83
+ - name: Checkout
84
+ uses: actions/checkout@v5
85
+ - name: Setup Node
86
+ uses: actions/setup-node@v4
87
+ with:
88
+ node-version: '20.x'
89
+ cache: 'npm'
90
+ registry-url: 'https://npm.pkg.github.com'
91
+ always-auth: true
92
+ - name: Install dependencies (clean)
93
+ run: npm ci
94
+ - name: Type check
95
+ run: npx tsc -p tsconfig.json --noEmit
96
+ - name: Run tests
97
+ run: npm test --if-present
98
+ - name: Build
99
+ run: |
100
+ if npm run | grep -q "build"; then
101
+ npm run build
102
+ else
103
+ # Fall back to a standard TS build if no script is defined
104
+ npx tsc -p tsconfig.json
105
+ fi
106
+ - name: Verify tag matches package.json version
107
+ run: |
108
+ PKG_VERSION="$(node -p "require('./package.json').version")"
109
+ TAG_VERSION="${GITHUB_REF_NAME#v}" # supports tags like v1.2.3
110
+ echo "package.json: $PKG_VERSION"
111
+ echo "release tag: $TAG_VERSION"
112
+ if [ "$PKG_VERSION" != "$TAG_VERSION" ]; then
113
+ echo "Release tag ($TAG_VERSION) does not match package.json version ($PKG_VERSION)."
114
+ exit 1
115
+ fi
116
+ - name: Show publish contents (dry run)
117
+ run: npm pack --dry-run
118
+ - name: Publish to npm (with provenance)
119
+ env:
120
+ NODE_AUTH_TOKEN: ${{ secrets.NPM_GITHUB_TOKEN }}
121
+ run: npm publish --provenance --access public
package/README.md CHANGED
@@ -1,2 +1,116 @@
1
1
  # llm-proxy
2
- Simple LLM Proxy for seamless API integration
2
+ A simple and lightweight proxy for seamless integration with multiple LLM providers including OpenAI, Ollama, Cloudflare AI, and Llama.cpp.
3
+
4
+ ## Features
5
+
6
+ - **Multi-provider support**: Switch between OpenAI, Ollama, Cloudflare AI, and Llama.cpp with environment variables.
7
+ - **TypeScript support**: Full TypeScript definitions included.
8
+ - **Simple API**: Single function interface for all providers.
9
+ - **Automatic provider detection**: Automatically selects the best available provider based on environment variables.
10
+
11
+ ## Installation
12
+
13
+ ```bash
14
+ npm install @derogab/llm-proxy
15
+ ```
16
+
17
+ ## Quick Start
18
+
19
+ ```typescript
20
+ import { generate } from '@derogab/llm-proxy';
21
+
22
+ const messages = [
23
+ { role: 'user', content: 'Hello, how are you?' }
24
+ ];
25
+
26
+ const response = await generate(messages);
27
+ console.log(response.content);
28
+ ```
29
+
30
+ ## Configuration
31
+
32
+ The package automatically detects which LLM provider to use based on your environment variables.
33
+ Configure one or more providers:
34
+
35
+ ### OpenAI
36
+ ```bash
37
+ OPENAI_API_KEY=your_openai_api_key # Required
38
+ OPENAI_BASE_URL=https://api.openai.com/v1 # Optional
39
+ OPENAI_MODEL=gpt-4o-mini # Optional, defaults to gpt-4o-mini
40
+ ```
41
+
42
+ ### Cloudflare AI
43
+ ```bash
44
+ CLOUDFLARE_ACCOUNT_ID=your_account_id # Required
45
+ CLOUDFLARE_AUTH_KEY=your_auth_key # Required
46
+ CLOUDFLARE_MODEL=your_model_name # Required
47
+ ```
48
+
49
+ ### Ollama (Local)
50
+ ```bash
51
+ OLLAMA_URI=http://localhost:11434 # Optional, defaults to http://localhost:11434
52
+ OLLAMA_MODEL=llama3.1 # Optional, defaults to llama3.1
53
+ ```
54
+
55
+ ### Llama.cpp (Local)
56
+ ```bash
57
+ LLAMA_CPP_MODEL_PATH=/path/to/your/model.gguf # Required, path to your GGUF model file
58
+ ```
59
+
60
+ ## API Reference
61
+
62
+ ### `generate(messages: MessageInputParam[]): Promise<MessageInputParam>`
63
+
64
+ Generates a response from the configured LLM provider.
65
+
66
+ **Parameters:**
67
+ - `messages`: Array of message objects with `role` and `content` properties
68
+
69
+ **Returns:**
70
+ - Promise that resolves to a message object with `role` and `content` properties
71
+
72
+ **Message Format:**
73
+ ```typescript
74
+ type MessageInputParam = {
75
+ role: 'user' | 'assistant' | 'system';
76
+ content: string;
77
+ };
78
+ ```
79
+
80
+ ## Provider Priority
81
+
82
+ The package selects providers in the following order:
83
+ 1. **OpenAI** (if `OPENAI_API_KEY` is set)
84
+ 2. **Cloudflare AI** (if `CLOUDFLARE_ACCOUNT_ID`, `CLOUDFLARE_AUTH_KEY`, and `CLOUDFLARE_MODEL` are set)
85
+ 3. **Ollama** (if `OLLAMA_URI` is set)
86
+ 4. **Llama.cpp** (if `LLAMA_CPP_MODEL_PATH` is set)
87
+
88
+ If no providers are configured, the function throws an error.
89
+
90
+ ## Development
91
+
92
+ ```bash
93
+ # Install dependencies
94
+ npm install
95
+
96
+ # Build the package
97
+ npm run build
98
+ ```
99
+
100
+ ## Credits
101
+ _LLM Proxy_ is made with ♥ by [derogab](https://github.com/derogab) and it's released under the [MIT license](./LICENSE).
102
+
103
+ ## Contributors
104
+
105
+ <a href="https://github.com/derogab/llm-proxy/graphs/contributors">
106
+ <img src="https://contrib.rocks/image?repo=derogab/llm-proxy" />
107
+ </a>
108
+
109
+ ## Tip
110
+ If you like this project or directly benefit from it, please consider buying me a coffee:
111
+ 🔗 `bc1qd0qatgz8h62uvnr74utwncc6j5ckfz2v2g4lef`
112
+ ⚡️ `derogab@sats.mobi`
113
+ 💶 [Sponsor on GitHub](https://github.com/sponsors/derogab)
114
+
115
+ ## Stargazers over time
116
+ [![Stargazers over time](https://starchart.cc/derogab/llm-proxy.svg?variant=adaptive)](https://starchart.cc/derogab/llm-proxy)
package/dist/index.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,kBAAkB,CAAC;AACnE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAC;AAEtC,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG,0BAA0B,GAAG,OAAO,GAAG,iBAAiB,CAAC;AAoEzF;;;;;GAKG;AACH,wBAAsB,QAAQ,CAAC,QAAQ,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAkBxF"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,kBAAkB,CAAC;AAEnE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAC;AAEtC,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG,0BAA0B,GAAG,OAAO,GAAG,iBAAiB,CAAC;AAiIzF;;;;;GAKG;AACH,wBAAsB,QAAQ,CAAC,QAAQ,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAsBxF"}
package/dist/index.js CHANGED
@@ -1,6 +1,7 @@
1
1
  // Dependencies.
2
2
  import axios from 'axios';
3
3
  import * as dotenv from 'dotenv';
4
+ import { getLlama, LlamaChatSession } from "node-llama-cpp";
4
5
  import { Ollama } from 'ollama';
5
6
  import OpenAI from 'openai';
6
7
  // Configs.
@@ -13,7 +14,7 @@ dotenv.config();
13
14
  */
14
15
  async function generate_openai(messages) {
15
16
  // Create a new instance of the OpenAI class.
16
- const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
17
+ const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY, baseURL: process.env.OPENAI_BASE_URL });
17
18
  // Call the OpenAI API.
18
19
  const chatCompletion = await openai.chat.completions.create({
19
20
  messages: messages,
@@ -39,6 +40,66 @@ async function generate_ollama(messages) {
39
40
  // Return the response.
40
41
  return response['message'];
41
42
  }
43
+ /**
44
+ * Convert messages to chat history.
45
+ *
46
+ * Llama.cpp expects the chat history in a custom format.
47
+ * Convert the default messages format to the Llama.cpp format.
48
+ *
49
+ * @param messages the messages to be sent to Llama.cpp.
50
+ * @returns the same messages in the Llama.cpp format.
51
+ */
52
+ function convert_messages_to_chat_history(messages) {
53
+ // Init chat history.
54
+ const chat_history = [];
55
+ // Loop through messages.
56
+ for (const message of messages) {
57
+ if (message.role === 'system' || message.role === 'user') {
58
+ chat_history.push({
59
+ type: message.role,
60
+ text: message.content
61
+ });
62
+ }
63
+ else if (message.role === 'assistant') {
64
+ chat_history.push({
65
+ type: "model",
66
+ response: [message.content]
67
+ });
68
+ }
69
+ }
70
+ // Return the chat history.
71
+ return chat_history;
72
+ }
73
+ /**
74
+ * Generate a response using Llama.cpp Local Model.
75
+ *
76
+ * @param messages the messages to be sent to Llama.cpp.
77
+ * @returns the response string.
78
+ */
79
+ async function generate_llama_cpp(messages) {
80
+ // Create a new instance of the Llama.cpp class.
81
+ const llama = await getLlama();
82
+ // Set model to use.
83
+ const modelPath = process.env.LLAMA_CPP_MODEL_PATH;
84
+ if (!modelPath) {
85
+ throw new Error('LLAMA_CPP_MODEL_PATH is not set.');
86
+ }
87
+ const model = await llama.loadModel({
88
+ modelPath: modelPath,
89
+ });
90
+ // Import history into the context.
91
+ const context = await model.createContext();
92
+ const session = new LlamaChatSession({
93
+ contextSequence: context.getSequence()
94
+ });
95
+ if (messages.length > 1)
96
+ session.setChatHistory(convert_messages_to_chat_history(messages.slice(0, -1)));
97
+ // Generate and return the response.
98
+ return {
99
+ role: 'assistant',
100
+ content: await session.prompt(messages[messages.length - 1]?.content || ''),
101
+ };
102
+ }
42
103
  /**
43
104
  * Generate a response using Cloudflare AI API.
44
105
  *
@@ -85,6 +146,10 @@ export async function generate(messages) {
85
146
  // If ollama is available, use ollama.
86
147
  return await generate_ollama(messages);
87
148
  }
149
+ else if (process.env.LLAMA_CPP_MODEL_PATH) {
150
+ // If llama_cpp is available, use llama_cpp.
151
+ return await generate_llama_cpp(messages);
152
+ }
88
153
  else {
89
154
  // Throw an error if no LLM is available.
90
155
  throw new Error('No available LLM found.');
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,gBAAgB;AAChB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAa5B,WAAW;AACX,MAAM,CAAC,MAAM,EAAE,CAAC;AAEhB;;;;;GAKG;AACH,KAAK,UAAU,eAAe,CAAC,QAAsC;IACnE,6CAA6C;IAC7C,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC,CAAC;IAClE,uBAAuB;IACvB,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;QAC1D,QAAQ,EAAE,QAAQ;QAClB,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,aAAa;KACjD,CAAC,CAAC;IACH,uBAAuB;IACvB,OAAO,cAAc,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,OAAqC,CAAC;AAC3E,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,eAAe,CAAC,QAAmB;IAChD,6CAA6C;IAC7C,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,wBAAwB,EAAE,CAAC,CAAC;IACxF,uBAAuB;IACvB,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC;QACjC,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,UAAU;QAC7C,QAAQ,EAAE,QAAQ;KACnB,CAAC,CAAC;IACH,uBAAuB;IACvB,OAAO,QAAQ,CAAC,SAAS,CAAC,CAAC;AAC7B,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,mBAAmB,CAAC,QAA6B;IAC9D,uDAAuD;IACvD,MAAM,SAAS,GAAG,gDAAgD,GAAG,OAAO,CAAC,GAAG,CAAC,qBAAqB,GAAG,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;IACnJ,8BAA8B;IAC9B,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC;QAC3B,MAAM,EAAE,MAAM;QACd,GAAG,EAAE,SAAS;QACd,OAAO,EAAE;YACP,eAAe,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB;YAC5D,cAAc,EAAG,kBAAkB;SACpC;QACD,IAAI,EAAE;YACJ,QAAQ,EAAE,QAAQ;SACnB;KACF,CAAC,CAAC;IACH,gCAAgC;IAChC,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;IACvE,uBAAuB;IACvB,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;AAC7C,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,QAA6B;IAC1D,6DAA6D;IAC7D,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC;QAC/B,0CAA0C;QAC1C,OAAO,MAAM,eAAe,CAAC,QAAwC,CAAC,CAAC;IAEzE,CAAC;SAAM,IAAI,OAAO,CAAC,GAAG,CAAC,qBAAqB,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QAChH,oDAAoD;QACpD,OAAO,MAAM,mBAAmB,CAAC,QAA+B,CAAC,CAAC;IAEpE,CAAC;SAAM,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC;QAClC,sCAAsC;QACtC,OAAO,MAAM,eAAe,CAAC,QAAqB,CAAC,CAAC;IAEtD,CAAC;SAAM,CAAC;QACN,yCAAyC;QACzC,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAC7C,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,gBAAgB;AAChB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAc5B,WAAW;AACX,MAAM,CAAC,MAAM,EAAE,CAAC;AAEhB;;;;;GAKG;AACH,KAAK,UAAU,eAAe,CAAC,QAAsC;IACnE,6CAA6C;IAC7C,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC,CAAC;IACxG,uBAAuB;IACvB,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;QAC1D,QAAQ,EAAE,QAAQ;QAClB,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,aAAa;KACjD,CAAC,CAAC;IACH,uBAAuB;IACvB,OAAO,cAAc,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,OAAqC,CAAC;AAC3E,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,eAAe,CAAC,QAAmB;IAChD,6CAA6C;IAC7C,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,wBAAwB,EAAE,CAAC,CAAC;IACxF,uBAAuB;IACvB,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC;QACjC,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,UAAU;QAC7C,QAAQ,EAAE,QAAQ;KACnB,CAAC,CAAC;IACH,uBAAuB;IACvB,OAAO,QAAQ,CAAC,SAAS,CAAC,CAAC;AAC7B,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,gCAAgC,CAAC,QAAmB;IAC3D,qBAAqB;IACrB,MAAM,YAAY,GAAsB,EAAE,CAAC;IAC3C,yBAAyB;IACzB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,OAAO,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACzD,YAAY,CAAC,IAAI,CAAC;gBAChB,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,IAAI,EAAE,OAAO,CAAC,OAAO;aACtB,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YACxC,YAAY,CAAC,IAAI,CAAC;gBAChB,IAAI,EAAE,OAAO;gBACb,QAAQ,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC;aAC5B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,2BAA2B;IAC3B,OAAO,YAAY,CAAC;AACtB,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,kBAAkB,CAAC,QAAmB;IACnD,gDAAgD;IAChD,MAAM,KAAK,GAAG,MAAM,QAAQ,EAAE,CAAC;IAC/B,oBAAoB;IACpB,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC;IACnD,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACtD,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC;QAClC,SAAS,EAAE,SAAS;KACrB,CAAC,CAAC;IACH,mCAAmC;IACnC,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,aAAa,EAAE,CAAC;IAC5C,MAAM,OAAO,GAAG,IAAI,gBAAgB,CAAC;QACnC,eAAe,EAAE,OAAO,CAAC,WAAW,EAAE;KACvC,CAAC,CAAC;IACH,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,cAAc,CAAC,gCAAgC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEzG,oCAAoC;IACpC,OAAO;QACL,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,OAAO,IAAI,EAAE,CAAC;KAC5E,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,mBAAmB,CAAC,QAA6B;IAC9D,uDAAuD;IACvD,MAAM,SAAS,GAAG,gDAAgD,GAAG,OAAO,CAAC,GAAG,CAAC,qBAAqB,GAAG,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;IACnJ,8BAA8B;IAC9B,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC;QAC3B,MAAM,EAAE,MAAM;QACd,GAAG,EAAE,SAAS;QACd,OAAO,EAAE;YACP,eAAe,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB;YAC5D,cAAc,EAAG,kBAAkB;SACpC;QACD,IAAI,EAAE;YACJ,QAAQ,EAAE,QAAQ;SACnB;KACF,CAAC,CAAC;IACH,gCAAgC;IAChC,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;IACvE,uBAAuB;IACvB,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;AAC7C,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,QAA6B;IAC1D,6DAA6D;IAC7D,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC;QAC/B,0CAA0C;QAC1C,OAAO,MAAM,eAAe,CAAC,QAAwC,CAAC,CAAC;IAEzE,CAAC;SAAM,IAAI,OAAO,CAAC,GAAG,CAAC,qBAAqB,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QAChH,oDAAoD;QACpD,OAAO,MAAM,mBAAmB,CAAC,QAA+B,CAAC,CAAC;IAEpE,CAAC;SAAM,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC;QAClC,sCAAsC;QACtC,OAAO,MAAM,eAAe,CAAC,QAAqB,CAAC,CAAC;IAEtD,CAAC;SAAM,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,CAAC;QAC5C,4CAA4C;QAC5C,OAAO,MAAM,kBAAkB,CAAC,QAAqB,CAAC,CAAC;IAEzD,CAAC;SAAM,CAAC;QACN,yCAAyC;QACzC,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAC7C,CAAC;AACH,CAAC"}
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@derogab/llm-proxy",
3
- "description": "Simple LLM Proxy for seamless API integration",
4
- "version": "0.1.0",
3
+ "description": "A simple and lightweight proxy for seamless integration with multiple LLM providers including OpenAI, Ollama, and Cloudflare AI",
4
+ "version": "0.3.0",
5
5
  "author": "derogab",
6
6
  "license": "MIT",
7
7
  "repository": {
@@ -26,6 +26,7 @@
26
26
  "dependencies": {
27
27
  "axios": "1.11.0",
28
28
  "dotenv": "17.2.1",
29
+ "node-llama-cpp": "3.13.0",
29
30
  "ollama": "0.5.17",
30
31
  "openai": "5.16.0"
31
32
  }
package/src/index.ts CHANGED
@@ -1,11 +1,13 @@
1
1
  // Dependencies.
2
2
  import axios from 'axios';
3
3
  import * as dotenv from 'dotenv';
4
+ import { getLlama, LlamaChatSession } from "node-llama-cpp";
4
5
  import { Ollama } from 'ollama';
5
6
  import OpenAI from 'openai';
6
7
 
7
8
  // Types.
8
9
  import type { ChatCompletionMessageParam } from 'openai/resources';
10
+ import type { ChatHistoryItem } from 'node-llama-cpp';
9
11
  import type { Message } from 'ollama';
10
12
 
11
13
  export type CloudflareMessage = {
@@ -26,7 +28,7 @@ dotenv.config();
26
28
  */
27
29
  async function generate_openai(messages: ChatCompletionMessageParam[]): Promise<ChatCompletionMessageParam> {
28
30
  // Create a new instance of the OpenAI class.
29
- const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
31
+ const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY, baseURL: process.env.OPENAI_BASE_URL });
30
32
  // Call the OpenAI API.
31
33
  const chatCompletion = await openai.chat.completions.create({
32
34
  messages: messages,
@@ -54,6 +56,67 @@ async function generate_ollama(messages: Message[]): Promise<Message> {
54
56
  return response['message'];
55
57
  }
56
58
 
59
+ /**
60
+ * Convert messages to chat history.
61
+ *
62
+ * Llama.cpp expects the chat history in a custom format.
63
+ * Convert the default messages format to the Llama.cpp format.
64
+ *
65
+ * @param messages the messages to be sent to Llama.cpp.
66
+ * @returns the same messages in the Llama.cpp format.
67
+ */
68
+ function convert_messages_to_chat_history(messages: Message[]): ChatHistoryItem[] {
69
+ // Init chat history.
70
+ const chat_history: ChatHistoryItem[] = [];
71
+ // Loop through messages.
72
+ for (const message of messages) {
73
+ if (message.role === 'system' || message.role === 'user') {
74
+ chat_history.push({
75
+ type: message.role,
76
+ text: message.content
77
+ });
78
+ } else if (message.role === 'assistant') {
79
+ chat_history.push({
80
+ type: "model",
81
+ response: [message.content]
82
+ });
83
+ }
84
+ }
85
+ // Return the chat history.
86
+ return chat_history;
87
+ }
88
+
89
+ /**
90
+ * Generate a response using Llama.cpp Local Model.
91
+ *
92
+ * @param messages the messages to be sent to Llama.cpp.
93
+ * @returns the response string.
94
+ */
95
+ async function generate_llama_cpp(messages: Message[]): Promise<Message> {
96
+ // Create a new instance of the Llama.cpp class.
97
+ const llama = await getLlama();
98
+ // Set model to use.
99
+ const modelPath = process.env.LLAMA_CPP_MODEL_PATH;
100
+ if (!modelPath) {
101
+ throw new Error('LLAMA_CPP_MODEL_PATH is not set.');
102
+ }
103
+ const model = await llama.loadModel({
104
+ modelPath: modelPath,
105
+ });
106
+ // Import history into the context.
107
+ const context = await model.createContext();
108
+ const session = new LlamaChatSession({
109
+ contextSequence: context.getSequence()
110
+ });
111
+ if (messages.length > 1) session.setChatHistory(convert_messages_to_chat_history(messages.slice(0, -1)));
112
+
113
+ // Generate and return the response.
114
+ return {
115
+ role: 'assistant',
116
+ content: await session.prompt(messages[messages.length - 1]?.content || ''),
117
+ };
118
+ }
119
+
57
120
  /**
58
121
  * Generate a response using Cloudflare AI API.
59
122
  *
@@ -101,6 +164,10 @@ export async function generate(messages: MessageInputParam[]): Promise<MessageIn
101
164
  // If ollama is available, use ollama.
102
165
  return await generate_ollama(messages as Message[]);
103
166
 
167
+ } else if (process.env.LLAMA_CPP_MODEL_PATH) {
168
+ // If llama_cpp is available, use llama_cpp.
169
+ return await generate_llama_cpp(messages as Message[]);
170
+
104
171
  } else {
105
172
  // Throw an error if no LLM is available.
106
173
  throw new Error('No available LLM found.');
package/.github/workflows/publish.yml DELETED
@@ -1,54 +0,0 @@
1
- name: Publish Package to NPM
2
-
3
- on:
4
- release:
5
- types: [published]
6
- workflow_dispatch:
7
-
8
- jobs:
9
- build:
10
- name: Publish Package
11
- runs-on: ubuntu-latest
12
- permissions:
13
- contents: read
14
- id-token: write
15
- steps:
16
- - name: Checkout
17
- uses: actions/checkout@v4
18
- - name: Setup Node
19
- uses: actions/setup-node@v4
20
- with:
21
- node-version: '20.x'
22
- cache: 'npm'
23
- registry-url: 'https://registry.npmjs.org'
24
- always-auth: true
25
- - name: Install dependencies (clean)
26
- run: npm ci
27
- - name: Type check
28
- run: npx tsc -p tsconfig.json --noEmit
29
- - name: Run tests
30
- run: npm test --if-present
31
- - name: Build
32
- run: |
33
- if npm run | grep -q "build"; then
34
- npm run build
35
- else
36
- # Fall back to a standard TS build if no script is defined
37
- npx tsc -p tsconfig.json
38
- fi
39
- - name: Verify tag matches package.json version
40
- run: |
41
- PKG_VERSION="$(node -p "require('./package.json').version")"
42
- TAG_VERSION="${GITHUB_REF_NAME#v}" # supports tags like v1.2.3
43
- echo "package.json: $PKG_VERSION"
44
- echo "release tag: $TAG_VERSION"
45
- if [ "$PKG_VERSION" != "$TAG_VERSION" ]; then
46
- echo "Release tag ($TAG_VERSION) does not match package.json version ($PKG_VERSION)."
47
- exit 1
48
- fi
49
- - name: Show publish contents (dry run)
50
- run: npm pack --dry-run
51
- - name: Publish to npm (with provenance)
52
- env:
53
- NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
54
- run: npm publish --provenance --access public