@memberjunction/ai-groq 3.4.0 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +89 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -17
- package/dist/index.js.map +1 -1
- package/dist/models/groq.d.ts +0 -1
- package/dist/models/groq.d.ts.map +1 -1
- package/dist/models/groq.js +16 -22
- package/dist/models/groq.js.map +1 -1
- package/package.json +7 -6
- package/readme.md +54 -257
package/README.md
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# @memberjunction/ai-groq
|
|
2
|
+
|
|
3
|
+
MemberJunction AI provider for Groq's ultra-fast inference platform. This package extends the OpenAI provider to work with Groq's OpenAI-compatible API, providing access to LLM inference powered by Groq's custom Language Processing Units (LPUs) for industry-leading inference speed.
|
|
4
|
+
|
|
5
|
+
## Architecture
|
|
6
|
+
|
|
7
|
+
```mermaid
|
|
8
|
+
graph TD
|
|
9
|
+
A["GroqLLM<br/>(Provider)"] -->|extends| B["OpenAILLM<br/>(@memberjunction/ai-openai)"]
|
|
10
|
+
B -->|extends| C["BaseLLM<br/>(@memberjunction/ai)"]
|
|
11
|
+
A -->|overrides base URL| D["Groq API<br/>(api.groq.com/openai/v1)"]
|
|
12
|
+
D -->|runs on| E["Groq LPU<br/>Inference Engine"]
|
|
13
|
+
C -->|registered via| F["@RegisterClass"]
|
|
14
|
+
|
|
15
|
+
style A fill:#7c5295,stroke:#563a6b,color:#fff
|
|
16
|
+
style B fill:#2d6a9f,stroke:#1a4971,color:#fff
|
|
17
|
+
style C fill:#2d6a9f,stroke:#1a4971,color:#fff
|
|
18
|
+
style D fill:#2d8659,stroke:#1a5c3a,color:#fff
|
|
19
|
+
style E fill:#b8762f,stroke:#8a5722,color:#fff
|
|
20
|
+
style F fill:#b8762f,stroke:#8a5722,color:#fff
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
## Features
|
|
24
|
+
|
|
25
|
+
- **Ultra-Fast Inference**: Groq's custom LPU hardware delivers extremely low latency
|
|
26
|
+
- **OpenAI Compatible**: Inherits all features from the OpenAI provider
|
|
27
|
+
- **Streaming**: Full streaming support for real-time responses
|
|
28
|
+
- **Thinking/Reasoning**: Thinking block extraction for reasoning models
|
|
29
|
+
- **Multiple Models**: Access to Llama, Mixtral, Gemma, and other open models optimized for Groq
|
|
30
|
+
|
|
31
|
+
## Installation
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
npm install @memberjunction/ai-groq
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
## Usage
|
|
38
|
+
|
|
39
|
+
```typescript
|
|
40
|
+
import { GroqLLM } from '@memberjunction/ai-groq';
|
|
41
|
+
|
|
42
|
+
const llm = new GroqLLM('your-groq-api-key');
|
|
43
|
+
|
|
44
|
+
const result = await llm.ChatCompletion({
|
|
45
|
+
model: 'llama-3.1-70b-versatile',
|
|
46
|
+
messages: [
|
|
47
|
+
{ role: 'system', content: 'You are a helpful assistant.' },
|
|
48
|
+
{ role: 'user', content: 'Explain neural networks.' }
|
|
49
|
+
],
|
|
50
|
+
temperature: 0.7,
|
|
51
|
+
maxOutputTokens: 1000
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
if (result.success) {
|
|
55
|
+
console.log(result.data.choices[0].message.content);
|
|
56
|
+
}
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
### Streaming
|
|
60
|
+
|
|
61
|
+
```typescript
|
|
62
|
+
const result = await llm.ChatCompletion({
|
|
63
|
+
model: 'llama-3.1-8b-instant',
|
|
64
|
+
messages: [{ role: 'user', content: 'Write a haiku about speed.' }],
|
|
65
|
+
streaming: true,
|
|
66
|
+
streamingCallbacks: {
|
|
67
|
+
OnContent: (content) => process.stdout.write(content),
|
|
68
|
+
OnComplete: () => console.log('\nDone!')
|
|
69
|
+
}
|
|
70
|
+
});
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
## How It Works
|
|
74
|
+
|
|
75
|
+
`GroqLLM` is a thin subclass of `OpenAILLM` that redirects API calls to Groq's endpoint at `https://api.groq.com/openai/v1`. Since Groq implements an OpenAI-compatible API, all chat, streaming, and parameter handling logic is inherited from the OpenAI provider.
|
|
76
|
+
|
|
77
|
+
## Supported Parameters
|
|
78
|
+
|
|
79
|
+
All parameters supported by the OpenAI provider are available, including `temperature`, `maxOutputTokens`, `topP`, `frequencyPenalty`, `presencePenalty`, `seed`, `stopSequences`, and `responseFormat`.
|
|
80
|
+
|
|
81
|
+
## Class Registration
|
|
82
|
+
|
|
83
|
+
Registered as `GroqLLM` via `@RegisterClass(BaseLLM, 'GroqLLM')`.
|
|
84
|
+
|
|
85
|
+
## Dependencies
|
|
86
|
+
|
|
87
|
+
- `@memberjunction/ai` - Core AI abstractions
|
|
88
|
+
- `@memberjunction/ai-openai` - OpenAI provider (parent class)
|
|
89
|
+
- `@memberjunction/global` - Class registration
|
package/dist/index.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export * from './models/groq';
|
|
1
|
+
export * from './models/groq.js';
|
|
2
2
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.js
CHANGED
|
@@ -1,18 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
-
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
-
};
|
|
16
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
__exportStar(require("./models/groq"), exports);
|
|
1
|
+
export * from './models/groq.js';
|
|
18
2
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC"}
|
package/dist/models/groq.d.ts
CHANGED
|
@@ -52,5 +52,4 @@ export declare class GroqLLM extends BaseLLM {
|
|
|
52
52
|
SummarizeText(params: SummarizeParams): Promise<SummarizeResult>;
|
|
53
53
|
ClassifyText(params: ClassifyParams): Promise<ClassifyResult>;
|
|
54
54
|
}
|
|
55
|
-
export declare function LoadGroqLLM(): void;
|
|
56
55
|
//# sourceMappingURL=groq.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"groq.d.ts","sourceRoot":"","sources":["../../src/models/groq.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAqC,cAAc,EAAE,cAAc,EAAE,eAAe,EAAE,eAAe,EAAmE,MAAM,oBAAoB,CAAC;AAE3O,OAAO,IAAI,MAAM,UAAU,CAAC;AAG5B;;GAEG;AACH,qBACa,OAAQ,SAAQ,OAAO;IAChC,OAAO,CAAC,OAAO,CAAO;gBAEV,MAAM,EAAE,MAAM;IAK1B;;OAEG;IACH,IAAW,UAAU,IAAI,IAAI,CAE5B;IAED;;OAEG;IACH,IAAW,MAAM,IAAI,IAAI,CAExB;IAED;;OAEG;IACH,IAAoB,iBAAiB,IAAI,OAAO,CAE/C;IAED;;;OAGG;IACH,SAAS,CAAC,sBAAsB,IAAI,OAAO;IAI3C,SAAS,CAAC,wBAAwB,CAAC,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,GAAG,IAAI;IAkC7E;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IA4C7B;;OAEG;cACa,0BAA0B,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;IA4HnF;;OAEG;cACa,sBAAsB,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC;IAuDxE;;OAEG;IACH,SAAS,CAAC,qBAAqB,CAAC,KAAK,EAAE,GAAG,GAAG;QACzC,OAAO,EAAE,MAAM,CAAC;QAChB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,KAAK,CAAC,EAAE,GAAG,CAAC;KACf;IA6BD;;OAEG;IACH,SAAS,CAAC,yBAAyB,CAC/B,kBAAkB,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,EAC7C,SAAS,EAAE,GAAG,GAAG,IAAI,GAAG,SAAS,EACjC,KAAK,EAAE,GAAG,GAAG,IAAI,GAAG,SAAS,GAC9B,UAAU;IAyCA,aAAa,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;IAIhE,YAAY,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC;CAG7E
|
|
1
|
+
{"version":3,"file":"groq.d.ts","sourceRoot":"","sources":["../../src/models/groq.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAqC,cAAc,EAAE,cAAc,EAAE,eAAe,EAAE,eAAe,EAAmE,MAAM,oBAAoB,CAAC;AAE3O,OAAO,IAAI,MAAM,UAAU,CAAC;AAG5B;;GAEG;AACH,qBACa,OAAQ,SAAQ,OAAO;IAChC,OAAO,CAAC,OAAO,CAAO;gBAEV,MAAM,EAAE,MAAM;IAK1B;;OAEG;IACH,IAAW,UAAU,IAAI,IAAI,CAE5B;IAED;;OAEG;IACH,IAAW,MAAM,IAAI,IAAI,CAExB;IAED;;OAEG;IACH,IAAoB,iBAAiB,IAAI,OAAO,CAE/C;IAED;;;OAGG;IACH,SAAS,CAAC,sBAAsB,IAAI,OAAO;IAI3C,SAAS,CAAC,wBAAwB,CAAC,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,GAAG,IAAI;IAkC7E;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IA4C7B;;OAEG;cACa,0BAA0B,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;IA4HnF;;OAEG;cACa,sBAAsB,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC;IAuDxE;;OAEG;IACH,SAAS,CAAC,qBAAqB,CAAC,KAAK,EAAE,GAAG,GAAG;QACzC,OAAO,EAAE,MAAM,CAAC;QAChB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,KAAK,CAAC,EAAE,GAAG,CAAC;KACf;IA6BD;;OAEG;IACH,SAAS,CAAC,yBAAyB,CAC/B,kBAAkB,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,EAC7C,SAAS,EAAE,GAAG,GAAG,IAAI,GAAG,SAAS,EACjC,KAAK,EAAE,GAAG,GAAG,IAAI,GAAG,SAAS,GAC9B,UAAU;IAyCA,aAAa,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;IAIhE,YAAY,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC;CAG7E"}
|
package/dist/models/groq.js
CHANGED
|
@@ -1,25 +1,22 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
2
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
3
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
4
|
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
5
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
6
|
};
|
|
8
|
-
var
|
|
9
|
-
|
|
7
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
8
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
9
|
};
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
const global_1 = require("@memberjunction/global");
|
|
15
|
-
const groq_sdk_1 = __importDefault(require("groq-sdk"));
|
|
10
|
+
import { BaseLLM, ChatResult, ChatMessageRole, ModelUsage } from '@memberjunction/ai';
|
|
11
|
+
import { RegisterClass } from '@memberjunction/global';
|
|
12
|
+
import Groq from 'groq-sdk';
|
|
16
13
|
/**
|
|
17
14
|
* Groq implementation of the BaseLLM class
|
|
18
15
|
*/
|
|
19
|
-
let GroqLLM = class GroqLLM extends
|
|
16
|
+
let GroqLLM = class GroqLLM extends BaseLLM {
|
|
20
17
|
constructor(apiKey) {
|
|
21
18
|
super(apiKey);
|
|
22
|
-
this._client = new
|
|
19
|
+
this._client = new Groq({ apiKey: apiKey });
|
|
23
20
|
}
|
|
24
21
|
/**
|
|
25
22
|
* Read only getter method to get the Groq client instance
|
|
@@ -189,7 +186,7 @@ let GroqLLM = class GroqLLM extends ai_1.BaseLLM {
|
|
|
189
186
|
const extracted = this.extractThinkingFromContent(rawMessage);
|
|
190
187
|
const res = {
|
|
191
188
|
message: {
|
|
192
|
-
role:
|
|
189
|
+
role: ChatMessageRole.assistant,
|
|
193
190
|
content: extracted.content,
|
|
194
191
|
thinking: extracted.thinking || choice.message.reasoning
|
|
195
192
|
},
|
|
@@ -199,7 +196,7 @@ let GroqLLM = class GroqLLM extends ai_1.BaseLLM {
|
|
|
199
196
|
return res;
|
|
200
197
|
});
|
|
201
198
|
// Create ModelUsage with timing data if available
|
|
202
|
-
const usage = new
|
|
199
|
+
const usage = new ModelUsage(chatResponse.usage.prompt_tokens, chatResponse.usage.completion_tokens);
|
|
203
200
|
// Groq provides detailed timing in the usage object
|
|
204
201
|
const groqUsage = chatResponse.usage;
|
|
205
202
|
if (groqUsage.queue_time !== undefined) {
|
|
@@ -327,20 +324,20 @@ let GroqLLM = class GroqLLM extends ai_1.BaseLLM {
|
|
|
327
324
|
// Create dates (will be overridden by base class)
|
|
328
325
|
const now = new Date();
|
|
329
326
|
// Create a proper ChatResult instance with constructor params
|
|
330
|
-
const result = new
|
|
327
|
+
const result = new ChatResult(true, now, now);
|
|
331
328
|
// Get thinking content from streaming state if available
|
|
332
329
|
const thinkingContent = this.thinkingStreamState?.accumulatedThinking.trim();
|
|
333
330
|
// Set all properties
|
|
334
331
|
result.data = {
|
|
335
332
|
choices: [{
|
|
336
333
|
message: this.addThinkingToMessage({
|
|
337
|
-
role:
|
|
334
|
+
role: ChatMessageRole.assistant,
|
|
338
335
|
content: accumulatedContent ? accumulatedContent : ''
|
|
339
336
|
}, thinkingContent),
|
|
340
337
|
finish_reason: finishReason,
|
|
341
338
|
index: 0
|
|
342
339
|
}],
|
|
343
|
-
usage: new
|
|
340
|
+
usage: new ModelUsage(promptTokens, completionTokens)
|
|
344
341
|
};
|
|
345
342
|
result.statusText = 'success';
|
|
346
343
|
result.errorMessage = null;
|
|
@@ -354,12 +351,9 @@ let GroqLLM = class GroqLLM extends ai_1.BaseLLM {
|
|
|
354
351
|
throw new Error("Method not implemented.");
|
|
355
352
|
}
|
|
356
353
|
};
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
(
|
|
354
|
+
GroqLLM = __decorate([
|
|
355
|
+
RegisterClass(BaseLLM, "GroqLLM"),
|
|
356
|
+
__metadata("design:paramtypes", [String])
|
|
360
357
|
], GroqLLM);
|
|
361
|
-
|
|
362
|
-
// this does nothing but prevents the class from being removed by the tree shaker
|
|
363
|
-
}
|
|
364
|
-
exports.LoadGroqLLM = LoadGroqLLM;
|
|
358
|
+
export { GroqLLM };
|
|
365
359
|
//# sourceMappingURL=groq.js.map
|
package/dist/models/groq.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"groq.js","sourceRoot":"","sources":["../../src/models/groq.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"groq.js","sourceRoot":"","sources":["../../src/models/groq.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,OAAO,EAAc,UAAU,EAAoB,eAAe,EAAoE,UAAU,EAAuD,MAAM,oBAAoB,CAAC;AAC3O,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,IAAI,MAAM,UAAU,CAAC;AAG5B;;GAEG;AAEI,IAAM,OAAO,GAAb,MAAM,OAAQ,SAAQ,OAAO;IAGhC,YAAY,MAAc;QACtB,KAAK,CAAC,MAAM,CAAC,CAAC;QACd,IAAI,CAAC,OAAO,GAAG,IAAI,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAChD,CAAC;IAED;;OAEG;IACH,IAAW,UAAU;QACjB,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED;;OAEG;IACH,IAAW,MAAM;QACb,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,IAAoB,iBAAiB;QACjC,OAAO,IAAI,CAAC;IAChB,CAAC;IAED;;;OAGG;IACO,sBAAsB;QAC5B,OAAO,IAAI,CAAC;IAChB,CAAC;IAES,wBAAwB,CAAC,UAAe,EAAE,MAAkB;QAClE,IAAI,oBAAoB,GAAG,MAAM,CAAC,WAAW,CAAC;QAC9C,IAAI,oBAAoB,EAAE,CAAC;YACvB,MAAM,aAAa,GAAG,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YACrE,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAChE,MAAM,kBAAkB,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;YACzG,IAAI,aAAa,EAAE,CAAC;gBAChB,8BAA8B;gBAC9B,eAAe;gBACf,mBAAmB;gBACnB,kBAAkB;gBAClB,IAAI,kBAAkB,KAAK,IAAI,EAAE,CAAC;oBAC9B,IAAI,kBAAkB,IAAI,CAAC,IAAI,kBAAkB,IAAI,EAAE,EAAE,CAAC;wBACtD,oBAAoB,GAAG,KAAK,CAAC;oBACjC,CAAC;yBAAM,IAAI,kBAAkB,GAAG,EAAE,IAAI,kBAAkB,IAAI,EAAE,EAAE,CAAC;wBAC7D,oBAAoB,GAAG,QAAQ,CAAC;oBACpC,CAAC;yBAAM,IAAI,kBAAkB,GAAG,EAAE,IAAI,kBAAkB,IAAI,GAAG,EAAE,CAAC;wBAC9D,oBAAoB,GAAG,MAAM,CAAC;oBAClC,CAAC;gBACL,CAAC;YACL,CAAC;iBACI,IAAI,WAAW,EAAE,CAAC;gBACnB,6IAA6I;gBAC7I,IAAI,oBAAoB,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,SAAS,EAAE,CAAC;oBAC1D,oBAAoB,GAAG,kBAAkB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC;gBACnE,CAAC;YACL,CAAC;YACD,IAAI,aAAa,IAAI,WAAW,EAAC,CAAC;gBAC9B,8EAA8E;gBAC9E,UAAU,CAAC,gBAAgB,GAAG,oBAAoB,CAAC;YACvD,CAAC;QACL,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,qBAAqB,CAAC,QAAuB;QACjD,MAAM,YAAY,GAAiC,EAAE,CAAC;QAEtD,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YACzB,wBAAwB;YACxB,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;gBAClC,YAAY,CAAC,IAAI,CAAC;oBACd,IAAI,EAAE,GAAG,CAAC,IAAuC;oBACjD,OAAO,EAAE,GAAG,CAAC,OAAO;iBACvB,CAAC,CAAC;gBACH,SAAS;YACb,CAAC;YAED,mDAAmD;YACnD,MAAM,aAAa,GAAG,GAAG,CAAC,OAAoC,CAAC;YAC/D,MAAM,WAAW,GAAgC,EAAE,CAAC;YAEpD,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;gBAChC,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;oBACxB,WAAW,CAAC,IAAI,CAAC;wBACb,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,KAAK,CAAC,OAAO;qBACtB,CAAC,CAAC;gBACP,CAAC;qBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;oBACpC,WAAW,CAAC,IAAI,CAAC;wBACb,IAAI,EAAE,WAAW;wBACjB,SAAS,EAAE;4BACP,GAAG,EAAE,KAAK,CAAC,OAAO;yBACrB;qBACJ,CAAC,CAAC;gBACP,CAAC;gBACD,iEAAiE;YACrE,CAAC;YAED,mFAAmF;YACnF,YAAY,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,GAAG,CAAC,IAAuC;gBACjD,OAAO,EAAE,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE;aACvB,CAAC,CAAC;QACrC,CAAC;QAED,OAAO,YAAY,CAAC;IACxB,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,0BAA0B,CAAC,MAAkB;QACzD,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;QAE7B,2EAA2E;QAC3E,MAAM,QAAQ,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAE7D,sDAAsD;QACtD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACvE,QAAQ,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,IAAI,CAAC,8CAA8C;aAC/D,CAAC,CAAC;QACP,CAAC;QAED,MAAM,UAAU,GAA2C;YACvD,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,QAAQ,EAAE,QAAQ;YAClB,UAAU,EAAE,MAAM,CAAC,eAAe;YAClC,WAAW,EAAE,MAAM,CAAC,WAAW;SAClC,CAAC;QAEF,iDAAiD;QACjD,IAAI,CAAC,wBAAwB,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAElD,yCAAyC;QACzC,IAAI,MAAM,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;YACtB,UAAU,CAAC,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC;QACnC,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;YACtB,UAAU,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QAClC,CAAC;QACD,IAAI,MAAM,CAAC,aAAa,IAAI,IAAI,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClE,UAAU,CAAC,IAAI,GAAG,MAAM,CAAC,aAAa,CAAC;QAC3C,CAAC;QACD,IAAI,MAAM,CAAC,gBAAgB,IAAI,IAAI,EAAE,CAAC;YAClC,UAAU,CAAC,iBAAiB,GAAG,MAAM,CAAC,gBAAgB,CAAC;QAC3D,CAAC;QACD,IAAI,MAAM,CAAC,eAAe,IAAI,IAAI,EAAE,CAAC;YACjC,UAAU,CAAC,gBAAgB,GAAG,MAAM,CAAC,eAAe,CAAC;QACzD,CAAC;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;YACtB,OAAO,CAAC,IAAI,CAAC,yDAAyD,CAAC,CAAC;QAC5E,CAAC;QAED,QAAQ,MAAM,CAAC,cAAc,EAAE,CAAC;YAC5B,KAAK,KAAK,CAAC;YACX,KAAK,MAAM,CAAC;YACZ,KAAK,UAAU;gBACX,MAAM;YACV,KAAK,MAAM;gBACP,UAAU,CAAC,eAAe,GAAG,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC;gBACrD,MAAM;YACV,KAAK,eAAe;gBAChB,UAAU,CAAC,eAAe,GAAG,MAAM,CAAC,2BAA2B,CAAC;gBAChE,MAAM;QACd,CAAC;QAED,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC3E,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;QAE3B,IAAI,OAAO,GAAuB,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAW,EAAE,EAAE;YACvE,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;YAC1C,4EAA4E;YAC5E,4EAA4E;YAC5E,uFAAuF;YACvF,8DAA8D;YAC9D,MAAM,SAAS,GAAG,IAAI,CAAC,0BAA0B,CAAC,UAAU,CAAC,CAAC;YAE9D,MAAM,GAAG,GAAqB;gBAC1B,OAAO,EAAE;oBACL,IAAI,EAAE,eAAe,CAAC,SAAS;oBAC/B,OAAO,EAAE,SAAS,CAAC,OAAO;oBAC1B,QAAQ,EAAE,SAAS,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS;iBAC3D;gBACD,aAAa,EAAE,MAAM,CAAC,aAAa;gBACnC,KAAK,EAAE,MAAM,CAAC,KAAK;aACtB,CAAC;YACF,OAAO,GAAG,CAAC;QACf,CAAC,CAAC,CAAC;QAEH,kDAAkD;QAClD,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,YAAY,CAAC,KAAK,CAAC,aAAa,EAAE,YAAY,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;QAErG,oDAAoD;QACpD,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC;QACrC,IAAI,SAAS,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YACrC,uCAAuC;YACvC,KAAK,CAAC,SAAS,GAAG,SAAS,CAAC,UAAU,GAAG,IAAI,CAAC;QAClD,CAAC;QACD,IAAI,SAAS,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACtC,uCAAuC;YACvC,KAAK,CAAC,UAAU,GAAG,SAAS,CAAC,WAAW,GAAG,IAAI,CAAC;QACpD,CAAC;QACD,IAAI,SAAS,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YAC1C,uCAAuC;YACvC,KAAK,CAAC,cAAc,GAAG,SAAS,CAAC,eAAe,GAAG,IAAI,CAAC;QAC5D,CAAC;QAED,MAAM,MAAM,GAAG;YACX,OAAO,EAAE,IAAI;YACb,UAAU,EAAE,IAAI;YAChB,SAAS,EAAE,SAAS;YACpB,OAAO,EAAE,OAAO;YAChB,WAAW,EAAE,OAAO,CAAC,OAAO,EAAE,GAAG,SAAS,CAAC,OAAO,EAAE;YACpD,IAAI,EAAE;gBACF,OAAO,EAAE,OAAO;gBAChB,KAAK,EAAE,KAAK;aACf;YACD,YAAY,EAAE,EAAE;YAChB,SAAS,EAAE,IAAI;SACJ,CAAC;QAEhB,sCAAsC;QACtC,MAAM,CAAC,4BAA4B,GAAG;YAClC,QAAQ,EAAE,MAAM;YAChB,KAAK,EAAE,YAAY,CAAC,KAAK;YACzB,iBAAiB,EAAG,YAAoB,CAAC,kBAAkB;SAC9D,CAAC;QAEF,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,sBAAsB,CAAC,MAAkB;QACrD,kEAAkE;QAClE,IAAI,IAAI,CAAC,sBAAsB,EAAE,EAAE,CAAC;YAChC,IAAI,CAAC,6BAA6B,EAAE,CAAC;QACzC,CAAC;QAED,2EAA2E;QAC3E,MAAM,QAAQ,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAE7D,MAAM,UAAU,GAAwC;YACpD,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,QAAQ,EAAE,QAAQ;YAClB,UAAU,EAAE,MAAM,CAAC,eAAe;YAClC,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,MAAM,EAAE,IAAI;SACf,CAAC;QAEF,iDAAiD;QACjD,IAAI,CAAC,wBAAwB,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAElD,yCAAyC;QACzC,IAAI,MAAM,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;YACtB,UAAU,CAAC,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC;QACnC,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;YACtB,UAAU,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QAClC,CAAC;QACD,IAAI,MAAM,CAAC,aAAa,IAAI,IAAI,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClE,UAAU,CAAC,IAAI,GAAG,MAAM,CAAC,aAAa,CAAC;QAC3C,CAAC;QACD,IAAI,MAAM,CAAC,gBAAgB,IAAI,IAAI,EAAE,CAAC;YAClC,UAAU,CAAC,iBAAiB,GAAG,MAAM,CAAC,gBAAgB,CAAC;QAC3D,CAAC;QACD,IAAI,MAAM,CAAC,eAAe,IAAI,IAAI,EAAE,CAAC;YACjC,UAAU,CAAC,gBAAgB,GAAG,MAAM,CAAC,eAAe,CAAC;QACzD,CAAC;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;YACtB,OAAO,CAAC,IAAI,CAAC,yDAAyD,CAAC,CAAC;QAC5E,CAAC;QAED,mCAAmC;QACnC,QAAQ,MAAM,CAAC,cAAc,EAAE,CAAC;YAC5B,KAAK,MAAM;gBACP,UAAU,CAAC,eAAe,GAAG,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC;gBACrD,MAAM;YACV,KAAK,eAAe;gBAChB,UAAU,CAAC,eAAe,GAAG,MAAM,CAAC,2BAA2B,CAAC;gBAChE,MAAM;QACd,CAAC;QAED,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACO,qBAAqB,CAAC,KAAU;QAKtC,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,IAAI,YAAY,GAAG,SAAS,CAAC;QAE7B,IAAI,KAAK,EAAE,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7C,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAEhC,IAAI,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;gBACzB,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC;gBAExC,4DAA4D;gBAC5D,OAAO,GAAG,IAAI,CAAC,sBAAsB,EAAE;oBACnC,CAAC,CAAC,IAAI,CAAC,8BAA8B,CAAC,UAAU,CAAC;oBACjD,CAAC,CAAC,UAAU,CAAC;YACrB,CAAC;YAED,IAAI,MAAM,EAAE,aAAa,EAAE,CAAC;gBACxB,YAAY,GAAG,MAAM,CAAC,aAAa,CAAC;YACxC,CAAC;QACL,CAAC;QAED,iDAAiD;QACjD,OAAO;YACH,OAAO;YACP,YAAY;YACZ,KAAK,EAAE,IAAI;SACd,CAAC;IACN,CAAC;IAED;;OAEG;IACO,yBAAyB,CAC/B,kBAA6C,EAC7C,SAAiC,EACjC,KAA6B;QAE7B,qDAAqD;QACrD,IAAI,YAAY,GAAG,MAAM,CAAC;QAC1B,IAAI,SAAS,EAAE,OAAO,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC;YAC3F,YAAY,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;QACtD,CAAC;QAED,+DAA+D;QAC/D,6CAA6C;QAC7C,MAAM,YAAY,GAAG,KAAK,EAAE,YAAY,IAAI,CAAC,CAAC;QAC9C,MAAM,gBAAgB,GAAG,KAAK,EAAE,gBAAgB,IAAI,CAAC,CAAC;QAEtD,kDAAkD;QAClD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QAEvB,8DAA8D;QAC9D,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;QAE9C,yDAAyD;QACzD,MAAM,eAAe,GAAG,IAAI,CAAC,mBAAmB,EAAE,mBAAmB,CAAC,IAAI,EAAE,CAAC;QAE7E,qBAAqB;QACrB,MAAM,CAAC,IAAI,GAAG;YACV,OAAO,EAAE,CAAC;oBACN,OAAO,EAAE,IAAI,CAAC,oBAAoB,CAAC;wBAC/B,IAAI,EAAE,eAAe,CAAC,SAAS;wBAC/B,OAAO,EAAE,kBAAkB,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE;qBACxD,EAAE,eAAe,CAAC;oBACnB,aAAa,EAAE,YAAY;oBAC3B,KAAK,EAAE,CAAC;iBACX,CAAC;YACF,KAAK,EAAE,IAAI,UAAU,CAAC,YAAY,EAAE,gBAAgB,CAAC;SACxD,CAAC;QAEF,MAAM,CAAC,UAAU,GAAG,SAAS,CAAC;QAC9B,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC;QAC3B,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC;QAExB,OAAO,MAAM,CAAC;IAClB,CAAC;IAEM,KAAK,CAAC,aAAa,CAAC,MAAuB;QAC9C,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAC/C,CAAC;IAEM,KAAK,CAAC,YAAY,CAAC,MAAsB;QAC5C,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAC/C,CAAC;CACJ,CAAA;AA3YY,OAAO;IADnB,aAAa,CAAC,OAAO,EAAE,SAAS,CAAC;;GACrB,OAAO,CA2YnB"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@memberjunction/ai-groq",
|
|
3
|
-
"
|
|
3
|
+
"type": "module",
|
|
4
|
+
"version": "4.1.0",
|
|
4
5
|
"description": "MemberJunction Wrapper for Groq AI LPU inference engine",
|
|
5
6
|
"main": "dist/index.js",
|
|
6
7
|
"types": "dist/index.d.ts",
|
|
@@ -9,19 +10,19 @@
|
|
|
9
10
|
],
|
|
10
11
|
"scripts": {
|
|
11
12
|
"start": "ts-node-dev src/index.ts",
|
|
12
|
-
"build": "tsc",
|
|
13
|
+
"build": "tsc && tsc-alias -f",
|
|
13
14
|
"test": "echo \"Error: no test specified\" && exit 1"
|
|
14
15
|
},
|
|
15
16
|
"author": "MemberJunction.com",
|
|
16
17
|
"license": "ISC",
|
|
17
18
|
"devDependencies": {
|
|
18
19
|
"ts-node-dev": "^2.0.0",
|
|
19
|
-
"typescript": "^5.
|
|
20
|
+
"typescript": "^5.9.3"
|
|
20
21
|
},
|
|
21
22
|
"dependencies": {
|
|
22
|
-
"@memberjunction/ai": "
|
|
23
|
-
"@memberjunction/global": "
|
|
24
|
-
"groq-sdk": "^0.
|
|
23
|
+
"@memberjunction/ai": "4.1.0",
|
|
24
|
+
"@memberjunction/global": "4.1.0",
|
|
25
|
+
"groq-sdk": "^0.37.0"
|
|
25
26
|
},
|
|
26
27
|
"repository": {
|
|
27
28
|
"type": "git",
|
package/readme.md
CHANGED
|
@@ -1,18 +1,32 @@
|
|
|
1
1
|
# @memberjunction/ai-groq
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
MemberJunction AI provider for Groq's ultra-fast inference platform. This package extends the OpenAI provider to work with Groq's OpenAI-compatible API, providing access to LLM inference powered by Groq's custom Language Processing Units (LPUs) for industry-leading inference speed.
|
|
4
|
+
|
|
5
|
+
## Architecture
|
|
6
|
+
|
|
7
|
+
```mermaid
|
|
8
|
+
graph TD
|
|
9
|
+
A["GroqLLM<br/>(Provider)"] -->|extends| B["OpenAILLM<br/>(@memberjunction/ai-openai)"]
|
|
10
|
+
B -->|extends| C["BaseLLM<br/>(@memberjunction/ai)"]
|
|
11
|
+
A -->|overrides base URL| D["Groq API<br/>(api.groq.com/openai/v1)"]
|
|
12
|
+
D -->|runs on| E["Groq LPU<br/>Inference Engine"]
|
|
13
|
+
C -->|registered via| F["@RegisterClass"]
|
|
14
|
+
|
|
15
|
+
style A fill:#7c5295,stroke:#563a6b,color:#fff
|
|
16
|
+
style B fill:#2d6a9f,stroke:#1a4971,color:#fff
|
|
17
|
+
style C fill:#2d6a9f,stroke:#1a4971,color:#fff
|
|
18
|
+
style D fill:#2d8659,stroke:#1a5c3a,color:#fff
|
|
19
|
+
style E fill:#b8762f,stroke:#8a5722,color:#fff
|
|
20
|
+
style F fill:#b8762f,stroke:#8a5722,color:#fff
|
|
21
|
+
```
|
|
4
22
|
|
|
5
23
|
## Features
|
|
6
24
|
|
|
7
|
-
- **
|
|
8
|
-
- **
|
|
9
|
-
- **Streaming
|
|
10
|
-
- **
|
|
11
|
-
- **
|
|
12
|
-
- **Token Usage Tracking**: Track token consumption for monitoring
|
|
13
|
-
- **Chat Completion**: Interactive chat completions with various LLMs hosted on Groq
|
|
14
|
-
- **Model Support**: Compatible with Llama, Mixtral, Gemma, and other models hosted on Groq
|
|
15
|
-
- **Response Format Control**: Support for JSON, text, and model-specific response formats
|
|
25
|
+
- **Ultra-Fast Inference**: Groq's custom LPU hardware delivers extremely low latency
|
|
26
|
+
- **OpenAI Compatible**: Inherits all features from the OpenAI provider
|
|
27
|
+
- **Streaming**: Full streaming support for real-time responses
|
|
28
|
+
- **Thinking/Reasoning**: Thinking block extraction for reasoning models
|
|
29
|
+
- **Multiple Models**: Access to Llama, Mixtral, Gemma, and other open models optimized for Groq
|
|
16
30
|
|
|
17
31
|
## Installation
|
|
18
32
|
|
|
@@ -20,273 +34,56 @@ A comprehensive wrapper for Groq's LPU (Language Processing Unit) inference engi
|
|
|
20
34
|
npm install @memberjunction/ai-groq
|
|
21
35
|
```
|
|
22
36
|
|
|
23
|
-
## Requirements
|
|
24
|
-
|
|
25
|
-
- Node.js 16+
|
|
26
|
-
- A Groq API key
|
|
27
|
-
- MemberJunction Core libraries
|
|
28
|
-
|
|
29
37
|
## Usage
|
|
30
38
|
|
|
31
|
-
### Basic Setup
|
|
32
|
-
|
|
33
39
|
```typescript
|
|
34
40
|
import { GroqLLM } from '@memberjunction/ai-groq';
|
|
35
41
|
|
|
36
|
-
|
|
37
|
-
const groqLLM = new GroqLLM('your-groq-api-key');
|
|
38
|
-
```
|
|
39
|
-
|
|
40
|
-
### Chat Completion
|
|
41
|
-
|
|
42
|
-
```typescript
|
|
43
|
-
import { ChatParams } from '@memberjunction/ai';
|
|
44
|
-
|
|
45
|
-
// Create chat parameters
|
|
46
|
-
const chatParams: ChatParams = {
|
|
47
|
-
model: 'llama2-70b-4096', // or other models like 'mixtral-8x7b-32768'
|
|
48
|
-
messages: [
|
|
49
|
-
{ role: 'system', content: 'You are a helpful AI assistant.' },
|
|
50
|
-
{ role: 'user', content: 'Explain how LPUs differ from traditional GPUs for AI inference.' }
|
|
51
|
-
],
|
|
52
|
-
temperature: 0.7,
|
|
53
|
-
maxOutputTokens: 1000
|
|
54
|
-
};
|
|
55
|
-
|
|
56
|
-
// Get a response
|
|
57
|
-
try {
|
|
58
|
-
const response = await groqLLM.ChatCompletion(chatParams);
|
|
59
|
-
if (response.success) {
|
|
60
|
-
console.log('Response:', response.data.choices[0].message.content);
|
|
61
|
-
console.log('Token Usage:', response.data.usage);
|
|
62
|
-
console.log('Time Elapsed (ms):', response.timeElapsed);
|
|
63
|
-
} else {
|
|
64
|
-
console.error('Error:', response.errorMessage);
|
|
65
|
-
}
|
|
66
|
-
} catch (error) {
|
|
67
|
-
console.error('Exception:', error);
|
|
68
|
-
}
|
|
69
|
-
```
|
|
70
|
-
|
|
71
|
-
### Streaming Responses
|
|
72
|
-
|
|
73
|
-
```typescript
|
|
74
|
-
import { ChatParams, ChatResult } from '@memberjunction/ai';
|
|
75
|
-
|
|
76
|
-
// Enable streaming in the chat parameters
|
|
77
|
-
const streamingParams: ChatParams = {
|
|
78
|
-
model: 'llama3-70b-8192',
|
|
79
|
-
messages: [
|
|
80
|
-
{ role: 'user', content: 'Write a short story about AI.' }
|
|
81
|
-
],
|
|
82
|
-
stream: true,
|
|
83
|
-
onStream: (content: string) => {
|
|
84
|
-
// Handle each chunk of streamed content
|
|
85
|
-
process.stdout.write(content);
|
|
86
|
-
},
|
|
87
|
-
maxOutputTokens: 2000
|
|
88
|
-
};
|
|
89
|
-
|
|
90
|
-
// The response will stream to the onStream callback
|
|
91
|
-
const response = await groqLLM.ChatCompletion(streamingParams);
|
|
92
|
-
console.log('\n\nFinal response:', response.data.choices[0].message.content);
|
|
93
|
-
```
|
|
94
|
-
|
|
95
|
-
### Response Format Control
|
|
96
|
-
|
|
97
|
-
```typescript
|
|
98
|
-
// Request JSON formatted response
|
|
99
|
-
const jsonParams: ChatParams = {
|
|
100
|
-
model: 'mixtral-8x7b-32768',
|
|
101
|
-
messages: [
|
|
102
|
-
{ role: 'system', content: 'You are a helpful assistant that responds in JSON format.' },
|
|
103
|
-
{ role: 'user', content: 'List 3 benefits of using Groq in JSON format with keys: benefit, description' }
|
|
104
|
-
],
|
|
105
|
-
responseFormat: 'JSON',
|
|
106
|
-
maxOutputTokens: 1000
|
|
107
|
-
};
|
|
42
|
+
const llm = new GroqLLM('your-groq-api-key');
|
|
108
43
|
|
|
109
|
-
const
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
const groqClient = groqLLM.GroqClient;
|
|
118
|
-
// or use the alias
|
|
119
|
-
const client = groqLLM.client;
|
|
120
|
-
|
|
121
|
-
// Use the client directly if needed
|
|
122
|
-
const customResponse = await groqClient.chat.completions.create({
|
|
123
|
-
model: 'mixtral-8x7b-32768',
|
|
124
|
-
messages: [{ role: 'user', content: 'Hello!' }],
|
|
125
|
-
max_tokens: 500
|
|
44
|
+
const result = await llm.ChatCompletion({
|
|
45
|
+
model: 'llama-3.1-70b-versatile',
|
|
46
|
+
messages: [
|
|
47
|
+
{ role: 'system', content: 'You are a helpful assistant.' },
|
|
48
|
+
{ role: 'user', content: 'Explain neural networks.' }
|
|
49
|
+
],
|
|
50
|
+
temperature: 0.7,
|
|
51
|
+
maxOutputTokens: 1000
|
|
126
52
|
});
|
|
127
|
-
```
|
|
128
|
-
|
|
129
|
-
## Supported Models
|
|
130
|
-
|
|
131
|
-
Groq provides access to various open models with optimized inference:
|
|
132
|
-
|
|
133
|
-
- **Llama Models**:
|
|
134
|
-
- `llama3-70b-8192` - Llama 3 70B with 8K context
|
|
135
|
-
- `llama3-8b-8192` - Llama 3 8B with 8K context
|
|
136
|
-
- `llama2-70b-4096` - Llama 2 70B with 4K context
|
|
137
|
-
|
|
138
|
-
- **Mixtral Models**:
|
|
139
|
-
- `mixtral-8x7b-32768` - Mixtral 8x7B with 32K context
|
|
140
|
-
|
|
141
|
-
- **Gemma Models**:
|
|
142
|
-
- `gemma-7b-it` - Gemma 7B Instruct
|
|
143
|
-
- `gemma2-9b-it` - Gemma 2 9B Instruct
|
|
144
|
-
|
|
145
|
-
Check the [Groq documentation](https://console.groq.com/docs/models) for the latest list of supported models and their capabilities.
|
|
146
|
-
|
|
147
|
-
## API Reference
|
|
148
|
-
|
|
149
|
-
### GroqLLM Class
|
|
150
|
-
|
|
151
|
-
A class that extends BaseLLM to provide Groq-specific functionality.
|
|
152
|
-
|
|
153
|
-
#### Constructor
|
|
154
|
-
|
|
155
|
-
```typescript
|
|
156
|
-
new GroqLLM(apiKey: string)
|
|
157
|
-
```
|
|
158
|
-
|
|
159
|
-
#### Properties
|
|
160
|
-
|
|
161
|
-
- `GroqClient`: (read-only) Returns the underlying Groq client instance
|
|
162
|
-
- `client`: (read-only) Alias for GroqClient
|
|
163
|
-
- `SupportsStreaming`: (read-only) Returns `true` - Groq supports streaming responses
|
|
164
|
-
|
|
165
|
-
#### Methods
|
|
166
53
|
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
ChatCompletion(params: ChatParams): Promise<ChatResult>
|
|
170
|
-
```
|
|
171
|
-
Perform a chat completion with support for both streaming and non-streaming responses.
|
|
172
|
-
|
|
173
|
-
##### SummarizeText
|
|
174
|
-
```typescript
|
|
175
|
-
SummarizeText(params: SummarizeParams): Promise<SummarizeResult>
|
|
176
|
-
```
|
|
177
|
-
*Note: Not yet implemented*
|
|
178
|
-
|
|
179
|
-
##### ClassifyText
|
|
180
|
-
```typescript
|
|
181
|
-
ClassifyText(params: ClassifyParams): Promise<ClassifyResult>
|
|
182
|
-
```
|
|
183
|
-
*Note: Not yet implemented*
|
|
184
|
-
|
|
185
|
-
## Performance Considerations
|
|
186
|
-
|
|
187
|
-
Groq is known for its extremely fast inference times:
|
|
188
|
-
|
|
189
|
-
- Response generation is typically 5-10x faster than traditional GPU-based inference
|
|
190
|
-
- Lower latency means better interactive experiences
|
|
191
|
-
- Benchmark different models to find the best performance/quality balance for your use case
|
|
192
|
-
|
|
193
|
-
## Error Handling
|
|
194
|
-
|
|
195
|
-
The wrapper provides detailed error information:
|
|
196
|
-
|
|
197
|
-
```typescript
|
|
198
|
-
try {
|
|
199
|
-
const response = await groqLLM.ChatCompletion(params);
|
|
200
|
-
if (!response.success) {
|
|
201
|
-
console.error('Error:', response.errorMessage);
|
|
202
|
-
console.error('Status:', response.statusText);
|
|
203
|
-
console.error('Time Elapsed:', response.timeElapsed, 'ms');
|
|
204
|
-
}
|
|
205
|
-
} catch (error) {
|
|
206
|
-
console.error('Exception occurred:', error);
|
|
54
|
+
if (result.success) {
|
|
55
|
+
console.log(result.data.choices[0].message.content);
|
|
207
56
|
}
|
|
208
57
|
```
|
|
209
58
|
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
This package seamlessly integrates with the MemberJunction AI framework:
|
|
59
|
+
### Streaming
|
|
213
60
|
|
|
214
61
|
```typescript
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
## Advanced Features
|
|
225
|
-
|
|
226
|
-
### Effort Level Support
|
|
227
|
-
|
|
228
|
-
For models that support reasoning effort levels (experimental):
|
|
229
|
-
|
|
230
|
-
```typescript
|
|
231
|
-
const params: ChatParams = {
|
|
232
|
-
model: 'llama3-70b-8192',
|
|
233
|
-
messages: [{ role: 'user', content: 'Solve this complex problem...' }],
|
|
234
|
-
effortLevel: 'high', // Experimental feature
|
|
235
|
-
maxOutputTokens: 2000
|
|
236
|
-
};
|
|
237
|
-
```
|
|
238
|
-
|
|
239
|
-
### Handling Groq-Specific Requirements
|
|
240
|
-
|
|
241
|
-
The wrapper automatically handles Groq's requirement that the last message must be from a user. If your message chain ends with an assistant message, the wrapper will automatically append a dummy user message to satisfy this requirement.
|
|
242
|
-
|
|
243
|
-
## Dependencies
|
|
244
|
-
|
|
245
|
-
- `groq-sdk` (0.21.0): Official Groq SDK
|
|
246
|
-
- `@memberjunction/ai` (2.43.0): MemberJunction AI core framework
|
|
247
|
-
- `@memberjunction/global` (2.43.0): MemberJunction global utilities
|
|
248
|
-
|
|
249
|
-
## Development
|
|
250
|
-
|
|
251
|
-
### Building
|
|
252
|
-
|
|
253
|
-
```bash
|
|
254
|
-
npm run build
|
|
255
|
-
```
|
|
256
|
-
|
|
257
|
-
### Development Mode
|
|
258
|
-
|
|
259
|
-
```bash
|
|
260
|
-
npm start
|
|
62
|
+
const result = await llm.ChatCompletion({
|
|
63
|
+
model: 'llama-3.1-8b-instant',
|
|
64
|
+
messages: [{ role: 'user', content: 'Write a haiku about speed.' }],
|
|
65
|
+
streaming: true,
|
|
66
|
+
streamingCallbacks: {
|
|
67
|
+
OnContent: (content) => process.stdout.write(content),
|
|
68
|
+
OnComplete: () => console.log('\nDone!')
|
|
69
|
+
}
|
|
70
|
+
});
|
|
261
71
|
```
|
|
262
72
|
|
|
263
|
-
##
|
|
73
|
+
## How It Works
|
|
264
74
|
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
1. Follow the MemberJunction coding standards
|
|
268
|
-
2. Ensure all TypeScript types are properly defined
|
|
269
|
-
3. Update tests when adding new features
|
|
270
|
-
4. Document any Groq-specific behaviors or limitations
|
|
75
|
+
`GroqLLM` is a thin subclass of `OpenAILLM` that redirects API calls to Groq's endpoint at `https://api.groq.com/openai/v1`. Since Groq implements an OpenAI-compatible API, all chat, streaming, and parameter handling logic is inherited from the OpenAI provider.
|
|
271
76
|
|
|
272
77
|
## Supported Parameters
|
|
273
78
|
|
|
274
|
-
|
|
79
|
+
All parameters supported by the OpenAI provider are available, including `temperature`, `maxOutputTokens`, `topP`, `frequencyPenalty`, `presencePenalty`, `seed`, `stopSequences`, and `responseFormat`.
|
|
275
80
|
|
|
276
|
-
|
|
277
|
-
- `temperature` - Controls randomness in the output (0.0-2.0)
|
|
278
|
-
- `maxOutputTokens` - Maximum number of tokens to generate
|
|
279
|
-
- `topP` - Nucleus sampling threshold (0.0-1.0)
|
|
280
|
-
- `seed` - For deterministic outputs
|
|
281
|
-
- `stopSequences` - Array of sequences where the API will stop generating
|
|
282
|
-
- `frequencyPenalty` - Reduces repetition of token sequences (-2.0 to 2.0)
|
|
283
|
-
- `presencePenalty` - Reduces repetition of specific tokens (-2.0 to 2.0)
|
|
284
|
-
- `responseFormat` - Output format (Text, JSON, etc.)
|
|
81
|
+
## Class Registration
|
|
285
82
|
|
|
286
|
-
|
|
287
|
-
- `topK` - Not available in Groq API
|
|
288
|
-
- `minP` - Not available in Groq API
|
|
83
|
+
Registered as `GroqLLM` via `@RegisterClass(BaseLLM, 'GroqLLM')`.
|
|
289
84
|
|
|
290
|
-
##
|
|
85
|
+
## Dependencies
|
|
291
86
|
|
|
292
|
-
|
|
87
|
+
- `@memberjunction/ai` - Core AI abstractions
|
|
88
|
+
- `@memberjunction/ai-openai` - OpenAI provider (parent class)
|
|
89
|
+
- `@memberjunction/global` - Class registration
|