@rimori/client 2.5.14-next.2 → 2.5.14-next.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/modules.d.ts +1 -0
- package/dist/modules.js +1 -0
- package/dist/plugin/RimoriClient.js +5 -1
- package/dist/plugin/TTS/ChunkedAudioPlayer.d.ts +29 -0
- package/dist/plugin/TTS/ChunkedAudioPlayer.js +205 -0
- package/dist/plugin/TTS/MessageSender.d.ts +21 -0
- package/dist/plugin/TTS/MessageSender.js +95 -0
- package/dist/plugin/module/AIModule.d.ts +10 -0
- package/dist/plugin/module/AIModule.js +43 -1
- package/dist/plugin/module/EventModule.d.ts +6 -2
- package/dist/plugin/module/EventModule.js +31 -2
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -17,3 +17,4 @@ export type { TriggerAction } from './plugin/module/ExerciseModule';
|
|
|
17
17
|
export type { MacroAccomplishmentPayload, MicroAccomplishmentPayload } from './controller/AccomplishmentController';
|
|
18
18
|
export type { EventBusMessage } from './fromRimori/EventBus';
|
|
19
19
|
export type { Theme, ApplicationMode } from './plugin/module/PluginModule';
|
|
20
|
+
export * from './plugin/TTS/MessageSender';
|
package/dist/index.js
CHANGED
|
@@ -9,3 +9,4 @@ export * from './plugin/StandaloneClient';
|
|
|
9
9
|
export { setupWorker } from './worker/WorkerSetup';
|
|
10
10
|
export { AudioController } from './controller/AudioController';
|
|
11
11
|
export { Translator } from './controller/TranslationController';
|
|
12
|
+
export * from './plugin/TTS/MessageSender';
|
package/dist/modules.d.ts
CHANGED
package/dist/modules.js
CHANGED
|
@@ -15,6 +15,7 @@ import { DbModule } from './module/DbModule';
|
|
|
15
15
|
import { EventModule } from './module/EventModule';
|
|
16
16
|
import { AIModule } from './module/AIModule';
|
|
17
17
|
import { ExerciseModule } from './module/ExerciseModule';
|
|
18
|
+
import { EventBus } from '../fromRimori/EventBus';
|
|
18
19
|
export class RimoriClient {
|
|
19
20
|
constructor(controller, supabase, info) {
|
|
20
21
|
this.navigation = {
|
|
@@ -30,7 +31,10 @@ export class RimoriClient {
|
|
|
30
31
|
this.rimoriInfo = info;
|
|
31
32
|
this.sharedContent = new SharedContentController(supabase, this);
|
|
32
33
|
this.ai = new AIModule(info.backendUrl, () => this.rimoriInfo.token);
|
|
33
|
-
this.
|
|
34
|
+
this.ai.setOnRateLimited((exercisesRemaining) => {
|
|
35
|
+
EventBus.emit(info.pluginId, 'global.quota.triggerExceeded', { exercises_remaining: exercisesRemaining });
|
|
36
|
+
});
|
|
37
|
+
this.event = new EventModule(info.pluginId, info.backendUrl, () => this.rimoriInfo.token, this.ai);
|
|
34
38
|
this.db = new DbModule(supabase, controller, info);
|
|
35
39
|
this.plugin = new PluginModule(supabase, controller, info, this.ai);
|
|
36
40
|
this.exercise = new ExerciseModule(supabase, controller, info, this.event);
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
export declare class ChunkedAudioPlayer {
|
|
2
|
+
private audioContext;
|
|
3
|
+
private chunkQueue;
|
|
4
|
+
private isPlaying;
|
|
5
|
+
private analyser;
|
|
6
|
+
private dataArray;
|
|
7
|
+
private shouldMonitorLoudness;
|
|
8
|
+
private isMonitoring;
|
|
9
|
+
private handle;
|
|
10
|
+
private volume;
|
|
11
|
+
private loudnessCallback;
|
|
12
|
+
private currentIndex;
|
|
13
|
+
private startedPlaying;
|
|
14
|
+
private onEndOfSpeech;
|
|
15
|
+
private readonly backgroundNoiseLevel;
|
|
16
|
+
constructor();
|
|
17
|
+
private init;
|
|
18
|
+
setOnLoudnessChange(callback: (value: number) => void): void;
|
|
19
|
+
setVolume(volume: number): void;
|
|
20
|
+
addChunk(chunk: ArrayBuffer, position: number): Promise<void>;
|
|
21
|
+
private playChunks;
|
|
22
|
+
stopPlayback(): void;
|
|
23
|
+
cleanup(): void;
|
|
24
|
+
private playChunk;
|
|
25
|
+
playAgain(): Promise<void>;
|
|
26
|
+
private monitorLoudness;
|
|
27
|
+
reset(): void;
|
|
28
|
+
setOnEndOfSpeech(callback: () => void): void;
|
|
29
|
+
}
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
export class ChunkedAudioPlayer {
|
|
11
|
+
constructor() {
|
|
12
|
+
this.chunkQueue = [];
|
|
13
|
+
this.isPlaying = false;
|
|
14
|
+
this.shouldMonitorLoudness = true;
|
|
15
|
+
this.isMonitoring = false;
|
|
16
|
+
this.handle = 0;
|
|
17
|
+
this.volume = 1.0;
|
|
18
|
+
this.loudnessCallback = () => { };
|
|
19
|
+
this.currentIndex = 0;
|
|
20
|
+
this.startedPlaying = false;
|
|
21
|
+
this.onEndOfSpeech = () => { };
|
|
22
|
+
this.backgroundNoiseLevel = 30; // Background noise level that should be treated as baseline (0)
|
|
23
|
+
this.init();
|
|
24
|
+
}
|
|
25
|
+
init() {
|
|
26
|
+
this.audioContext = new AudioContext();
|
|
27
|
+
this.analyser = this.audioContext.createAnalyser();
|
|
28
|
+
this.analyser.fftSize = 256; // Set the FFT size (smaller values provide faster updates, larger ones give better resolution)
|
|
29
|
+
const bufferLength = this.analyser.frequencyBinCount;
|
|
30
|
+
this.dataArray = new Uint8Array(bufferLength); // Array to hold frequency data
|
|
31
|
+
}
|
|
32
|
+
setOnLoudnessChange(callback) {
|
|
33
|
+
this.loudnessCallback = callback;
|
|
34
|
+
}
|
|
35
|
+
setVolume(volume) {
|
|
36
|
+
this.volume = volume;
|
|
37
|
+
}
|
|
38
|
+
addChunk(chunk, position) {
|
|
39
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
40
|
+
console.log('Adding chunk', position, chunk);
|
|
41
|
+
this.chunkQueue[position] = chunk;
|
|
42
|
+
// console.log("received chunk", {
|
|
43
|
+
// chunkQueue: this.chunkQueue.length,
|
|
44
|
+
// isPlaying: this.isPlaying,
|
|
45
|
+
// })
|
|
46
|
+
if (position === 0 && !this.startedPlaying) {
|
|
47
|
+
this.startedPlaying = true;
|
|
48
|
+
this.playChunks();
|
|
49
|
+
}
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
playChunks() {
|
|
53
|
+
// console.log({ isPlaying: this.isPlaying });
|
|
54
|
+
if (this.isPlaying)
|
|
55
|
+
return;
|
|
56
|
+
if (!this.chunkQueue[this.currentIndex]) {
|
|
57
|
+
// wait until the correct chunk arrives
|
|
58
|
+
setTimeout(() => this.playChunks(), 10);
|
|
59
|
+
}
|
|
60
|
+
this.isPlaying = true;
|
|
61
|
+
this.playChunk(this.chunkQueue[this.currentIndex]).then(() => {
|
|
62
|
+
this.isPlaying = false;
|
|
63
|
+
this.currentIndex++;
|
|
64
|
+
if (this.chunkQueue[this.currentIndex]) {
|
|
65
|
+
this.shouldMonitorLoudness = true;
|
|
66
|
+
this.playChunks();
|
|
67
|
+
}
|
|
68
|
+
else {
|
|
69
|
+
// console.log('Playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
|
|
70
|
+
setTimeout(() => {
|
|
71
|
+
// console.log('Check again if really playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
|
|
72
|
+
if (this.chunkQueue.length > this.currentIndex) {
|
|
73
|
+
this.playChunks();
|
|
74
|
+
}
|
|
75
|
+
else {
|
|
76
|
+
this.startedPlaying = false;
|
|
77
|
+
this.shouldMonitorLoudness = false;
|
|
78
|
+
}
|
|
79
|
+
}, 1000);
|
|
80
|
+
}
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
stopPlayback() {
|
|
84
|
+
// console.log('Stopping playback');
|
|
85
|
+
// Implement logic to stop the current playback
|
|
86
|
+
this.isPlaying = false;
|
|
87
|
+
this.chunkQueue = [];
|
|
88
|
+
this.startedPlaying = false;
|
|
89
|
+
this.shouldMonitorLoudness = false;
|
|
90
|
+
cancelAnimationFrame(this.handle);
|
|
91
|
+
}
|
|
92
|
+
cleanup() {
|
|
93
|
+
// Stop playback first
|
|
94
|
+
this.stopPlayback();
|
|
95
|
+
// Close AudioContext to free resources
|
|
96
|
+
if (this.audioContext && this.audioContext.state !== 'closed') {
|
|
97
|
+
this.audioContext.close().catch((e) => {
|
|
98
|
+
console.warn('Error closing AudioContext:', e);
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
playChunk(chunk) {
|
|
103
|
+
// console.log({queue: this.chunkQueue})
|
|
104
|
+
if (!chunk) {
|
|
105
|
+
return Promise.resolve();
|
|
106
|
+
}
|
|
107
|
+
// console.log('Playing chunk', chunk);
|
|
108
|
+
return new Promise((resolve) => {
|
|
109
|
+
const source = this.audioContext.createBufferSource();
|
|
110
|
+
this.audioContext.decodeAudioData(chunk.slice(0)).then((audioBuffer) => {
|
|
111
|
+
source.buffer = audioBuffer;
|
|
112
|
+
// Create a GainNode for volume control
|
|
113
|
+
const gainNode = this.audioContext.createGain();
|
|
114
|
+
gainNode.gain.value = this.volume;
|
|
115
|
+
// Connect the source to the GainNode, then to the analyser node, then to the destination (speakers)
|
|
116
|
+
source.connect(gainNode);
|
|
117
|
+
gainNode.connect(this.analyser);
|
|
118
|
+
this.analyser.connect(this.audioContext.destination);
|
|
119
|
+
source.start(0);
|
|
120
|
+
// console.log('Playing chunk', this.currentIndex);
|
|
121
|
+
gainNode.gain.value = this.volume;
|
|
122
|
+
source.onended = () => {
|
|
123
|
+
// console.log('Chunk ended');
|
|
124
|
+
resolve();
|
|
125
|
+
};
|
|
126
|
+
// Start monitoring loudness only once
|
|
127
|
+
if (!this.isMonitoring) {
|
|
128
|
+
this.isMonitoring = true;
|
|
129
|
+
this.shouldMonitorLoudness = true;
|
|
130
|
+
this.monitorLoudness();
|
|
131
|
+
}
|
|
132
|
+
});
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
playAgain() {
|
|
136
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
137
|
+
console.log('Playing again');
|
|
138
|
+
if (this.chunkQueue.length > 0 && !this.isPlaying) {
|
|
139
|
+
this.playChunks();
|
|
140
|
+
}
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
monitorLoudness() {
|
|
144
|
+
// Stop monitoring when the flag is false
|
|
145
|
+
if (!this.shouldMonitorLoudness) {
|
|
146
|
+
// console.log('Loudness monitoring stopped.');
|
|
147
|
+
cancelAnimationFrame(this.handle);
|
|
148
|
+
this.loudnessCallback(0);
|
|
149
|
+
this.onEndOfSpeech();
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
// Get the time domain data from the analyser (this is a snapshot of the waveform)
|
|
153
|
+
this.analyser.getByteTimeDomainData(this.dataArray);
|
|
154
|
+
// Calculate the RMS (root mean square) of the waveform values to get the perceived loudness
|
|
155
|
+
let sum = 0;
|
|
156
|
+
for (let i = 0; i < this.dataArray.length; i++) {
|
|
157
|
+
const value = this.dataArray[i] / 128.0 - 1.0; // Normalize between -1 and 1
|
|
158
|
+
sum += value * value;
|
|
159
|
+
}
|
|
160
|
+
const rms = Math.sqrt(sum / this.dataArray.length);
|
|
161
|
+
// Handle the case where RMS is 0 to avoid log10(0)
|
|
162
|
+
if (rms === 0) {
|
|
163
|
+
// console.log('Current loudness: Silent');
|
|
164
|
+
}
|
|
165
|
+
else {
|
|
166
|
+
let loudnessInDb = 20 * Math.log10(rms); // Convert to dB
|
|
167
|
+
// console.log('Current loudness:' + loudnessInDb);
|
|
168
|
+
const minDb = -57;
|
|
169
|
+
const maxDb = -15;
|
|
170
|
+
if (loudnessInDb < minDb) {
|
|
171
|
+
loudnessInDb = minDb;
|
|
172
|
+
}
|
|
173
|
+
if (loudnessInDb > maxDb) {
|
|
174
|
+
loudnessInDb = maxDb;
|
|
175
|
+
}
|
|
176
|
+
let loudnessScale = ((loudnessInDb - minDb) / (maxDb - minDb)) * 100;
|
|
177
|
+
// Adjust loudness: shift zero level up by background noise amount
|
|
178
|
+
// Values below background noise level are set to 0
|
|
179
|
+
// Values above are remapped to 0-100 scale
|
|
180
|
+
if (loudnessScale < this.backgroundNoiseLevel) {
|
|
181
|
+
loudnessScale = 0;
|
|
182
|
+
}
|
|
183
|
+
else {
|
|
184
|
+
// Remap from [backgroundNoiseLevel, 100] to [0, 100]
|
|
185
|
+
loudnessScale = ((loudnessScale - this.backgroundNoiseLevel) / (100 - this.backgroundNoiseLevel)) * 100;
|
|
186
|
+
}
|
|
187
|
+
this.loudnessCallback(Math.round(loudnessScale));
|
|
188
|
+
}
|
|
189
|
+
// Call this method again at regular intervals if you want continuous loudness monitoring
|
|
190
|
+
this.handle = requestAnimationFrame(() => this.monitorLoudness());
|
|
191
|
+
}
|
|
192
|
+
reset() {
|
|
193
|
+
// console.log('Resetting player');
|
|
194
|
+
this.stopPlayback();
|
|
195
|
+
this.currentIndex = 0;
|
|
196
|
+
this.shouldMonitorLoudness = true;
|
|
197
|
+
//reset to the beginning when the class gets initialized
|
|
198
|
+
this.isMonitoring = false;
|
|
199
|
+
this.isPlaying = false;
|
|
200
|
+
this.init();
|
|
201
|
+
}
|
|
202
|
+
setOnEndOfSpeech(callback) {
|
|
203
|
+
this.onEndOfSpeech = callback;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
type VoiceBackend = (text: string, voice?: string, speed?: number, language?: string, cache?: boolean) => Promise<Blob>;
|
|
2
|
+
export declare class MessageSender {
|
|
3
|
+
private player;
|
|
4
|
+
private fetchedSentences;
|
|
5
|
+
private lastLoading;
|
|
6
|
+
private voice;
|
|
7
|
+
private voiceBackend;
|
|
8
|
+
private cache;
|
|
9
|
+
constructor(voiceBackend: VoiceBackend, voice: string, cache?: boolean);
|
|
10
|
+
private getCompletedSentences;
|
|
11
|
+
handleNewText(currentText: string | undefined, isLoading: boolean): Promise<void>;
|
|
12
|
+
private generateSpeech;
|
|
13
|
+
play(): void;
|
|
14
|
+
stop(): void;
|
|
15
|
+
cleanup(): void;
|
|
16
|
+
private reset;
|
|
17
|
+
setVolume(volume: number): void;
|
|
18
|
+
setOnLoudnessChange(callback: (value: number) => void): void;
|
|
19
|
+
setOnEndOfSpeech(callback: () => void): void;
|
|
20
|
+
}
|
|
21
|
+
export {};
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
import { ChunkedAudioPlayer } from './ChunkedAudioPlayer';
|
|
11
|
+
export class MessageSender {
|
|
12
|
+
constructor(voiceBackend, voice, cache = false) {
|
|
13
|
+
this.player = new ChunkedAudioPlayer();
|
|
14
|
+
this.fetchedSentences = new Set();
|
|
15
|
+
this.lastLoading = false;
|
|
16
|
+
if ((voice === null || voice === void 0 ? void 0 : voice.split('_').length) !== 2) {
|
|
17
|
+
throw new Error("Invalid voice id format '" + voice + "'. Voice id needs to look like <provider>_<voice_id>");
|
|
18
|
+
}
|
|
19
|
+
this.voiceBackend = voiceBackend;
|
|
20
|
+
this.voice = voice;
|
|
21
|
+
this.cache = cache;
|
|
22
|
+
}
|
|
23
|
+
getCompletedSentences(currentText, isLoading) {
|
|
24
|
+
// Split the text based on the following characters: .?!
|
|
25
|
+
// Only split on : when followed by a space
|
|
26
|
+
const pattern = /(.+?[.?!]|.+?:\s+|.+?\n+)/g;
|
|
27
|
+
const result = [];
|
|
28
|
+
let match;
|
|
29
|
+
while ((match = pattern.exec(currentText)) !== null) {
|
|
30
|
+
const sentence = match[0].trim();
|
|
31
|
+
if (sentence.length > 0) {
|
|
32
|
+
result.push(sentence);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
if (!isLoading) {
|
|
36
|
+
const lastFullSentence = result[result.length - 1];
|
|
37
|
+
const leftoverIndex = currentText.lastIndexOf(lastFullSentence) + lastFullSentence.length;
|
|
38
|
+
if (leftoverIndex < currentText.length) {
|
|
39
|
+
result.push(currentText.slice(leftoverIndex).trim());
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
return result;
|
|
43
|
+
}
|
|
44
|
+
handleNewText(currentText, isLoading) {
|
|
45
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
46
|
+
if (!this.lastLoading && isLoading) {
|
|
47
|
+
this.reset();
|
|
48
|
+
}
|
|
49
|
+
this.lastLoading = isLoading;
|
|
50
|
+
if (!currentText) {
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
const sentences = this.getCompletedSentences(currentText, isLoading);
|
|
54
|
+
for (let i = 0; i < sentences.length; i++) {
|
|
55
|
+
const sentence = sentences[i];
|
|
56
|
+
if (!this.fetchedSentences.has(sentence)) {
|
|
57
|
+
this.fetchedSentences.add(sentence);
|
|
58
|
+
const audioData = yield this.generateSpeech(sentence);
|
|
59
|
+
yield this.player.addChunk(audioData, i);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
generateSpeech(sentence) {
|
|
65
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
66
|
+
const blob = yield this.voiceBackend(sentence, this.voice, 1.0, undefined, this.cache);
|
|
67
|
+
return yield blob.arrayBuffer();
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
play() {
|
|
71
|
+
this.player.playAgain();
|
|
72
|
+
}
|
|
73
|
+
stop() {
|
|
74
|
+
this.player.stopPlayback();
|
|
75
|
+
}
|
|
76
|
+
cleanup() {
|
|
77
|
+
this.player.cleanup();
|
|
78
|
+
}
|
|
79
|
+
reset() {
|
|
80
|
+
this.stop();
|
|
81
|
+
this.fetchedSentences.clear();
|
|
82
|
+
this.player.reset();
|
|
83
|
+
}
|
|
84
|
+
setVolume(volume) {
|
|
85
|
+
this.player.setVolume(volume);
|
|
86
|
+
}
|
|
87
|
+
setOnLoudnessChange(callback) {
|
|
88
|
+
this.player.setOnLoudnessChange((loudness) => {
|
|
89
|
+
callback(loudness);
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
setOnEndOfSpeech(callback) {
|
|
93
|
+
this.player.setOnEndOfSpeech(callback);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
@@ -58,7 +58,17 @@ export interface ObjectRequest {
|
|
|
58
58
|
export declare class AIModule {
|
|
59
59
|
private getToken;
|
|
60
60
|
private backendUrl;
|
|
61
|
+
private sessionTokenId;
|
|
62
|
+
private onRateLimitedCb?;
|
|
61
63
|
constructor(backendUrl: string, getToken: () => string);
|
|
64
|
+
/** Returns the current exercise session token ID (null if no active session). */
|
|
65
|
+
getSessionTokenId(): string | null;
|
|
66
|
+
/** Sets the session token ID (used by workers inheriting a session from the plugin). */
|
|
67
|
+
setSessionToken(id: string): void;
|
|
68
|
+
/** Clears the stored session token (called after macro accomplishment). */
|
|
69
|
+
clearSessionToken(): void;
|
|
70
|
+
/** Registers a callback invoked whenever a 429 rate-limit response is received. */
|
|
71
|
+
setOnRateLimited(cb: (exercisesRemaining: number) => void): void;
|
|
62
72
|
/**
|
|
63
73
|
* Generate text from messages using AI.
|
|
64
74
|
* @param messages The messages to generate text from.
|
|
@@ -13,9 +13,26 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
13
13
|
*/
|
|
14
14
|
export class AIModule {
|
|
15
15
|
constructor(backendUrl, getToken) {
|
|
16
|
+
this.sessionTokenId = null;
|
|
16
17
|
this.backendUrl = backendUrl;
|
|
17
18
|
this.getToken = getToken;
|
|
18
19
|
}
|
|
20
|
+
/** Returns the current exercise session token ID (null if no active session). */
|
|
21
|
+
getSessionTokenId() {
|
|
22
|
+
return this.sessionTokenId;
|
|
23
|
+
}
|
|
24
|
+
/** Sets the session token ID (used by workers inheriting a session from the plugin). */
|
|
25
|
+
setSessionToken(id) {
|
|
26
|
+
this.sessionTokenId = id;
|
|
27
|
+
}
|
|
28
|
+
/** Clears the stored session token (called after macro accomplishment). */
|
|
29
|
+
clearSessionToken() {
|
|
30
|
+
this.sessionTokenId = null;
|
|
31
|
+
}
|
|
32
|
+
/** Registers a callback invoked whenever a 429 rate-limit response is received. */
|
|
33
|
+
setOnRateLimited(cb) {
|
|
34
|
+
this.onRateLimitedCb = cb;
|
|
35
|
+
}
|
|
19
36
|
/**
|
|
20
37
|
* Generate text from messages using AI.
|
|
21
38
|
* @param messages The messages to generate text from.
|
|
@@ -78,13 +95,14 @@ export class AIModule {
|
|
|
78
95
|
*/
|
|
79
96
|
getVoice(text_1) {
|
|
80
97
|
return __awaiter(this, arguments, void 0, function* (text, voice = 'alloy', speed = 1, language, cache = false) {
|
|
98
|
+
var _a;
|
|
81
99
|
return yield fetch(`${this.backendUrl}/voice/tts`, {
|
|
82
100
|
method: 'POST',
|
|
83
101
|
headers: {
|
|
84
102
|
'Content-Type': 'application/json',
|
|
85
103
|
Authorization: `Bearer ${this.getToken()}`,
|
|
86
104
|
},
|
|
87
|
-
body: JSON.stringify({ input: text, voice, speed, language, cache }),
|
|
105
|
+
body: JSON.stringify({ input: text, voice, speed, language, cache, session_token_id: (_a = this.sessionTokenId) !== null && _a !== void 0 ? _a : undefined }),
|
|
88
106
|
}).then((r) => r.blob());
|
|
89
107
|
});
|
|
90
108
|
}
|
|
@@ -101,6 +119,9 @@ export class AIModule {
|
|
|
101
119
|
if (language) {
|
|
102
120
|
formData.append('language', language.code);
|
|
103
121
|
}
|
|
122
|
+
if (this.sessionTokenId) {
|
|
123
|
+
formData.append('session_token_id', this.sessionTokenId);
|
|
124
|
+
}
|
|
104
125
|
return yield fetch(`${this.backendUrl}/voice/stt`, {
|
|
105
126
|
method: 'POST',
|
|
106
127
|
headers: { Authorization: `Bearer ${this.getToken()}` },
|
|
@@ -172,6 +193,7 @@ export class AIModule {
|
|
|
172
193
|
}
|
|
173
194
|
streamObject(params) {
|
|
174
195
|
return __awaiter(this, void 0, void 0, function* () {
|
|
196
|
+
var _a, _b, _c, _d;
|
|
175
197
|
const { messages, responseSchema, onResult = () => null, cache = false, tools = [], model = undefined, knowledgeId, } = params;
|
|
176
198
|
const chatMessages = messages.map((message, index) => (Object.assign(Object.assign({}, message), { id: `${index + 1}` })));
|
|
177
199
|
const response = yield fetch(`${this.backendUrl}/ai/llm`, {
|
|
@@ -183,11 +205,18 @@ export class AIModule {
|
|
|
183
205
|
messages: chatMessages,
|
|
184
206
|
model,
|
|
185
207
|
knowledge_id: knowledgeId,
|
|
208
|
+
session_token_id: (_a = this.sessionTokenId) !== null && _a !== void 0 ? _a : undefined,
|
|
186
209
|
}),
|
|
187
210
|
method: 'POST',
|
|
188
211
|
headers: { Authorization: `Bearer ${this.getToken()}`, 'Content-Type': 'application/json' },
|
|
189
212
|
});
|
|
190
213
|
if (!response.ok) {
|
|
214
|
+
if (response.status === 429) {
|
|
215
|
+
const body = yield response.json().catch(() => ({}));
|
|
216
|
+
const remaining = (_b = body.exercises_remaining) !== null && _b !== void 0 ? _b : 0;
|
|
217
|
+
(_c = this.onRateLimitedCb) === null || _c === void 0 ? void 0 : _c.call(this, remaining);
|
|
218
|
+
throw new Error(`Rate limit exceeded: ${(_d = body.error) !== null && _d !== void 0 ? _d : 'Daily exercise limit reached'}. exercises_remaining: ${remaining}`);
|
|
219
|
+
}
|
|
191
220
|
throw new Error(`Failed to stream object: ${response.status} ${response.statusText}`);
|
|
192
221
|
}
|
|
193
222
|
if (!response.body) {
|
|
@@ -212,6 +241,19 @@ export class AIModule {
|
|
|
212
241
|
const chunk = decoder.decode(value, { stream: true });
|
|
213
242
|
const lines = chunk.split('\n').filter((line) => line.trim());
|
|
214
243
|
for (const line of lines) {
|
|
244
|
+
// Handle token: line (session token issued by backend on first AI call)
|
|
245
|
+
if (line.startsWith('token:')) {
|
|
246
|
+
try {
|
|
247
|
+
const tokenData = JSON.parse(line.slice(6).trim());
|
|
248
|
+
if (tokenData.token_id) {
|
|
249
|
+
this.sessionTokenId = tokenData.token_id;
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
catch (_e) {
|
|
253
|
+
console.error('Failed to parse token: line', line);
|
|
254
|
+
}
|
|
255
|
+
continue;
|
|
256
|
+
}
|
|
215
257
|
const command = line.substring(0, 5);
|
|
216
258
|
const dataStr = line.substring(5).trim();
|
|
217
259
|
if (dataStr === '[DONE]') {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { MainPanelAction, SidebarAction } from '../../fromRimori/PluginTypes';
|
|
2
2
|
import { AccomplishmentPayload } from '../../controller/AccomplishmentController';
|
|
3
3
|
import { EventBusMessage, EventHandler, EventPayload, EventListener } from '../../fromRimori/EventBus';
|
|
4
|
+
import { AIModule } from './AIModule';
|
|
4
5
|
/**
|
|
5
6
|
* Event module for plugin event bus operations.
|
|
6
7
|
* Provides methods for emitting, listening to, and responding to events.
|
|
@@ -8,7 +9,10 @@ import { EventBusMessage, EventHandler, EventPayload, EventListener } from '../.
|
|
|
8
9
|
export declare class EventModule {
|
|
9
10
|
private pluginId;
|
|
10
11
|
private accomplishmentController;
|
|
11
|
-
|
|
12
|
+
private aiModule;
|
|
13
|
+
private backendUrl;
|
|
14
|
+
private getToken;
|
|
15
|
+
constructor(pluginId: string, backendUrl: string, getToken: () => string, aiModule: AIModule);
|
|
12
16
|
getGlobalEventTopic(preliminaryTopic: string): string;
|
|
13
17
|
/**
|
|
14
18
|
* Emit an event to Rimori or a plugin.
|
|
@@ -51,7 +55,7 @@ export declare class EventModule {
|
|
|
51
55
|
* Emit an accomplishment.
|
|
52
56
|
* @param payload The payload to emit.
|
|
53
57
|
*/
|
|
54
|
-
emitAccomplishment(payload: AccomplishmentPayload): void
|
|
58
|
+
emitAccomplishment(payload: AccomplishmentPayload): Promise<void>;
|
|
55
59
|
/**
|
|
56
60
|
* Subscribe to an accomplishment.
|
|
57
61
|
* @param accomplishmentTopic The topic to subscribe to.
|
|
@@ -1,3 +1,12 @@
|
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
1
10
|
import { AccomplishmentController } from '../../controller/AccomplishmentController';
|
|
2
11
|
import { EventBus } from '../../fromRimori/EventBus';
|
|
3
12
|
/**
|
|
@@ -5,8 +14,11 @@ import { EventBus } from '../../fromRimori/EventBus';
|
|
|
5
14
|
* Provides methods for emitting, listening to, and responding to events.
|
|
6
15
|
*/
|
|
7
16
|
export class EventModule {
|
|
8
|
-
constructor(pluginId) {
|
|
17
|
+
constructor(pluginId, backendUrl, getToken, aiModule) {
|
|
9
18
|
this.pluginId = pluginId;
|
|
19
|
+
this.backendUrl = backendUrl;
|
|
20
|
+
this.getToken = getToken;
|
|
21
|
+
this.aiModule = aiModule;
|
|
10
22
|
this.accomplishmentController = new AccomplishmentController(pluginId);
|
|
11
23
|
}
|
|
12
24
|
getGlobalEventTopic(preliminaryTopic) {
|
|
@@ -86,7 +98,24 @@ export class EventModule {
|
|
|
86
98
|
* @param payload The payload to emit.
|
|
87
99
|
*/
|
|
88
100
|
emitAccomplishment(payload) {
|
|
89
|
-
this
|
|
101
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
102
|
+
if (payload.type === 'macro') {
|
|
103
|
+
const sessionId = this.aiModule.getSessionTokenId();
|
|
104
|
+
if (sessionId) {
|
|
105
|
+
try {
|
|
106
|
+
yield fetch(`${this.backendUrl}/ai/session/${sessionId}/complete`, {
|
|
107
|
+
method: 'PATCH',
|
|
108
|
+
headers: { Authorization: `Bearer ${this.getToken()}` },
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
catch (_a) {
|
|
112
|
+
// non-fatal — session will expire naturally
|
|
113
|
+
}
|
|
114
|
+
this.aiModule.clearSessionToken();
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
this.accomplishmentController.emitAccomplishment(payload);
|
|
118
|
+
});
|
|
90
119
|
}
|
|
91
120
|
/**
|
|
92
121
|
* Subscribe to an accomplishment.
|