@ztimson/ai-utils 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/audio.d.ts +3 -2
- package/dist/index.js +26 -26
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +419 -384
- package/dist/index.mjs.map +1 -1
- package/dist/llm.d.ts +3 -7
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -1,40 +1,40 @@
|
|
|
1
|
-
import * as
|
|
2
|
-
import { tmpdir as
|
|
3
|
-
import { objectMap as
|
|
4
|
-
import { Anthropic as
|
|
5
|
-
import { OpenAI as
|
|
6
|
-
import { fileURLToPath as
|
|
7
|
-
import { join as
|
|
8
|
-
import { spawn as
|
|
9
|
-
import { mkdtempSync as
|
|
10
|
-
import
|
|
11
|
-
import * as
|
|
12
|
-
import
|
|
13
|
-
import { createWorker as
|
|
14
|
-
import * as
|
|
15
|
-
import { $ as
|
|
16
|
-
class
|
|
1
|
+
import * as z from "node:os";
|
|
2
|
+
import { tmpdir as U } from "node:os";
|
|
3
|
+
import { objectMap as A, JSONAttemptParse as _, findByProp as O, JSONSanitize as S, clean as W, Http as N, consoleInterceptor as L, fn as C, ASet as J } from "@ztimson/utils";
|
|
4
|
+
import { Anthropic as F } from "@anthropic-ai/sdk";
|
|
5
|
+
import { OpenAI as H } from "openai";
|
|
6
|
+
import { fileURLToPath as D } from "url";
|
|
7
|
+
import { join as G, dirname as I } from "path";
|
|
8
|
+
import { spawn as b, execSync as B } from "node:child_process";
|
|
9
|
+
import { mkdtempSync as K } from "node:fs";
|
|
10
|
+
import w from "node:fs/promises";
|
|
11
|
+
import * as v from "node:path";
|
|
12
|
+
import M, { join as P } from "node:path";
|
|
13
|
+
import { createWorker as V } from "tesseract.js";
|
|
14
|
+
import * as Y from "cheerio";
|
|
15
|
+
import { $ as Z, $Sync as Q } from "@ztimson/node-utils";
|
|
16
|
+
class R {
|
|
17
17
|
}
|
|
18
|
-
class
|
|
18
|
+
class X extends R {
|
|
19
19
|
constructor(r, e, t) {
|
|
20
|
-
super(), this.ai = r, this.apiToken = e, this.model = t, this.client = new
|
|
20
|
+
super(), this.ai = r, this.apiToken = e, this.model = t, this.client = new F({ apiKey: e });
|
|
21
21
|
}
|
|
22
22
|
client;
|
|
23
23
|
toStandard(r) {
|
|
24
24
|
const e = Date.now(), t = [];
|
|
25
|
-
for (let
|
|
26
|
-
if (typeof
|
|
27
|
-
t.push({ timestamp: e, ...
|
|
25
|
+
for (let a of r)
|
|
26
|
+
if (typeof a.content == "string")
|
|
27
|
+
t.push({ timestamp: e, ...a });
|
|
28
28
|
else {
|
|
29
|
-
const
|
|
29
|
+
const s = a.content?.filter((n) => n.type == "text").map((n) => n.text).join(`
|
|
30
30
|
|
|
31
31
|
`);
|
|
32
|
-
|
|
33
|
-
if (
|
|
34
|
-
t.push({ timestamp: e, role: "tool", id:
|
|
35
|
-
else if (
|
|
36
|
-
const
|
|
37
|
-
|
|
32
|
+
s && t.push({ timestamp: e, role: a.role, content: s }), a.content.forEach((n) => {
|
|
33
|
+
if (n.type == "tool_use")
|
|
34
|
+
t.push({ timestamp: e, role: "tool", id: n.id, name: n.name, args: n.input, content: void 0 });
|
|
35
|
+
else if (n.type == "tool_result") {
|
|
36
|
+
const c = t.findLast((u) => u.id == n.tool_use_id);
|
|
37
|
+
c && (c[n.is_error ? "error" : "content"] = n.content);
|
|
38
38
|
}
|
|
39
39
|
});
|
|
40
40
|
}
|
|
@@ -55,80 +55,80 @@ class Y extends v {
|
|
|
55
55
|
}
|
|
56
56
|
ask(r, e = {}) {
|
|
57
57
|
const t = new AbortController();
|
|
58
|
-
return Object.assign(new Promise(async (
|
|
59
|
-
let
|
|
60
|
-
const
|
|
58
|
+
return Object.assign(new Promise(async (a) => {
|
|
59
|
+
let s = this.fromStandard([...e.history || [], { role: "user", content: r, timestamp: Date.now() }]);
|
|
60
|
+
const n = e.tools || this.ai.options.llm?.tools || [], c = {
|
|
61
61
|
model: e.model || this.model,
|
|
62
62
|
max_tokens: e.max_tokens || this.ai.options.llm?.max_tokens || 4096,
|
|
63
63
|
system: e.system || this.ai.options.llm?.system || "",
|
|
64
64
|
temperature: e.temperature || this.ai.options.llm?.temperature || 0.7,
|
|
65
|
-
tools:
|
|
66
|
-
name:
|
|
67
|
-
description:
|
|
65
|
+
tools: n.map((m) => ({
|
|
66
|
+
name: m.name,
|
|
67
|
+
description: m.description,
|
|
68
68
|
input_schema: {
|
|
69
69
|
type: "object",
|
|
70
|
-
properties:
|
|
71
|
-
required:
|
|
70
|
+
properties: m.args ? A(m.args, (o, l) => ({ ...l, required: void 0 })) : {},
|
|
71
|
+
required: m.args ? Object.entries(m.args).filter((o) => o[1].required).map((o) => o[0]) : []
|
|
72
72
|
},
|
|
73
73
|
fn: void 0
|
|
74
74
|
})),
|
|
75
|
-
messages:
|
|
75
|
+
messages: s,
|
|
76
76
|
stream: !!e.stream
|
|
77
77
|
};
|
|
78
|
-
let
|
|
78
|
+
let u, i = !0;
|
|
79
79
|
do {
|
|
80
|
-
if (
|
|
81
|
-
throw
|
|
80
|
+
if (u = await this.client.messages.create(c).catch((o) => {
|
|
81
|
+
throw o.message += `
|
|
82
82
|
|
|
83
83
|
Messages:
|
|
84
|
-
${JSON.stringify(
|
|
84
|
+
${JSON.stringify(s, null, 2)}`, o;
|
|
85
85
|
}), e.stream) {
|
|
86
|
-
|
|
86
|
+
i ? i = !1 : e.stream({ text: `
|
|
87
87
|
|
|
88
|
-
` }),
|
|
89
|
-
for await (const
|
|
88
|
+
` }), u.content = [];
|
|
89
|
+
for await (const o of u) {
|
|
90
90
|
if (t.signal.aborted) break;
|
|
91
|
-
if (
|
|
92
|
-
|
|
93
|
-
else if (
|
|
94
|
-
if (
|
|
95
|
-
const
|
|
96
|
-
|
|
97
|
-
} else
|
|
98
|
-
else if (
|
|
99
|
-
const
|
|
100
|
-
|
|
101
|
-
} else if (
|
|
91
|
+
if (o.type === "content_block_start")
|
|
92
|
+
o.content_block.type === "text" ? u.content.push({ type: "text", text: "" }) : o.content_block.type === "tool_use" && u.content.push({ type: "tool_use", id: o.content_block.id, name: o.content_block.name, input: "" });
|
|
93
|
+
else if (o.type === "content_block_delta")
|
|
94
|
+
if (o.delta.type === "text_delta") {
|
|
95
|
+
const l = o.delta.text;
|
|
96
|
+
u.content.at(-1).text += l, e.stream({ text: l });
|
|
97
|
+
} else o.delta.type === "input_json_delta" && (u.content.at(-1).input += o.delta.partial_json);
|
|
98
|
+
else if (o.type === "content_block_stop") {
|
|
99
|
+
const l = u.content.at(-1);
|
|
100
|
+
l.input != null && (l.input = l.input ? _(l.input, {}) : {});
|
|
101
|
+
} else if (o.type === "message_stop")
|
|
102
102
|
break;
|
|
103
103
|
}
|
|
104
104
|
}
|
|
105
|
-
const
|
|
106
|
-
if (
|
|
107
|
-
|
|
108
|
-
const
|
|
109
|
-
const
|
|
110
|
-
if (e.stream && e.stream({ tool:
|
|
105
|
+
const m = u.content.filter((o) => o.type === "tool_use");
|
|
106
|
+
if (m.length && !t.signal.aborted) {
|
|
107
|
+
s.push({ role: "assistant", content: u.content });
|
|
108
|
+
const o = await Promise.all(m.map(async (l) => {
|
|
109
|
+
const d = n.find(O("name", l.name));
|
|
110
|
+
if (e.stream && e.stream({ tool: l.name }), !d) return { tool_use_id: l.id, is_error: !0, content: "Tool not found" };
|
|
111
111
|
try {
|
|
112
|
-
const
|
|
113
|
-
return { type: "tool_result", tool_use_id:
|
|
114
|
-
} catch (
|
|
115
|
-
return { type: "tool_result", tool_use_id:
|
|
112
|
+
const p = await d.fn(l.input, e?.stream, this.ai);
|
|
113
|
+
return { type: "tool_result", tool_use_id: l.id, content: S(p) };
|
|
114
|
+
} catch (p) {
|
|
115
|
+
return { type: "tool_result", tool_use_id: l.id, is_error: !0, content: p?.message || p?.toString() || "Unknown" };
|
|
116
116
|
}
|
|
117
117
|
}));
|
|
118
|
-
|
|
118
|
+
s.push({ role: "user", content: o }), c.messages = s;
|
|
119
119
|
}
|
|
120
|
-
} while (!t.signal.aborted &&
|
|
121
|
-
|
|
120
|
+
} while (!t.signal.aborted && u.content.some((m) => m.type === "tool_use"));
|
|
121
|
+
s.push({ role: "assistant", content: u.content.filter((m) => m.type == "text").map((m) => m.text).join(`
|
|
122
122
|
|
|
123
|
-
`) }),
|
|
123
|
+
`) }), s = this.toStandard(s), e.stream && e.stream({ done: !0 }), e.history && e.history.splice(0, e.history.length, ...s), a(s.at(-1)?.content);
|
|
124
124
|
}), { abort: () => t.abort() });
|
|
125
125
|
}
|
|
126
126
|
}
|
|
127
|
-
class
|
|
128
|
-
constructor(r, e, t,
|
|
129
|
-
super(), this.ai = r, this.host = e, this.token = t, this.model =
|
|
127
|
+
class q extends R {
|
|
128
|
+
constructor(r, e, t, a) {
|
|
129
|
+
super(), this.ai = r, this.host = e, this.token = t, this.model = a, this.client = new H(W({
|
|
130
130
|
baseURL: e,
|
|
131
|
-
apiKey: t
|
|
131
|
+
apiKey: t || e ? "ignored" : void 0
|
|
132
132
|
}));
|
|
133
133
|
}
|
|
134
134
|
client;
|
|
@@ -136,17 +136,17 @@ class M extends v {
|
|
|
136
136
|
for (let e = 0; e < r.length; e++) {
|
|
137
137
|
const t = r[e];
|
|
138
138
|
if (t.role === "assistant" && t.tool_calls) {
|
|
139
|
-
const
|
|
139
|
+
const a = t.tool_calls.map((s) => ({
|
|
140
140
|
role: "tool",
|
|
141
|
-
id:
|
|
142
|
-
name:
|
|
143
|
-
args: _(
|
|
141
|
+
id: s.id,
|
|
142
|
+
name: s.function.name,
|
|
143
|
+
args: _(s.function.arguments, {}),
|
|
144
144
|
timestamp: t.timestamp
|
|
145
145
|
}));
|
|
146
|
-
r.splice(e, 1, ...
|
|
146
|
+
r.splice(e, 1, ...a), e += a.length - 1;
|
|
147
147
|
} else if (t.role === "tool" && t.content) {
|
|
148
|
-
const
|
|
149
|
-
|
|
148
|
+
const a = r.find((s) => t.tool_call_id == s.id);
|
|
149
|
+
a && (t.content.includes('"error":') ? a.error = t.content : a.content = t.content), r.splice(e, 1), e--;
|
|
150
150
|
}
|
|
151
151
|
r[e]?.timestamp || (r[e].timestamp = Date.now());
|
|
152
152
|
}
|
|
@@ -167,76 +167,76 @@ class M extends v {
|
|
|
167
167
|
content: t.error || t.content
|
|
168
168
|
});
|
|
169
169
|
else {
|
|
170
|
-
const { timestamp:
|
|
171
|
-
e.push(
|
|
170
|
+
const { timestamp: a, ...s } = t;
|
|
171
|
+
e.push(s);
|
|
172
172
|
}
|
|
173
173
|
return e;
|
|
174
174
|
}, []);
|
|
175
175
|
}
|
|
176
176
|
ask(r, e = {}) {
|
|
177
177
|
const t = new AbortController();
|
|
178
|
-
return Object.assign(new Promise(async (
|
|
179
|
-
e.system && e.history?.[0]?.role != "system"
|
|
180
|
-
let
|
|
181
|
-
const
|
|
178
|
+
return Object.assign(new Promise(async (a, s) => {
|
|
179
|
+
e.system && (e.history?.[0]?.role != "system" ? e.history?.splice(0, 0, { role: "system", content: e.system, timestamp: Date.now() }) : e.history[0].content = e.system);
|
|
180
|
+
let n = this.fromStandard([...e.history || [], { role: "user", content: r, timestamp: Date.now() }]);
|
|
181
|
+
const c = e.tools || this.ai.options.llm?.tools || [], u = {
|
|
182
182
|
model: e.model || this.model,
|
|
183
|
-
messages:
|
|
183
|
+
messages: n,
|
|
184
184
|
stream: !!e.stream,
|
|
185
185
|
max_tokens: e.max_tokens || this.ai.options.llm?.max_tokens || 4096,
|
|
186
186
|
temperature: e.temperature || this.ai.options.llm?.temperature || 0.7,
|
|
187
|
-
tools:
|
|
187
|
+
tools: c.map((o) => ({
|
|
188
188
|
type: "function",
|
|
189
189
|
function: {
|
|
190
|
-
name:
|
|
191
|
-
description:
|
|
190
|
+
name: o.name,
|
|
191
|
+
description: o.description,
|
|
192
192
|
parameters: {
|
|
193
193
|
type: "object",
|
|
194
|
-
properties:
|
|
195
|
-
required:
|
|
194
|
+
properties: o.args ? A(o.args, (l, d) => ({ ...d, required: void 0 })) : {},
|
|
195
|
+
required: o.args ? Object.entries(o.args).filter((l) => l[1].required).map((l) => l[0]) : []
|
|
196
196
|
}
|
|
197
197
|
}
|
|
198
198
|
}))
|
|
199
199
|
};
|
|
200
|
-
let
|
|
200
|
+
let i, m = !0;
|
|
201
201
|
do {
|
|
202
|
-
if (
|
|
203
|
-
throw
|
|
202
|
+
if (i = await this.client.chat.completions.create(u).catch((l) => {
|
|
203
|
+
throw l.message += `
|
|
204
204
|
|
|
205
205
|
Messages:
|
|
206
|
-
${JSON.stringify(
|
|
206
|
+
${JSON.stringify(n, null, 2)}`, l;
|
|
207
207
|
}), e.stream) {
|
|
208
|
-
|
|
208
|
+
m ? m = !1 : e.stream({ text: `
|
|
209
209
|
|
|
210
|
-
` }),
|
|
211
|
-
for await (const
|
|
210
|
+
` }), i.choices = [{ message: { content: "", tool_calls: [] } }];
|
|
211
|
+
for await (const l of i) {
|
|
212
212
|
if (t.signal.aborted) break;
|
|
213
|
-
|
|
213
|
+
l.choices[0].delta.content && (i.choices[0].message.content += l.choices[0].delta.content, e.stream({ text: l.choices[0].delta.content })), l.choices[0].delta.tool_calls && (i.choices[0].message.tool_calls = l.choices[0].delta.tool_calls);
|
|
214
214
|
}
|
|
215
215
|
}
|
|
216
|
-
const
|
|
217
|
-
if (
|
|
218
|
-
|
|
219
|
-
const
|
|
220
|
-
const
|
|
221
|
-
if (e.stream && e.stream({ tool:
|
|
216
|
+
const o = i.choices[0].message.tool_calls || [];
|
|
217
|
+
if (o.length && !t.signal.aborted) {
|
|
218
|
+
n.push(i.choices[0].message);
|
|
219
|
+
const l = await Promise.all(o.map(async (d) => {
|
|
220
|
+
const p = c?.find(O("name", d.function.name));
|
|
221
|
+
if (e.stream && e.stream({ tool: d.function.name }), !p) return { role: "tool", tool_call_id: d.id, content: '{"error": "Tool not found"}' };
|
|
222
222
|
try {
|
|
223
|
-
const
|
|
224
|
-
return { role: "tool", tool_call_id:
|
|
225
|
-
} catch (
|
|
226
|
-
return { role: "tool", tool_call_id:
|
|
223
|
+
const f = _(d.function.arguments, {}), g = await p.fn(f, e.stream, this.ai);
|
|
224
|
+
return { role: "tool", tool_call_id: d.id, content: S(g) };
|
|
225
|
+
} catch (f) {
|
|
226
|
+
return { role: "tool", tool_call_id: d.id, content: S({ error: f?.message || f?.toString() || "Unknown" }) };
|
|
227
227
|
}
|
|
228
228
|
}));
|
|
229
|
-
|
|
229
|
+
n.push(...l), u.messages = n;
|
|
230
230
|
}
|
|
231
|
-
} while (!t.signal.aborted &&
|
|
232
|
-
|
|
231
|
+
} while (!t.signal.aborted && i.choices?.[0]?.message?.tool_calls?.length);
|
|
232
|
+
n.push({ role: "assistant", content: i.choices[0].message.content || "" }), n = this.toStandard(n), e.stream && e.stream({ done: !0 }), e.history && e.history.splice(0, e.history.length, ...n), a(n.at(-1)?.content);
|
|
233
233
|
}), { abort: () => t.abort() });
|
|
234
234
|
}
|
|
235
235
|
}
|
|
236
|
-
class
|
|
236
|
+
class ee {
|
|
237
237
|
constructor(r) {
|
|
238
238
|
this.ai = r, r.options.llm?.models && Object.entries(r.options.llm.models).forEach(([e, t]) => {
|
|
239
|
-
this.defaultModel || (this.defaultModel = e), t.proto == "anthropic" ? this.models[e] = new
|
|
239
|
+
this.defaultModel || (this.defaultModel = e), t.proto == "anthropic" ? this.models[e] = new X(this.ai, t.token, e) : t.proto == "ollama" ? this.models[e] = new q(this.ai, t.host, "not-needed", e) : t.proto == "openai" && (this.models[e] = new q(this.ai, t.host || null, t.token, e));
|
|
240
240
|
});
|
|
241
241
|
}
|
|
242
242
|
defaultModel;
|
|
@@ -248,58 +248,80 @@ class Q {
|
|
|
248
248
|
* @returns {{abort: () => void, response: Promise<string>}} Function to abort response and chat history
|
|
249
249
|
*/
|
|
250
250
|
ask(r, e = {}) {
|
|
251
|
+
e = {
|
|
252
|
+
system: "",
|
|
253
|
+
temperature: 0.8,
|
|
254
|
+
...this.ai.options.llm,
|
|
255
|
+
models: void 0,
|
|
256
|
+
history: [],
|
|
257
|
+
...e
|
|
258
|
+
};
|
|
251
259
|
const t = e.model || this.defaultModel;
|
|
252
260
|
if (!this.models[t]) throw new Error(`Model does not exist: ${t}`);
|
|
253
|
-
let
|
|
261
|
+
let a = () => {
|
|
254
262
|
};
|
|
255
|
-
return Object.assign(new Promise(async (
|
|
263
|
+
return Object.assign(new Promise(async (s) => {
|
|
256
264
|
if (e.history || (e.history = []), e.memory) {
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
const [m, p] = await Promise.all([
|
|
262
|
-
d ? this.embedding(d) : Promise.resolve(null),
|
|
263
|
-
a ? this.embedding(a) : Promise.resolve(null)
|
|
265
|
+
const c = async (i, m, o = 10) => {
|
|
266
|
+
const [l, d] = await Promise.all([
|
|
267
|
+
m ? this.embedding(m) : Promise.resolve(null),
|
|
268
|
+
i ? this.embedding(i) : Promise.resolve(null)
|
|
264
269
|
]);
|
|
265
|
-
return (e.memory || []).map((
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
270
|
+
return (e.memory || []).map((p) => {
|
|
271
|
+
const f = (l ? this.cosineSimilarity(p.embeddings[0], l[0].embedding) : 0) + (d ? this.cosineSimilarity(p.embeddings[1], d[0].embedding) : 0);
|
|
272
|
+
return { ...p, score: f };
|
|
273
|
+
}).toSorted((p, f) => p.score - f.score).slice(0, o);
|
|
274
|
+
};
|
|
275
|
+
e.system += `
|
|
276
|
+
You have RAG memory and will be given the top_k closest memories regarding the users query. Save anything new you have learned worth remembering from the user message using the remember tool and feel free to recall memories manually.
|
|
277
|
+
`;
|
|
278
|
+
const u = await c(r);
|
|
279
|
+
u.length && e.history.push({ role: "tool", name: "recall", id: "auto_recall_" + Math.random().toString(), args: {}, content: `Things I remembered:
|
|
280
|
+
` + u.map((i) => `${i.owner}: ${i.fact}`).join(`
|
|
281
|
+
`) }), e.tools = [{
|
|
282
|
+
name: "recall",
|
|
283
|
+
description: "Recall the closest memories you have regarding a query using RAG",
|
|
272
284
|
args: {
|
|
273
285
|
subject: { type: "string", description: "Find information by a subject topic, can be used with or without query argument" },
|
|
274
286
|
query: { type: "string", description: "Search memory based on a query, can be used with or without subject argument" },
|
|
275
|
-
|
|
287
|
+
topK: { type: "number", description: "Result limit, default 5" }
|
|
276
288
|
},
|
|
277
|
-
fn: (
|
|
278
|
-
if (!
|
|
279
|
-
return
|
|
289
|
+
fn: (i) => {
|
|
290
|
+
if (!i.subject && !i.query) throw new Error("Either a subject or query argument is required");
|
|
291
|
+
return c(i.query, i.subject, i.topK);
|
|
280
292
|
}
|
|
281
|
-
}
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
293
|
+
}, {
|
|
294
|
+
name: "remember",
|
|
295
|
+
description: "Store important facts user shares for future recall",
|
|
296
|
+
args: {
|
|
297
|
+
owner: { type: "string", description: "Subject/person this fact is about" },
|
|
298
|
+
fact: { type: "string", description: "The information to remember" }
|
|
299
|
+
},
|
|
300
|
+
fn: async (i) => {
|
|
301
|
+
if (!e.memory) return;
|
|
302
|
+
const m = await Promise.all([
|
|
303
|
+
this.embedding(i.owner),
|
|
304
|
+
this.embedding(`${i.owner}: ${i.fact}`)
|
|
305
|
+
]), o = { owner: i.owner, fact: i.fact, embeddings: [m[0][0].embedding, m[1][0].embedding] };
|
|
306
|
+
return e.memory.splice(0, e.memory.length, ...e.memory.filter((l) => this.cosineSimilarity(o.embeddings[0], l.embeddings[0]) < 0.9 && this.cosineSimilarity(o.embeddings[1], l.embeddings[1]) < 0.8), o), "Remembered!";
|
|
307
|
+
}
|
|
308
|
+
}, ...e.tools || []];
|
|
287
309
|
}
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
else {
|
|
293
|
-
const n = e.history?.findLastIndex((a) => a.role == "user") ?? -1;
|
|
294
|
-
l = await this.ai.language.compressHistory(n != -1 ? e.history.slice(n) : e.history, 0, 0, e);
|
|
295
|
-
}
|
|
296
|
-
if (e.memory) {
|
|
297
|
-
const n = e.memory.filter((a) => !l.memory.some((d) => this.cosineSimilarity(a.embeddings[1], d.embeddings[1]) > 0.8)).concat(l.memory);
|
|
298
|
-
e.memory.splice(0, e.memory.length, ...n);
|
|
299
|
-
}
|
|
310
|
+
const n = await this.models[t].ask(r, e);
|
|
311
|
+
if (e.memory && e.history.splice(0, e.history.length, ...e.history.filter((c) => c.role != "tool" || c.name != "recall" && c.name != "remember")), e.compress) {
|
|
312
|
+
const c = await this.ai.language.compressHistory(e.history, e.compress.max, e.compress.min, e);
|
|
313
|
+
e.history.splice(0, e.history.length, ...c);
|
|
300
314
|
}
|
|
301
|
-
return
|
|
302
|
-
}), { abort:
|
|
315
|
+
return s(n);
|
|
316
|
+
}), { abort: a });
|
|
317
|
+
}
|
|
318
|
+
async code(r, e) {
|
|
319
|
+
const t = await this.ask(r, { ...e, system: [
|
|
320
|
+
e?.system,
|
|
321
|
+
"Return your response in a code block"
|
|
322
|
+
].filter((s) => !!s).join(`
|
|
323
|
+
`) }), a = /```(?:.+)?\s*([\s\S]*?)```/.exec(t);
|
|
324
|
+
return a ? a[1].trim() : null;
|
|
303
325
|
}
|
|
304
326
|
/**
|
|
305
327
|
* Compress chat history to reduce context size
|
|
@@ -309,24 +331,17 @@ You have passive persistent memory never make any mention of your memory capabil
|
|
|
309
331
|
* @param {LLMRequest} options LLM options
|
|
310
332
|
* @returns {Promise<LLMMessage[]>} New chat history will summary at index 0
|
|
311
333
|
*/
|
|
312
|
-
async compressHistory(r, e, t,
|
|
313
|
-
if (this.estimateTokens(r) < e) return
|
|
314
|
-
let
|
|
315
|
-
for (let
|
|
316
|
-
if (
|
|
334
|
+
async compressHistory(r, e, t, a) {
|
|
335
|
+
if (this.estimateTokens(r) < e) return r;
|
|
336
|
+
let s = 0, n = 0;
|
|
337
|
+
for (let d of r.toReversed())
|
|
338
|
+
if (n += this.estimateTokens(d.content), n < t) s++;
|
|
317
339
|
else break;
|
|
318
|
-
if (r.length <=
|
|
319
|
-
const
|
|
340
|
+
if (r.length <= s) return r;
|
|
341
|
+
const c = r[0].role == "system" ? r[0] : null, u = s == 0 ? [] : r.slice(-s), i = (s == 0 ? r : r.slice(0, -s)).filter((d) => d.role === "assistant" || d.role === "user"), m = await this.summarize(i.map((d) => `[${d.role}]: ${d.content}`).join(`
|
|
320
342
|
|
|
321
|
-
`), "
|
|
322
|
-
|
|
323
|
-
model: c?.model,
|
|
324
|
-
temperature: c?.temperature || 0.3
|
|
325
|
-
}), i = /* @__PURE__ */ new Date(), m = await Promise.all((d?.facts || [])?.map(async ([u, h]) => {
|
|
326
|
-
const y = await Promise.all([this.embedding(u), this.embedding(`${u}: ${h}`)]);
|
|
327
|
-
return { owner: u, fact: h, embeddings: [y[0][0].embedding, y[1][0].embedding], timestamp: i };
|
|
328
|
-
})), p = [{ role: "assistant", content: `Conversation Summary: ${d?.summary}`, timestamp: Date.now() }, ...n];
|
|
329
|
-
return l && p.splice(0, 0, l), { history: p, memory: m };
|
|
343
|
+
`), 500, a), o = Date.now(), l = [{ role: "tool", name: "summary", id: "summary_" + o, args: {}, content: `Conversation Summary: ${m?.summary}`, timestamp: o }, ...u];
|
|
344
|
+
return c && l.splice(0, 0, c), l;
|
|
330
345
|
}
|
|
331
346
|
/**
|
|
332
347
|
* Compare the difference between embeddings (calculates the angle between two vectors)
|
|
@@ -336,11 +351,11 @@ You have passive persistent memory never make any mention of your memory capabil
|
|
|
336
351
|
*/
|
|
337
352
|
cosineSimilarity(r, e) {
|
|
338
353
|
if (r.length !== e.length) throw new Error("Vectors must be same length");
|
|
339
|
-
let t = 0,
|
|
340
|
-
for (let
|
|
341
|
-
t += r[
|
|
342
|
-
const
|
|
343
|
-
return
|
|
354
|
+
let t = 0, a = 0, s = 0;
|
|
355
|
+
for (let c = 0; c < r.length; c++)
|
|
356
|
+
t += r[c] * e[c], a += r[c] * r[c], s += e[c] * e[c];
|
|
357
|
+
const n = Math.sqrt(a) * Math.sqrt(s);
|
|
358
|
+
return n === 0 ? 0 : t / n;
|
|
344
359
|
}
|
|
345
360
|
/**
|
|
346
361
|
* Chunk text into parts for AI digestion
|
|
@@ -350,25 +365,25 @@ You have passive persistent memory never make any mention of your memory capabil
|
|
|
350
365
|
* @returns {string[]} Chunked strings
|
|
351
366
|
*/
|
|
352
367
|
chunk(r, e = 500, t = 50) {
|
|
353
|
-
const
|
|
354
|
-
const
|
|
355
|
-
return typeof
|
|
356
|
-
}) : [],
|
|
357
|
-
`)).flatMap((
|
|
358
|
-
`]),
|
|
359
|
-
for (let
|
|
360
|
-
let
|
|
361
|
-
for (;
|
|
362
|
-
const
|
|
363
|
-
if (this.estimateTokens(
|
|
364
|
-
`)) > e &&
|
|
365
|
-
|
|
368
|
+
const a = (u, i = "") => u ? Object.entries(u).flatMap(([m, o]) => {
|
|
369
|
+
const l = i ? `${i}${isNaN(+m) ? `.${m}` : `[${m}]`}` : m;
|
|
370
|
+
return typeof o == "object" && !Array.isArray(o) ? a(o, l) : `${l}: ${Array.isArray(o) ? o.join(", ") : o}`;
|
|
371
|
+
}) : [], n = (typeof r == "object" ? a(r) : r.toString().split(`
|
|
372
|
+
`)).flatMap((u) => [...u.split(/\s+/).filter(Boolean), `
|
|
373
|
+
`]), c = [];
|
|
374
|
+
for (let u = 0; u < n.length; ) {
|
|
375
|
+
let i = "", m = u;
|
|
376
|
+
for (; m < n.length; ) {
|
|
377
|
+
const l = i + (i ? " " : "") + n[m];
|
|
378
|
+
if (this.estimateTokens(l.replace(/\s*\n\s*/g, `
|
|
379
|
+
`)) > e && i) break;
|
|
380
|
+
i = l, m++;
|
|
366
381
|
}
|
|
367
|
-
const
|
|
382
|
+
const o = i.replace(/\s*\n\s*/g, `
|
|
368
383
|
`).trim();
|
|
369
|
-
|
|
384
|
+
o && c.push(o), u = Math.max(m - t, m === u ? u + 1 : m);
|
|
370
385
|
}
|
|
371
|
-
return
|
|
386
|
+
return c;
|
|
372
387
|
}
|
|
373
388
|
/**
|
|
374
389
|
* Create a vector representation of a string
|
|
@@ -377,39 +392,39 @@ You have passive persistent memory never make any mention of your memory capabil
|
|
|
377
392
|
* @returns {Promise<Awaited<{index: number, embedding: number[], text: string, tokens: number}>[]>} Chunked embeddings
|
|
378
393
|
*/
|
|
379
394
|
embedding(r, e = {}) {
|
|
380
|
-
let { maxTokens: t = 500, overlapTokens:
|
|
381
|
-
const
|
|
382
|
-
|
|
383
|
-
},
|
|
384
|
-
if (
|
|
385
|
-
const
|
|
386
|
-
|
|
395
|
+
let { maxTokens: t = 500, overlapTokens: a = 50 } = e, s = !1;
|
|
396
|
+
const n = () => {
|
|
397
|
+
s = !0;
|
|
398
|
+
}, c = (i) => new Promise((m, o) => {
|
|
399
|
+
if (s) return o(new Error("Aborted"));
|
|
400
|
+
const l = [
|
|
401
|
+
G(I(D(import.meta.url)), "embedder.js"),
|
|
387
402
|
this.ai.options.path,
|
|
388
403
|
this.ai.options?.embedder || "bge-small-en-v1.5"
|
|
389
|
-
],
|
|
390
|
-
|
|
391
|
-
let
|
|
392
|
-
|
|
393
|
-
if (
|
|
394
|
-
if (
|
|
404
|
+
], d = b("node", l, { stdio: ["pipe", "pipe", "ignore"] });
|
|
405
|
+
d.stdin.write(i), d.stdin.end();
|
|
406
|
+
let p = "";
|
|
407
|
+
d.stdout.on("data", (f) => p += f.toString()), d.on("close", (f) => {
|
|
408
|
+
if (s) return o(new Error("Aborted"));
|
|
409
|
+
if (f === 0)
|
|
395
410
|
try {
|
|
396
|
-
const
|
|
397
|
-
|
|
411
|
+
const g = JSON.parse(p);
|
|
412
|
+
m(g.embedding);
|
|
398
413
|
} catch {
|
|
399
|
-
|
|
414
|
+
o(new Error("Failed to parse embedding output"));
|
|
400
415
|
}
|
|
401
416
|
else
|
|
402
|
-
|
|
403
|
-
}),
|
|
404
|
-
}),
|
|
405
|
-
const
|
|
406
|
-
for (let
|
|
407
|
-
const
|
|
408
|
-
|
|
417
|
+
o(new Error(`Embedder process exited with code ${f}`));
|
|
418
|
+
}), d.on("error", o);
|
|
419
|
+
}), u = (async () => {
|
|
420
|
+
const i = this.chunk(r, t, a), m = [];
|
|
421
|
+
for (let o = 0; o < i.length && !s; o++) {
|
|
422
|
+
const l = i[o], d = await c(l);
|
|
423
|
+
m.push({ index: o, embedding: d, text: l, tokens: this.estimateTokens(l) });
|
|
409
424
|
}
|
|
410
|
-
return
|
|
425
|
+
return m;
|
|
411
426
|
})();
|
|
412
|
-
return Object.assign(
|
|
427
|
+
return Object.assign(u, { abort: n });
|
|
413
428
|
}
|
|
414
429
|
/**
|
|
415
430
|
* Estimate variable as tokens
|
|
@@ -428,8 +443,8 @@ You have passive persistent memory never make any mention of your memory capabil
|
|
|
428
443
|
*/
|
|
429
444
|
fuzzyMatch(r, ...e) {
|
|
430
445
|
if (e.length < 2) throw new Error("Requires at least 2 strings to compare");
|
|
431
|
-
const t = (
|
|
432
|
-
return { avg:
|
|
446
|
+
const t = (n, c = 10) => n.toLowerCase().split("").map((u, i) => u.charCodeAt(0) * (i + 1) % c / c).slice(0, c), a = t(r), s = e.map((n) => t(n)).map((n) => this.cosineSimilarity(a, n));
|
|
447
|
+
return { avg: s.reduce((n, c) => n + c, 0) / s.length, max: Math.max(...s), similarities: s };
|
|
433
448
|
}
|
|
434
449
|
/**
|
|
435
450
|
* Ask a question with JSON response
|
|
@@ -439,14 +454,15 @@ You have passive persistent memory never make any mention of your memory capabil
|
|
|
439
454
|
* @returns {Promise<{} | {} | RegExpExecArray | null>}
|
|
440
455
|
*/
|
|
441
456
|
async json(r, e, t) {
|
|
442
|
-
|
|
443
|
-
|
|
457
|
+
const a = await this.code(r, { ...t, system: [
|
|
458
|
+
t?.system,
|
|
459
|
+
`Only respond using JSON matching this schema:
|
|
444
460
|
\`\`\`json
|
|
445
461
|
${e}
|
|
446
|
-
\`\`\``
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
return _(
|
|
462
|
+
\`\`\``
|
|
463
|
+
].filter((s) => !!s).join(`
|
|
464
|
+
`) });
|
|
465
|
+
return a ? _(a, {}) : null;
|
|
450
466
|
}
|
|
451
467
|
/**
|
|
452
468
|
* Create a summary of some text
|
|
@@ -455,11 +471,11 @@ ${e}
|
|
|
455
471
|
* @param options LLM request options
|
|
456
472
|
* @returns {Promise<string>} Summary
|
|
457
473
|
*/
|
|
458
|
-
summarize(r, e, t) {
|
|
459
|
-
return this.ask(r, { system: `Generate
|
|
474
|
+
summarize(r, e = 500, t) {
|
|
475
|
+
return this.ask(r, { system: `Generate the shortest summary possible <= ${e} tokens. Output nothing else`, temperature: 0.3, ...t });
|
|
460
476
|
}
|
|
461
477
|
}
|
|
462
|
-
class
|
|
478
|
+
class te {
|
|
463
479
|
constructor(r) {
|
|
464
480
|
this.ai = r, r.options.whisper && (this.whisperModel = r.options.asr || "ggml-base.en.bin", this.downloadAsrModel()), this.pyannote = `
|
|
465
481
|
import sys
|
|
@@ -481,147 +497,166 @@ print(json.dumps(segments))
|
|
|
481
497
|
downloads = {};
|
|
482
498
|
pyannote;
|
|
483
499
|
whisperModel;
|
|
500
|
+
async addPunctuation(r, e, t = 150) {
|
|
501
|
+
const a = (n) => {
|
|
502
|
+
if (n = n.toLowerCase().replace(/[^a-z]/g, ""), n.length <= 3) return 1;
|
|
503
|
+
const c = n.match(/[aeiouy]+/g);
|
|
504
|
+
let u = c ? c.length : 1;
|
|
505
|
+
return n.endsWith("e") && u--, Math.max(1, u);
|
|
506
|
+
};
|
|
507
|
+
let s = "";
|
|
508
|
+
return r.transcription.filter((n, c) => {
|
|
509
|
+
let u = !1;
|
|
510
|
+
const i = r.transcription[c - 1], m = r.transcription[c + 1];
|
|
511
|
+
return !n.text && m ? (m.offsets.from = n.offsets.from, m.timestamps.from = n.offsets.from) : n.text && n.text[0] != " " && i && (i.offsets.to = n.offsets.to, i.timestamps.to = n.timestamps.to, i.text += n.text, u = !0), !!n.text && !u;
|
|
512
|
+
}).forEach((n) => {
|
|
513
|
+
const c = /^[A-Z]/.test(n.text.trim()), u = n.offsets.to - n.offsets.from, m = a(n.text.trim()) * t;
|
|
514
|
+
c && u > m * 2 && n.text[0] == " " && (s += "."), s += n.text;
|
|
515
|
+
}), e ? this.ai.language.ask(s, {
|
|
516
|
+
system: "Remove any misplaced punctuation from the following ASR transcript using the replace tool. Avoid modifying words unless there is an obvious typo",
|
|
517
|
+
temperature: 0.1,
|
|
518
|
+
tools: [{
|
|
519
|
+
name: "replace",
|
|
520
|
+
description: "Use find and replace to fix errors",
|
|
521
|
+
args: {
|
|
522
|
+
find: { type: "string", description: "Text to find", required: !0 },
|
|
523
|
+
replace: { type: "string", description: "Text to replace", required: !0 }
|
|
524
|
+
},
|
|
525
|
+
fn: (n) => s = s.replace(n.find, n.replace)
|
|
526
|
+
}]
|
|
527
|
+
}).then(() => s) : s.trim();
|
|
528
|
+
}
|
|
529
|
+
async diarizeTranscript(r, e, t) {
|
|
530
|
+
const a = /* @__PURE__ */ new Map();
|
|
531
|
+
let s = 0;
|
|
532
|
+
e.forEach((p) => {
|
|
533
|
+
a.has(p.speaker) || a.set(p.speaker, ++s);
|
|
534
|
+
});
|
|
535
|
+
const n = await this.addPunctuation(r, t), c = n.match(/[^.!?]+[.!?]+/g) || [n], u = r.transcription.filter((p) => p.text.trim()), i = c.map((p) => {
|
|
536
|
+
if (p = p.trim(), !p) return null;
|
|
537
|
+
const f = p.toLowerCase().replace(/[^\w\s]/g, "").split(/\s+/), g = /* @__PURE__ */ new Map();
|
|
538
|
+
f.forEach((k) => {
|
|
539
|
+
const x = u.find((y) => k === y.text.trim().toLowerCase().replace(/[^\w]/g, ""));
|
|
540
|
+
if (!x) return;
|
|
541
|
+
const E = x.offsets.from / 1e3, $ = e.find((y) => E >= y.start && E <= y.end);
|
|
542
|
+
if ($) {
|
|
543
|
+
const y = a.get($.speaker);
|
|
544
|
+
g.set(y, (g.get(y) || 0) + 1);
|
|
545
|
+
}
|
|
546
|
+
});
|
|
547
|
+
let j = 1, T = 0;
|
|
548
|
+
return g.forEach((k, x) => {
|
|
549
|
+
k > T && (T = k, j = x);
|
|
550
|
+
}), { speaker: j, text: p };
|
|
551
|
+
}).filter((p) => p !== null), m = [];
|
|
552
|
+
i.forEach((p) => {
|
|
553
|
+
const f = m[m.length - 1];
|
|
554
|
+
f && f.speaker === p.speaker ? f.text += " " + p.text : m.push({ ...p });
|
|
555
|
+
});
|
|
556
|
+
let o = m.map((p) => `[Speaker ${p.speaker}]: ${p.text}`).join(`
|
|
557
|
+
`).trim();
|
|
558
|
+
if (!t) return o;
|
|
559
|
+
let l = this.ai.language.chunk(o, 500, 0);
|
|
560
|
+
l.length > 4 && (l = [...l.slice(0, 3), l.at(-1)]);
|
|
561
|
+
const d = await this.ai.language.json(l.join(`
|
|
562
|
+
`), '{1: "Detected Name", 2: "Second Name"}', {
|
|
563
|
+
system: "Use the following transcript to identify speakers. Only identify speakers you are positive about, dont mention speakers you are unsure about in your response",
|
|
564
|
+
temperature: 0.1
|
|
565
|
+
});
|
|
566
|
+
return Object.entries(d).forEach(([p, f]) => o = o.replaceAll(`[Speaker ${p}]`, `[${f}]`)), o;
|
|
567
|
+
}
|
|
484
568
|
runAsr(r, e = {}) {
|
|
485
569
|
let t;
|
|
486
|
-
const
|
|
487
|
-
this.downloadAsrModel(e.model).then((
|
|
570
|
+
const a = new Promise((s, n) => {
|
|
571
|
+
this.downloadAsrModel(e.model).then((c) => {
|
|
488
572
|
if (e.diarization) {
|
|
489
|
-
let
|
|
490
|
-
t =
|
|
573
|
+
let u = v.join(v.dirname(r), "transcript");
|
|
574
|
+
t = b(
|
|
491
575
|
this.ai.options.whisper,
|
|
492
|
-
["-m",
|
|
576
|
+
["-m", c, "-f", r, "-np", "-ml", "1", "-oj", "-of", u],
|
|
493
577
|
{ stdio: ["ignore", "ignore", "pipe"] }
|
|
494
|
-
), t.on("error", (
|
|
495
|
-
if (
|
|
496
|
-
|
|
578
|
+
), t.on("error", (i) => n(i)), t.on("close", async (i) => {
|
|
579
|
+
if (i === 0) {
|
|
580
|
+
u = await w.readFile(u + ".json", "utf-8"), w.rm(u + ".json").catch(() => {
|
|
497
581
|
});
|
|
498
582
|
try {
|
|
499
|
-
|
|
583
|
+
s(JSON.parse(u));
|
|
500
584
|
} catch {
|
|
501
|
-
|
|
585
|
+
n(new Error("Failed to parse whisper JSON"));
|
|
502
586
|
}
|
|
503
587
|
} else
|
|
504
|
-
|
|
588
|
+
n(new Error(`Exit code ${i}`));
|
|
505
589
|
});
|
|
506
590
|
} else {
|
|
507
|
-
let
|
|
508
|
-
t =
|
|
509
|
-
|
|
591
|
+
let u = "";
|
|
592
|
+
t = b(this.ai.options.whisper, ["-m", c, "-f", r, "-np", "-nt"]), t.on("error", (i) => n(i)), t.stdout.on("data", (i) => u += i.toString()), t.on("close", async (i) => {
|
|
593
|
+
i === 0 ? s(u.trim() || null) : n(new Error(`Exit code ${i}`));
|
|
510
594
|
});
|
|
511
595
|
}
|
|
512
596
|
});
|
|
513
597
|
});
|
|
514
|
-
return Object.assign(
|
|
598
|
+
return Object.assign(a, { abort: () => t?.kill("SIGTERM") });
|
|
515
599
|
}
|
|
516
600
|
runDiarization(r) {
|
|
517
601
|
let e = !1, t = () => {
|
|
518
602
|
e = !0;
|
|
519
603
|
};
|
|
520
|
-
const
|
|
521
|
-
const
|
|
522
|
-
|
|
523
|
-
}),
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
]).then((async ([
|
|
604
|
+
const a = (n) => new Promise((c) => {
|
|
605
|
+
const u = b(n, ["-W", "ignore", "-c", "import pyannote.audio"]);
|
|
606
|
+
u.on("close", (i) => c(i === 0)), u.on("error", () => c(!1));
|
|
607
|
+
}), s = Promise.all([
|
|
608
|
+
a("python"),
|
|
609
|
+
a("python3")
|
|
610
|
+
]).then((async ([n, c]) => {
|
|
527
611
|
if (e) return;
|
|
528
|
-
if (!
|
|
529
|
-
const
|
|
530
|
-
return new Promise((
|
|
612
|
+
if (!n && !c) throw new Error("Pyannote is not installed: pip install pyannote.audio");
|
|
613
|
+
const u = c ? "python3" : "python";
|
|
614
|
+
return new Promise((i, m) => {
|
|
531
615
|
if (e) return;
|
|
532
|
-
let
|
|
533
|
-
const
|
|
534
|
-
|
|
535
|
-
if (
|
|
616
|
+
let o = "";
|
|
617
|
+
const l = b(u, ["-W", "ignore", "-c", this.pyannote, r]);
|
|
618
|
+
l.stdout.on("data", (d) => o += d.toString()), l.stderr.on("data", (d) => console.error(d.toString())), l.on("close", (d) => {
|
|
619
|
+
if (d === 0)
|
|
536
620
|
try {
|
|
537
|
-
|
|
621
|
+
i(JSON.parse(o));
|
|
538
622
|
} catch {
|
|
539
|
-
|
|
623
|
+
m(new Error("Failed to parse diarization output"));
|
|
540
624
|
}
|
|
541
625
|
else
|
|
542
|
-
|
|
543
|
-
}),
|
|
626
|
+
m(new Error(`Python process exited with code ${d}`));
|
|
627
|
+
}), l.on("error", m), t = () => l.kill("SIGTERM");
|
|
544
628
|
});
|
|
545
629
|
}));
|
|
546
|
-
return Object.assign(
|
|
547
|
-
}
|
|
548
|
-
async combineSpeakerTranscript(r, e, t) {
|
|
549
|
-
const c = /* @__PURE__ */ new Map();
|
|
550
|
-
let o = 0;
|
|
551
|
-
t.forEach((n) => {
|
|
552
|
-
c.has(n.speaker) || c.set(n.speaker, ++o);
|
|
553
|
-
});
|
|
554
|
-
const s = r.match(/[^.!?]+[.!?]+/g) || [r], l = [];
|
|
555
|
-
return s.forEach((n) => {
|
|
556
|
-
if (n = n.trim(), !n) return;
|
|
557
|
-
const a = n.toLowerCase().replace(/[^\w\s]/g, "").split(/\s+/);
|
|
558
|
-
let d = 1 / 0;
|
|
559
|
-
const i = [];
|
|
560
|
-
if (e.transcription.forEach((h) => {
|
|
561
|
-
const y = h.text.trim().toLowerCase();
|
|
562
|
-
if (a.some((g) => y.includes(g))) {
|
|
563
|
-
const g = h.offsets.from / 1e3, w = h.offsets.to / 1e3;
|
|
564
|
-
i.push({ start: g, end: w }), g < d && (d = g);
|
|
565
|
-
}
|
|
566
|
-
}), d === 1 / 0) return;
|
|
567
|
-
const m = /* @__PURE__ */ new Map();
|
|
568
|
-
i.forEach((h) => {
|
|
569
|
-
t.forEach((y) => {
|
|
570
|
-
const g = Math.max(0, Math.min(h.end, y.end) - Math.max(h.start, y.start)), w = h.end - h.start;
|
|
571
|
-
if (w > 0) {
|
|
572
|
-
const A = g / w, S = c.get(y.speaker);
|
|
573
|
-
m.set(S, (m.get(S) || 0) + A);
|
|
574
|
-
}
|
|
575
|
-
});
|
|
576
|
-
});
|
|
577
|
-
let p = 1, u = 0;
|
|
578
|
-
m.forEach((h, y) => {
|
|
579
|
-
h > u && (u = h, p = y);
|
|
580
|
-
}), l.push(`[Speaker ${p}]: ${n}`);
|
|
581
|
-
}), l.join(`
|
|
582
|
-
`).trim();
|
|
630
|
+
return Object.assign(s, { abort: t });
|
|
583
631
|
}
|
|
584
632
|
asr(r, e = {}) {
|
|
585
633
|
if (!this.ai.options.whisper) throw new Error("Whisper not configured");
|
|
586
|
-
const t =
|
|
587
|
-
|
|
588
|
-
const
|
|
589
|
-
})
|
|
590
|
-
|
|
591
|
-
|
|
634
|
+
const t = P(K(P(U(), "audio-")), "converted.wav");
|
|
635
|
+
B(`ffmpeg -i "${r}" -ar 16000 -ac 1 -f wav "${t}"`, { stdio: "ignore" });
|
|
636
|
+
const a = () => w.rm(M.dirname(t), { recursive: !0, force: !0 }).catch(() => {
|
|
637
|
+
});
|
|
638
|
+
if (!e.diarization) return this.runAsr(t, { model: e.model });
|
|
639
|
+
const s = this.runAsr(t, { model: e.model, diarization: !0 }), n = this.runDiarization(t);
|
|
640
|
+
let c = !1, u = () => {
|
|
641
|
+
c = !0, s.abort(), n.abort(), a();
|
|
592
642
|
};
|
|
593
|
-
const
|
|
594
|
-
if (i.status == "rejected") throw new Error(`Whisper.cpp punctuated:
|
|
595
|
-
` + i.reason);
|
|
643
|
+
const i = Promise.allSettled([s, n]).then(async ([m, o]) => {
|
|
596
644
|
if (m.status == "rejected") throw new Error(`Whisper.cpp timestamps:
|
|
597
645
|
` + m.reason);
|
|
598
|
-
if (
|
|
599
|
-
` +
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
if (!this.ai.language.defaultModel) throw new Error("Configure an LLM for advanced ASR speaker detection");
|
|
604
|
-
let h = this.ai.language.chunk(u, 500, 0);
|
|
605
|
-
h.length > 4 && (h = [...h.slice(0, 3), h.at(-1)]);
|
|
606
|
-
const y = await this.ai.language.json(h.join(`
|
|
607
|
-
`), '{1: "Detected Name", 2: "Second Name"}', {
|
|
608
|
-
system: "Use the following transcript to identify speakers. Only identify speakers you are positive about, dont mention speakers you are unsure about in your response",
|
|
609
|
-
temperature: 0.1
|
|
610
|
-
});
|
|
611
|
-
Object.entries(y).forEach(([g, w]) => u = u.replaceAll(`[Speaker ${g}]`, `[${w}]`));
|
|
612
|
-
}
|
|
613
|
-
return u;
|
|
614
|
-
}).finally(() => c());
|
|
615
|
-
return Object.assign(d, { abort: a });
|
|
646
|
+
if (o.status == "rejected") throw new Error(`Pyannote:
|
|
647
|
+
` + o.reason);
|
|
648
|
+
return c || !e.diarization ? m.value : this.diarizeTranscript(m.value, o.value, e.diarization == "llm");
|
|
649
|
+
}).finally(() => a());
|
|
650
|
+
return Object.assign(i, { abort: u });
|
|
616
651
|
}
|
|
617
652
|
async downloadAsrModel(r = this.whisperModel) {
|
|
618
653
|
if (!this.ai.options.whisper) throw new Error("Whisper not configured");
|
|
619
654
|
r.endsWith(".bin") || (r += ".bin");
|
|
620
|
-
const e =
|
|
621
|
-
return await
|
|
655
|
+
const e = M.join(this.ai.options.path, r);
|
|
656
|
+
return await w.stat(e).then(() => !0).catch(() => !1) ? e : this.downloads[r] ? this.downloads[r] : (this.downloads[r] = fetch(`https://huggingface.co/ggerganov/whisper.cpp/resolve/main/${r}`).then((t) => t.arrayBuffer()).then((t) => Buffer.from(t)).then(async (t) => (await w.writeFile(e, t), delete this.downloads[r], e)), this.downloads[r]);
|
|
622
657
|
}
|
|
623
658
|
}
|
|
624
|
-
class
|
|
659
|
+
class re {
|
|
625
660
|
constructor(r) {
|
|
626
661
|
this.ai = r;
|
|
627
662
|
}
|
|
@@ -632,17 +667,17 @@ class Z {
|
|
|
632
667
|
*/
|
|
633
668
|
ocr(r) {
|
|
634
669
|
let e;
|
|
635
|
-
const t = new Promise(async (
|
|
636
|
-
e = await
|
|
637
|
-
const { data:
|
|
638
|
-
await e.terminate(),
|
|
670
|
+
const t = new Promise(async (a) => {
|
|
671
|
+
e = await V(this.ai.options.ocr || "eng", 2, { cachePath: this.ai.options.path });
|
|
672
|
+
const { data: s } = await e.recognize(r);
|
|
673
|
+
await e.terminate(), a(s.text.trim() || null);
|
|
639
674
|
});
|
|
640
675
|
return Object.assign(t, { abort: () => e?.terminate() });
|
|
641
676
|
}
|
|
642
677
|
}
|
|
643
|
-
class
|
|
678
|
+
class we {
|
|
644
679
|
constructor(r) {
|
|
645
|
-
this.options = r, r.path || (r.path =
|
|
680
|
+
this.options = r, r.path || (r.path = z.tmpdir()), process.env.TRANSFORMERS_CACHE = r.path, this.audio = new te(this), this.language = new ee(this), this.vision = new re(this);
|
|
646
681
|
}
|
|
647
682
|
/** Audio processing AI */
|
|
648
683
|
audio;
|
|
@@ -651,38 +686,38 @@ class fe {
|
|
|
651
686
|
/** Vision processing AI */
|
|
652
687
|
vision;
|
|
653
688
|
}
|
|
654
|
-
const
|
|
689
|
+
const ne = {
|
|
655
690
|
name: "cli",
|
|
656
691
|
description: "Use the command line interface, returns any output",
|
|
657
692
|
args: { command: { type: "string", description: "Command to run", required: !0 } },
|
|
658
|
-
fn: (
|
|
659
|
-
},
|
|
693
|
+
fn: (h) => Z`${h.command}`
|
|
694
|
+
}, be = {
|
|
660
695
|
name: "get_datetime",
|
|
661
696
|
description: "Get current UTC date / time",
|
|
662
697
|
args: {},
|
|
663
698
|
fn: async () => (/* @__PURE__ */ new Date()).toUTCString()
|
|
664
|
-
},
|
|
699
|
+
}, ke = {
|
|
665
700
|
name: "exec",
|
|
666
701
|
description: "Run code/scripts",
|
|
667
702
|
args: {
|
|
668
703
|
language: { type: "string", description: "Execution language", enum: ["cli", "node", "python"], required: !0 },
|
|
669
704
|
code: { type: "string", description: "Code to execute", required: !0 }
|
|
670
705
|
},
|
|
671
|
-
fn: async (
|
|
706
|
+
fn: async (h, r, e) => {
|
|
672
707
|
try {
|
|
673
|
-
switch (
|
|
708
|
+
switch (h.type) {
|
|
674
709
|
case "bash":
|
|
675
|
-
return await
|
|
710
|
+
return await ne.fn({ command: h.code }, r, e);
|
|
676
711
|
case "node":
|
|
677
|
-
return await
|
|
712
|
+
return await se.fn({ code: h.code }, r, e);
|
|
678
713
|
case "python":
|
|
679
|
-
return await
|
|
714
|
+
return await oe.fn({ code: h.code }, r, e);
|
|
680
715
|
}
|
|
681
716
|
} catch (t) {
|
|
682
717
|
return { error: t?.message || t.toString() };
|
|
683
718
|
}
|
|
684
719
|
}
|
|
685
|
-
},
|
|
720
|
+
}, xe = {
|
|
686
721
|
name: "fetch",
|
|
687
722
|
description: "Make HTTP request to URL",
|
|
688
723
|
args: {
|
|
@@ -691,85 +726,85 @@ const ee = {
|
|
|
691
726
|
headers: { type: "object", description: "HTTP headers to send", default: {} },
|
|
692
727
|
body: { type: "object", description: "HTTP body to send" }
|
|
693
728
|
},
|
|
694
|
-
fn: (
|
|
695
|
-
},
|
|
729
|
+
fn: (h) => new N({ url: h.url, headers: h.headers }).request({ method: h.method || "GET", body: h.body })
|
|
730
|
+
}, se = {
|
|
696
731
|
name: "exec_javascript",
|
|
697
732
|
description: "Execute commonjs javascript",
|
|
698
733
|
args: {
|
|
699
734
|
code: { type: "string", description: "CommonJS javascript", required: !0 }
|
|
700
735
|
},
|
|
701
|
-
fn: async (
|
|
702
|
-
const r =
|
|
736
|
+
fn: async (h) => {
|
|
737
|
+
const r = L(null), e = await C({ console: r }, h.code, !0).catch((t) => r.output.error.push(t));
|
|
703
738
|
return { ...r.output, return: e, stdout: void 0, stderr: void 0 };
|
|
704
739
|
}
|
|
705
|
-
},
|
|
740
|
+
}, oe = {
|
|
706
741
|
name: "exec_javascript",
|
|
707
742
|
description: "Execute commonjs javascript",
|
|
708
743
|
args: {
|
|
709
744
|
code: { type: "string", description: "CommonJS javascript", required: !0 }
|
|
710
745
|
},
|
|
711
|
-
fn: async (
|
|
712
|
-
},
|
|
746
|
+
fn: async (h) => ({ result: Q`python -c "${h.code}"` })
|
|
747
|
+
}, _e = {
|
|
713
748
|
name: "read_webpage",
|
|
714
749
|
description: "Extract clean, structured content from a webpage. Use after web_search to read specific URLs",
|
|
715
750
|
args: {
|
|
716
751
|
url: { type: "string", description: "URL to extract content from", required: !0 },
|
|
717
752
|
focus: { type: "string", description: 'Optional: What aspect to focus on (e.g., "pricing", "features", "contact info")' }
|
|
718
753
|
},
|
|
719
|
-
fn: async (
|
|
720
|
-
const r = await fetch(
|
|
721
|
-
throw new Error(`Failed to fetch: ${
|
|
722
|
-
}), e =
|
|
754
|
+
fn: async (h) => {
|
|
755
|
+
const r = await fetch(h.url, { headers: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)" } }).then((n) => n.text()).catch((n) => {
|
|
756
|
+
throw new Error(`Failed to fetch: ${n.message}`);
|
|
757
|
+
}), e = Y.load(r);
|
|
723
758
|
e('script, style, nav, footer, header, aside, iframe, noscript, [role="navigation"], [role="banner"], .ad, .ads, .cookie, .popup').remove();
|
|
724
759
|
const t = {
|
|
725
760
|
title: e('meta[property="og:title"]').attr("content") || e("title").text() || "",
|
|
726
761
|
description: e('meta[name="description"]').attr("content") || e('meta[property="og:description"]').attr("content") || ""
|
|
727
762
|
};
|
|
728
|
-
let
|
|
729
|
-
const
|
|
730
|
-
for (const
|
|
731
|
-
const
|
|
732
|
-
if (
|
|
733
|
-
|
|
763
|
+
let a = "";
|
|
764
|
+
const s = ["article", "main", '[role="main"]', ".content", ".post", ".entry", "body"];
|
|
765
|
+
for (const n of s) {
|
|
766
|
+
const c = e(n).first();
|
|
767
|
+
if (c.length && c.text().trim().length > 200) {
|
|
768
|
+
a = c.text();
|
|
734
769
|
break;
|
|
735
770
|
}
|
|
736
771
|
}
|
|
737
|
-
return
|
|
772
|
+
return a || (a = e("body").text()), a = a.replace(/\s+/g, " ").trim().slice(0, 8e3), { url: h.url, title: t.title.trim(), description: t.description.trim(), content: a, focus: h.focus };
|
|
738
773
|
}
|
|
739
|
-
},
|
|
774
|
+
}, Se = {
|
|
740
775
|
name: "web_search",
|
|
741
776
|
description: "Use duckduckgo (anonymous) to find find relevant online resources. Returns a list of URLs that works great with the `read_webpage` tool",
|
|
742
777
|
args: {
|
|
743
778
|
query: { type: "string", description: "Search string", required: !0 },
|
|
744
779
|
length: { type: "string", description: "Number of results to return", default: 5 }
|
|
745
780
|
},
|
|
746
|
-
fn: async (
|
|
747
|
-
const r = await fetch(`https://html.duckduckgo.com/html/?q=${encodeURIComponent(
|
|
781
|
+
fn: async (h) => {
|
|
782
|
+
const r = await fetch(`https://html.duckduckgo.com/html/?q=${encodeURIComponent(h.query)}`, {
|
|
748
783
|
headers: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)", "Accept-Language": "en-US,en;q=0.9" }
|
|
749
|
-
}).then((
|
|
784
|
+
}).then((s) => s.text());
|
|
750
785
|
let e, t = /<a .*?href="(.+?)".+?<\/a>/g;
|
|
751
|
-
const
|
|
786
|
+
const a = new J();
|
|
752
787
|
for (; (e = t.exec(r)) !== null; ) {
|
|
753
|
-
let
|
|
754
|
-
if (
|
|
788
|
+
let s = /uddg=(.+)&?/.exec(decodeURIComponent(e[1]))?.[1];
|
|
789
|
+
if (s && (s = decodeURIComponent(s)), s && a.add(s), a.size >= (h.length || 5)) break;
|
|
755
790
|
}
|
|
756
|
-
return
|
|
791
|
+
return a;
|
|
757
792
|
}
|
|
758
793
|
};
|
|
759
794
|
export {
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
795
|
+
we as Ai,
|
|
796
|
+
X as Anthropic,
|
|
797
|
+
te as Audio,
|
|
798
|
+
ne as CliTool,
|
|
799
|
+
be as DateTimeTool,
|
|
800
|
+
ke as ExecTool,
|
|
801
|
+
xe as FetchTool,
|
|
802
|
+
se as JSTool,
|
|
803
|
+
R as LLMProvider,
|
|
804
|
+
q as OpenAi,
|
|
805
|
+
oe as PythonTool,
|
|
806
|
+
_e as ReadWebpageTool,
|
|
807
|
+
re as Vision,
|
|
808
|
+
Se as WebSearchTool
|
|
774
809
|
};
|
|
775
810
|
//# sourceMappingURL=index.mjs.map
|