@ztimson/ai-utils 0.8.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/audio.d.ts +3 -2
- package/dist/index.js +24 -24
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +366 -338
- package/dist/index.mjs.map +1 -1
- package/dist/llm.d.ts +1 -0
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -1,40 +1,40 @@
|
|
|
1
|
-
import * as
|
|
2
|
-
import { tmpdir as
|
|
3
|
-
import { objectMap as
|
|
4
|
-
import { Anthropic as
|
|
5
|
-
import { OpenAI as
|
|
6
|
-
import { fileURLToPath as
|
|
7
|
-
import { join as
|
|
8
|
-
import { spawn as
|
|
9
|
-
import { mkdtempSync as
|
|
10
|
-
import
|
|
11
|
-
import * as
|
|
12
|
-
import
|
|
13
|
-
import { createWorker as
|
|
14
|
-
import * as
|
|
15
|
-
import { $ as
|
|
16
|
-
class
|
|
1
|
+
import * as U from "node:os";
|
|
2
|
+
import { tmpdir as W } from "node:os";
|
|
3
|
+
import { objectMap as q, JSONAttemptParse as _, findByProp as O, JSONSanitize as S, clean as z, Http as N, consoleInterceptor as C, fn as L, ASet as J } from "@ztimson/utils";
|
|
4
|
+
import { Anthropic as F } from "@anthropic-ai/sdk";
|
|
5
|
+
import { OpenAI as I } from "openai";
|
|
6
|
+
import { fileURLToPath as H } from "url";
|
|
7
|
+
import { join as D, dirname as G } from "path";
|
|
8
|
+
import { spawn as b, execSync as B } from "node:child_process";
|
|
9
|
+
import { mkdtempSync as K } from "node:fs";
|
|
10
|
+
import w from "node:fs/promises";
|
|
11
|
+
import * as $ from "node:path";
|
|
12
|
+
import v, { join as M } from "node:path";
|
|
13
|
+
import { createWorker as V } from "tesseract.js";
|
|
14
|
+
import * as Y from "cheerio";
|
|
15
|
+
import { $ as Z, $Sync as Q } from "@ztimson/node-utils";
|
|
16
|
+
class R {
|
|
17
17
|
}
|
|
18
|
-
class
|
|
18
|
+
class X extends R {
|
|
19
19
|
constructor(r, e, t) {
|
|
20
|
-
super(), this.ai = r, this.apiToken = e, this.model = t, this.client = new
|
|
20
|
+
super(), this.ai = r, this.apiToken = e, this.model = t, this.client = new F({ apiKey: e });
|
|
21
21
|
}
|
|
22
22
|
client;
|
|
23
23
|
toStandard(r) {
|
|
24
24
|
const e = Date.now(), t = [];
|
|
25
|
-
for (let
|
|
26
|
-
if (typeof
|
|
27
|
-
t.push({ timestamp: e, ...
|
|
25
|
+
for (let l of r)
|
|
26
|
+
if (typeof l.content == "string")
|
|
27
|
+
t.push({ timestamp: e, ...l });
|
|
28
28
|
else {
|
|
29
|
-
const
|
|
29
|
+
const n = l.content?.filter((s) => s.type == "text").map((s) => s.text).join(`
|
|
30
30
|
|
|
31
31
|
`);
|
|
32
|
-
|
|
32
|
+
n && t.push({ timestamp: e, role: l.role, content: n }), l.content.forEach((s) => {
|
|
33
33
|
if (s.type == "tool_use")
|
|
34
34
|
t.push({ timestamp: e, role: "tool", id: s.id, name: s.name, args: s.input, content: void 0 });
|
|
35
35
|
else if (s.type == "tool_result") {
|
|
36
|
-
const
|
|
37
|
-
|
|
36
|
+
const c = t.findLast((a) => a.id == s.tool_use_id);
|
|
37
|
+
c && (c[s.is_error ? "error" : "content"] = s.content);
|
|
38
38
|
}
|
|
39
39
|
});
|
|
40
40
|
}
|
|
@@ -55,78 +55,78 @@ class Y extends v {
|
|
|
55
55
|
}
|
|
56
56
|
ask(r, e = {}) {
|
|
57
57
|
const t = new AbortController();
|
|
58
|
-
return Object.assign(new Promise(async (
|
|
59
|
-
let
|
|
60
|
-
const s = e.tools || this.ai.options.llm?.tools || [],
|
|
58
|
+
return Object.assign(new Promise(async (l) => {
|
|
59
|
+
let n = this.fromStandard([...e.history || [], { role: "user", content: r, timestamp: Date.now() }]);
|
|
60
|
+
const s = e.tools || this.ai.options.llm?.tools || [], c = {
|
|
61
61
|
model: e.model || this.model,
|
|
62
62
|
max_tokens: e.max_tokens || this.ai.options.llm?.max_tokens || 4096,
|
|
63
63
|
system: e.system || this.ai.options.llm?.system || "",
|
|
64
64
|
temperature: e.temperature || this.ai.options.llm?.temperature || 0.7,
|
|
65
|
-
tools: s.map((
|
|
66
|
-
name:
|
|
67
|
-
description:
|
|
65
|
+
tools: s.map((m) => ({
|
|
66
|
+
name: m.name,
|
|
67
|
+
description: m.description,
|
|
68
68
|
input_schema: {
|
|
69
69
|
type: "object",
|
|
70
|
-
properties:
|
|
71
|
-
required:
|
|
70
|
+
properties: m.args ? q(m.args, (i, u) => ({ ...u, required: void 0 })) : {},
|
|
71
|
+
required: m.args ? Object.entries(m.args).filter((i) => i[1].required).map((i) => i[0]) : []
|
|
72
72
|
},
|
|
73
73
|
fn: void 0
|
|
74
74
|
})),
|
|
75
|
-
messages:
|
|
75
|
+
messages: n,
|
|
76
76
|
stream: !!e.stream
|
|
77
77
|
};
|
|
78
|
-
let
|
|
78
|
+
let a, o = !0;
|
|
79
79
|
do {
|
|
80
|
-
if (
|
|
80
|
+
if (a = await this.client.messages.create(c).catch((i) => {
|
|
81
81
|
throw i.message += `
|
|
82
82
|
|
|
83
83
|
Messages:
|
|
84
|
-
${JSON.stringify(
|
|
84
|
+
${JSON.stringify(n, null, 2)}`, i;
|
|
85
85
|
}), e.stream) {
|
|
86
|
-
|
|
86
|
+
o ? o = !1 : e.stream({ text: `
|
|
87
87
|
|
|
88
|
-
` }),
|
|
89
|
-
for await (const i of
|
|
88
|
+
` }), a.content = [];
|
|
89
|
+
for await (const i of a) {
|
|
90
90
|
if (t.signal.aborted) break;
|
|
91
91
|
if (i.type === "content_block_start")
|
|
92
|
-
i.content_block.type === "text" ?
|
|
92
|
+
i.content_block.type === "text" ? a.content.push({ type: "text", text: "" }) : i.content_block.type === "tool_use" && a.content.push({ type: "tool_use", id: i.content_block.id, name: i.content_block.name, input: "" });
|
|
93
93
|
else if (i.type === "content_block_delta")
|
|
94
94
|
if (i.delta.type === "text_delta") {
|
|
95
|
-
const
|
|
96
|
-
|
|
97
|
-
} else i.delta.type === "input_json_delta" && (
|
|
95
|
+
const u = i.delta.text;
|
|
96
|
+
a.content.at(-1).text += u, e.stream({ text: u });
|
|
97
|
+
} else i.delta.type === "input_json_delta" && (a.content.at(-1).input += i.delta.partial_json);
|
|
98
98
|
else if (i.type === "content_block_stop") {
|
|
99
|
-
const
|
|
100
|
-
|
|
99
|
+
const u = a.content.at(-1);
|
|
100
|
+
u.input != null && (u.input = u.input ? _(u.input, {}) : {});
|
|
101
101
|
} else if (i.type === "message_stop")
|
|
102
102
|
break;
|
|
103
103
|
}
|
|
104
104
|
}
|
|
105
|
-
const
|
|
106
|
-
if (
|
|
107
|
-
|
|
108
|
-
const i = await Promise.all(
|
|
109
|
-
const p = s.find(
|
|
110
|
-
if (e.stream && e.stream({ tool:
|
|
105
|
+
const m = a.content.filter((i) => i.type === "tool_use");
|
|
106
|
+
if (m.length && !t.signal.aborted) {
|
|
107
|
+
n.push({ role: "assistant", content: a.content });
|
|
108
|
+
const i = await Promise.all(m.map(async (u) => {
|
|
109
|
+
const p = s.find(O("name", u.name));
|
|
110
|
+
if (e.stream && e.stream({ tool: u.name }), !p) return { tool_use_id: u.id, is_error: !0, content: "Tool not found" };
|
|
111
111
|
try {
|
|
112
|
-
const
|
|
113
|
-
return { type: "tool_result", tool_use_id:
|
|
114
|
-
} catch (
|
|
115
|
-
return { type: "tool_result", tool_use_id:
|
|
112
|
+
const d = await p.fn(u.input, e?.stream, this.ai);
|
|
113
|
+
return { type: "tool_result", tool_use_id: u.id, content: S(d) };
|
|
114
|
+
} catch (d) {
|
|
115
|
+
return { type: "tool_result", tool_use_id: u.id, is_error: !0, content: d?.message || d?.toString() || "Unknown" };
|
|
116
116
|
}
|
|
117
117
|
}));
|
|
118
|
-
|
|
118
|
+
n.push({ role: "user", content: i }), c.messages = n;
|
|
119
119
|
}
|
|
120
|
-
} while (!t.signal.aborted &&
|
|
121
|
-
|
|
120
|
+
} while (!t.signal.aborted && a.content.some((m) => m.type === "tool_use"));
|
|
121
|
+
n.push({ role: "assistant", content: a.content.filter((m) => m.type == "text").map((m) => m.text).join(`
|
|
122
122
|
|
|
123
|
-
`) }),
|
|
123
|
+
`) }), n = this.toStandard(n), e.stream && e.stream({ done: !0 }), e.history && e.history.splice(0, e.history.length, ...n), l(n.at(-1)?.content);
|
|
124
124
|
}), { abort: () => t.abort() });
|
|
125
125
|
}
|
|
126
126
|
}
|
|
127
|
-
class
|
|
128
|
-
constructor(r, e, t,
|
|
129
|
-
super(), this.ai = r, this.host = e, this.token = t, this.model =
|
|
127
|
+
class A extends R {
|
|
128
|
+
constructor(r, e, t, l) {
|
|
129
|
+
super(), this.ai = r, this.host = e, this.token = t, this.model = l, this.client = new I(z({
|
|
130
130
|
baseURL: e,
|
|
131
131
|
apiKey: t
|
|
132
132
|
}));
|
|
@@ -136,17 +136,17 @@ class M extends v {
|
|
|
136
136
|
for (let e = 0; e < r.length; e++) {
|
|
137
137
|
const t = r[e];
|
|
138
138
|
if (t.role === "assistant" && t.tool_calls) {
|
|
139
|
-
const
|
|
139
|
+
const l = t.tool_calls.map((n) => ({
|
|
140
140
|
role: "tool",
|
|
141
|
-
id:
|
|
142
|
-
name:
|
|
143
|
-
args: _(
|
|
141
|
+
id: n.id,
|
|
142
|
+
name: n.function.name,
|
|
143
|
+
args: _(n.function.arguments, {}),
|
|
144
144
|
timestamp: t.timestamp
|
|
145
145
|
}));
|
|
146
|
-
r.splice(e, 1, ...
|
|
146
|
+
r.splice(e, 1, ...l), e += l.length - 1;
|
|
147
147
|
} else if (t.role === "tool" && t.content) {
|
|
148
|
-
const
|
|
149
|
-
|
|
148
|
+
const l = r.find((n) => t.tool_call_id == n.id);
|
|
149
|
+
l && (t.content.includes('"error":') ? l.error = t.content : l.content = t.content), r.splice(e, 1), e--;
|
|
150
150
|
}
|
|
151
151
|
r[e]?.timestamp || (r[e].timestamp = Date.now());
|
|
152
152
|
}
|
|
@@ -167,76 +167,76 @@ class M extends v {
|
|
|
167
167
|
content: t.error || t.content
|
|
168
168
|
});
|
|
169
169
|
else {
|
|
170
|
-
const { timestamp:
|
|
171
|
-
e.push(
|
|
170
|
+
const { timestamp: l, ...n } = t;
|
|
171
|
+
e.push(n);
|
|
172
172
|
}
|
|
173
173
|
return e;
|
|
174
174
|
}, []);
|
|
175
175
|
}
|
|
176
176
|
ask(r, e = {}) {
|
|
177
177
|
const t = new AbortController();
|
|
178
|
-
return Object.assign(new Promise(async (
|
|
178
|
+
return Object.assign(new Promise(async (l, n) => {
|
|
179
179
|
e.system && e.history?.[0]?.role != "system" && e.history?.splice(0, 0, { role: "system", content: e.system, timestamp: Date.now() });
|
|
180
180
|
let s = this.fromStandard([...e.history || [], { role: "user", content: r, timestamp: Date.now() }]);
|
|
181
|
-
const
|
|
181
|
+
const c = e.tools || this.ai.options.llm?.tools || [], a = {
|
|
182
182
|
model: e.model || this.model,
|
|
183
183
|
messages: s,
|
|
184
184
|
stream: !!e.stream,
|
|
185
185
|
max_tokens: e.max_tokens || this.ai.options.llm?.max_tokens || 4096,
|
|
186
186
|
temperature: e.temperature || this.ai.options.llm?.temperature || 0.7,
|
|
187
|
-
tools:
|
|
187
|
+
tools: c.map((i) => ({
|
|
188
188
|
type: "function",
|
|
189
189
|
function: {
|
|
190
190
|
name: i.name,
|
|
191
191
|
description: i.description,
|
|
192
192
|
parameters: {
|
|
193
193
|
type: "object",
|
|
194
|
-
properties: i.args ?
|
|
195
|
-
required: i.args ? Object.entries(i.args).filter((
|
|
194
|
+
properties: i.args ? q(i.args, (u, p) => ({ ...p, required: void 0 })) : {},
|
|
195
|
+
required: i.args ? Object.entries(i.args).filter((u) => u[1].required).map((u) => u[0]) : []
|
|
196
196
|
}
|
|
197
197
|
}
|
|
198
198
|
}))
|
|
199
199
|
};
|
|
200
|
-
let
|
|
200
|
+
let o, m = !0;
|
|
201
201
|
do {
|
|
202
|
-
if (
|
|
203
|
-
throw
|
|
202
|
+
if (o = await this.client.chat.completions.create(a).catch((u) => {
|
|
203
|
+
throw u.message += `
|
|
204
204
|
|
|
205
205
|
Messages:
|
|
206
|
-
${JSON.stringify(s, null, 2)}`,
|
|
206
|
+
${JSON.stringify(s, null, 2)}`, u;
|
|
207
207
|
}), e.stream) {
|
|
208
|
-
|
|
208
|
+
m ? m = !1 : e.stream({ text: `
|
|
209
209
|
|
|
210
|
-
` }),
|
|
211
|
-
for await (const
|
|
210
|
+
` }), o.choices = [{ message: { content: "", tool_calls: [] } }];
|
|
211
|
+
for await (const u of o) {
|
|
212
212
|
if (t.signal.aborted) break;
|
|
213
|
-
|
|
213
|
+
u.choices[0].delta.content && (o.choices[0].message.content += u.choices[0].delta.content, e.stream({ text: u.choices[0].delta.content })), u.choices[0].delta.tool_calls && (o.choices[0].message.tool_calls = u.choices[0].delta.tool_calls);
|
|
214
214
|
}
|
|
215
215
|
}
|
|
216
|
-
const i =
|
|
216
|
+
const i = o.choices[0].message.tool_calls || [];
|
|
217
217
|
if (i.length && !t.signal.aborted) {
|
|
218
|
-
s.push(
|
|
219
|
-
const
|
|
220
|
-
const
|
|
221
|
-
if (e.stream && e.stream({ tool: p.function.name }), !
|
|
218
|
+
s.push(o.choices[0].message);
|
|
219
|
+
const u = await Promise.all(i.map(async (p) => {
|
|
220
|
+
const d = c?.find(O("name", p.function.name));
|
|
221
|
+
if (e.stream && e.stream({ tool: p.function.name }), !d) return { role: "tool", tool_call_id: p.id, content: '{"error": "Tool not found"}' };
|
|
222
222
|
try {
|
|
223
|
-
const
|
|
224
|
-
return { role: "tool", tool_call_id: p.id, content:
|
|
225
|
-
} catch (
|
|
226
|
-
return { role: "tool", tool_call_id: p.id, content:
|
|
223
|
+
const f = _(p.function.arguments, {}), y = await d.fn(f, e.stream, this.ai);
|
|
224
|
+
return { role: "tool", tool_call_id: p.id, content: S(y) };
|
|
225
|
+
} catch (f) {
|
|
226
|
+
return { role: "tool", tool_call_id: p.id, content: S({ error: f?.message || f?.toString() || "Unknown" }) };
|
|
227
227
|
}
|
|
228
228
|
}));
|
|
229
|
-
s.push(...
|
|
229
|
+
s.push(...u), a.messages = s;
|
|
230
230
|
}
|
|
231
|
-
} while (!t.signal.aborted &&
|
|
232
|
-
s.push({ role: "assistant", content:
|
|
231
|
+
} while (!t.signal.aborted && o.choices?.[0]?.message?.tool_calls?.length);
|
|
232
|
+
s.push({ role: "assistant", content: o.choices[0].message.content || "" }), s = this.toStandard(s), e.stream && e.stream({ done: !0 }), e.history && e.history.splice(0, e.history.length, ...s), l(s.at(-1)?.content);
|
|
233
233
|
}), { abort: () => t.abort() });
|
|
234
234
|
}
|
|
235
235
|
}
|
|
236
|
-
class
|
|
236
|
+
class ee {
|
|
237
237
|
constructor(r) {
|
|
238
238
|
this.ai = r, r.options.llm?.models && Object.entries(r.options.llm.models).forEach(([e, t]) => {
|
|
239
|
-
this.defaultModel || (this.defaultModel = e), t.proto == "anthropic" ? this.models[e] = new
|
|
239
|
+
this.defaultModel || (this.defaultModel = e), t.proto == "anthropic" ? this.models[e] = new X(this.ai, t.token, e) : t.proto == "ollama" ? this.models[e] = new A(this.ai, t.host, "not-needed", e) : t.proto == "openai" && (this.models[e] = new A(this.ai, t.host || null, t.token, e));
|
|
240
240
|
});
|
|
241
241
|
}
|
|
242
242
|
defaultModel;
|
|
@@ -250,22 +250,22 @@ class Q {
|
|
|
250
250
|
ask(r, e = {}) {
|
|
251
251
|
const t = e.model || this.defaultModel;
|
|
252
252
|
if (!this.models[t]) throw new Error(`Model does not exist: ${t}`);
|
|
253
|
-
let
|
|
253
|
+
let l = () => {
|
|
254
254
|
};
|
|
255
|
-
return Object.assign(new Promise(async (
|
|
255
|
+
return Object.assign(new Promise(async (n) => {
|
|
256
256
|
if (e.history || (e.history = []), e.memory) {
|
|
257
257
|
e.system = (e.system || "") + `
|
|
258
258
|
You have passive persistent memory never make any mention of your memory capabilities and what you can/cannot remember
|
|
259
259
|
`;
|
|
260
|
-
const
|
|
261
|
-
const [
|
|
262
|
-
|
|
263
|
-
|
|
260
|
+
const c = async (o, m, i = 50) => {
|
|
261
|
+
const [u, p] = await Promise.all([
|
|
262
|
+
m ? this.embedding(m) : Promise.resolve(null),
|
|
263
|
+
o ? this.embedding(o) : Promise.resolve(null)
|
|
264
264
|
]);
|
|
265
|
-
return (e.memory || []).map((
|
|
266
|
-
},
|
|
267
|
-
|
|
268
|
-
` +
|
|
265
|
+
return (e.memory || []).map((d) => ({ ...d, score: u ? this.cosineSimilarity(d.embeddings[0], u[0].embedding) : 1 })).filter((d) => d.score >= 0.8).map((d) => ({ ...d, score: p ? this.cosineSimilarity(d.embeddings[1], p[0].embedding) : d.score })).filter((d) => d.score >= 0.2).toSorted((d, f) => d.score - f.score).slice(0, i);
|
|
266
|
+
}, a = await c(r);
|
|
267
|
+
a.length && e.history.push({ role: "assistant", content: `Things I remembered:
|
|
268
|
+
` + a.map((o) => `${o.owner}: ${o.fact}`).join(`
|
|
269
269
|
`) }), e.tools = [...e.tools || [], {
|
|
270
270
|
name: "read_memory",
|
|
271
271
|
description: "Check your long-term memory for more information",
|
|
@@ -274,32 +274,40 @@ You have passive persistent memory never make any mention of your memory capabil
|
|
|
274
274
|
query: { type: "string", description: "Search memory based on a query, can be used with or without subject argument" },
|
|
275
275
|
limit: { type: "number", description: "Result limit, default 5" }
|
|
276
276
|
},
|
|
277
|
-
fn: (
|
|
278
|
-
if (!
|
|
279
|
-
return
|
|
277
|
+
fn: (o) => {
|
|
278
|
+
if (!o.subject && !o.query) throw new Error("Either a subject or query argument is required");
|
|
279
|
+
return c(o.query, o.subject, o.limit || 5);
|
|
280
280
|
}
|
|
281
281
|
}];
|
|
282
282
|
}
|
|
283
283
|
const s = await this.models[t].ask(r, e);
|
|
284
284
|
if (e.memory) {
|
|
285
|
-
const
|
|
286
|
-
|
|
285
|
+
const c = e.history?.findIndex((a) => a.role == "assistant" && a.content.startsWith("Things I remembered:"));
|
|
286
|
+
c != null && c >= 0 && e.history?.splice(c, 1);
|
|
287
287
|
}
|
|
288
288
|
if (e.compress || e.memory) {
|
|
289
|
-
let
|
|
289
|
+
let c = null;
|
|
290
290
|
if (e.compress)
|
|
291
|
-
|
|
291
|
+
c = await this.ai.language.compressHistory(e.history, e.compress.max, e.compress.min, e), e.history.splice(0, e.history.length, ...c.history);
|
|
292
292
|
else {
|
|
293
|
-
const
|
|
294
|
-
|
|
293
|
+
const a = e.history?.findLastIndex((o) => o.role == "user") ?? -1;
|
|
294
|
+
c = await this.ai.language.compressHistory(a != -1 ? e.history.slice(a) : e.history, 0, 0, e);
|
|
295
295
|
}
|
|
296
296
|
if (e.memory) {
|
|
297
|
-
const
|
|
298
|
-
e.memory.splice(0, e.memory.length, ...
|
|
297
|
+
const a = e.memory.filter((o) => !c.memory.some((m) => this.cosineSimilarity(o.embeddings[1], m.embeddings[1]) > 0.8)).concat(c.memory);
|
|
298
|
+
e.memory.splice(0, e.memory.length, ...a);
|
|
299
299
|
}
|
|
300
300
|
}
|
|
301
|
-
return
|
|
302
|
-
}), { abort:
|
|
301
|
+
return n(s);
|
|
302
|
+
}), { abort: l });
|
|
303
|
+
}
|
|
304
|
+
async code(r, e) {
|
|
305
|
+
const t = await this.ask(r, { ...e, system: [
|
|
306
|
+
e?.system,
|
|
307
|
+
"Return your response in a code block"
|
|
308
|
+
].filter((n) => !!n).join(`
|
|
309
|
+
`) }), l = /```(?:.+)?\s*([\s\S]*?)```/.exec(t);
|
|
310
|
+
return l ? l[1].trim() : null;
|
|
303
311
|
}
|
|
304
312
|
/**
|
|
305
313
|
* Compress chat history to reduce context size
|
|
@@ -309,24 +317,24 @@ You have passive persistent memory never make any mention of your memory capabil
|
|
|
309
317
|
* @param {LLMRequest} options LLM options
|
|
310
318
|
* @returns {Promise<LLMMessage[]>} New chat history will summary at index 0
|
|
311
319
|
*/
|
|
312
|
-
async compressHistory(r, e, t,
|
|
320
|
+
async compressHistory(r, e, t, l) {
|
|
313
321
|
if (this.estimateTokens(r) < e) return { history: r, memory: [] };
|
|
314
|
-
let
|
|
315
|
-
for (let
|
|
316
|
-
if (s += this.estimateTokens(
|
|
322
|
+
let n = 0, s = 0;
|
|
323
|
+
for (let d of r.toReversed())
|
|
324
|
+
if (s += this.estimateTokens(d.content), s < t) n++;
|
|
317
325
|
else break;
|
|
318
|
-
if (r.length <=
|
|
319
|
-
const
|
|
326
|
+
if (r.length <= n) return { history: r, memory: [] };
|
|
327
|
+
const c = r[0].role == "system" ? r[0] : null, a = n == 0 ? [] : r.slice(-n), o = (n == 0 ? r : r.slice(0, -n)).filter((d) => d.role === "assistant" || d.role === "user"), m = await this.json(o.map((d) => `${d.role}: ${d.content}`).join(`
|
|
320
328
|
|
|
321
329
|
`), "{summary: string, facts: [[subject, fact]]}", {
|
|
322
330
|
system: "Create the smallest summary possible, no more than 500 tokens. Create a list of NEW facts (split by subject [pro]noun and fact) about what you learned from this conversation that you didn't already know or get from a tool call or system prompt. Focus only on new information about people, topics, or facts. Avoid generating facts about the AI.",
|
|
323
|
-
model:
|
|
324
|
-
temperature:
|
|
325
|
-
}), i = /* @__PURE__ */ new Date(),
|
|
326
|
-
const y = await Promise.all([this.embedding(
|
|
327
|
-
return { owner:
|
|
328
|
-
})), p = [{ role: "assistant", content: `Conversation Summary: ${
|
|
329
|
-
return
|
|
331
|
+
model: l?.model,
|
|
332
|
+
temperature: l?.temperature || 0.3
|
|
333
|
+
}), i = /* @__PURE__ */ new Date(), u = await Promise.all((m?.facts || [])?.map(async ([d, f]) => {
|
|
334
|
+
const y = await Promise.all([this.embedding(d), this.embedding(`${d}: ${f}`)]);
|
|
335
|
+
return { owner: d, fact: f, embeddings: [y[0][0].embedding, y[1][0].embedding], timestamp: i };
|
|
336
|
+
})), p = [{ role: "assistant", content: `Conversation Summary: ${m?.summary}`, timestamp: Date.now() }, ...a];
|
|
337
|
+
return c && p.splice(0, 0, c), { history: p, memory: u };
|
|
330
338
|
}
|
|
331
339
|
/**
|
|
332
340
|
* Compare the difference between embeddings (calculates the angle between two vectors)
|
|
@@ -336,10 +344,10 @@ You have passive persistent memory never make any mention of your memory capabil
|
|
|
336
344
|
*/
|
|
337
345
|
cosineSimilarity(r, e) {
|
|
338
346
|
if (r.length !== e.length) throw new Error("Vectors must be same length");
|
|
339
|
-
let t = 0,
|
|
340
|
-
for (let
|
|
341
|
-
t += r[
|
|
342
|
-
const s = Math.sqrt(
|
|
347
|
+
let t = 0, l = 0, n = 0;
|
|
348
|
+
for (let c = 0; c < r.length; c++)
|
|
349
|
+
t += r[c] * e[c], l += r[c] * r[c], n += e[c] * e[c];
|
|
350
|
+
const s = Math.sqrt(l) * Math.sqrt(n);
|
|
343
351
|
return s === 0 ? 0 : t / s;
|
|
344
352
|
}
|
|
345
353
|
/**
|
|
@@ -350,25 +358,25 @@ You have passive persistent memory never make any mention of your memory capabil
|
|
|
350
358
|
* @returns {string[]} Chunked strings
|
|
351
359
|
*/
|
|
352
360
|
chunk(r, e = 500, t = 50) {
|
|
353
|
-
const
|
|
354
|
-
const
|
|
355
|
-
return typeof i == "object" && !Array.isArray(i) ?
|
|
356
|
-
}) : [], s = (typeof r == "object" ?
|
|
357
|
-
`)).flatMap((
|
|
358
|
-
`]),
|
|
359
|
-
for (let
|
|
360
|
-
let
|
|
361
|
-
for (;
|
|
362
|
-
const
|
|
363
|
-
if (this.estimateTokens(
|
|
364
|
-
`)) > e &&
|
|
365
|
-
|
|
361
|
+
const l = (a, o = "") => a ? Object.entries(a).flatMap(([m, i]) => {
|
|
362
|
+
const u = o ? `${o}${isNaN(+m) ? `.${m}` : `[${m}]`}` : m;
|
|
363
|
+
return typeof i == "object" && !Array.isArray(i) ? l(i, u) : `${u}: ${Array.isArray(i) ? i.join(", ") : i}`;
|
|
364
|
+
}) : [], s = (typeof r == "object" ? l(r) : r.split(`
|
|
365
|
+
`)).flatMap((a) => [...a.split(/\s+/).filter(Boolean), `
|
|
366
|
+
`]), c = [];
|
|
367
|
+
for (let a = 0; a < s.length; ) {
|
|
368
|
+
let o = "", m = a;
|
|
369
|
+
for (; m < s.length; ) {
|
|
370
|
+
const u = o + (o ? " " : "") + s[m];
|
|
371
|
+
if (this.estimateTokens(u.replace(/\s*\n\s*/g, `
|
|
372
|
+
`)) > e && o) break;
|
|
373
|
+
o = u, m++;
|
|
366
374
|
}
|
|
367
|
-
const i =
|
|
375
|
+
const i = o.replace(/\s*\n\s*/g, `
|
|
368
376
|
`).trim();
|
|
369
|
-
i &&
|
|
377
|
+
i && c.push(i), a = Math.max(m - t, m === a ? a + 1 : m);
|
|
370
378
|
}
|
|
371
|
-
return
|
|
379
|
+
return c;
|
|
372
380
|
}
|
|
373
381
|
/**
|
|
374
382
|
* Create a vector representation of a string
|
|
@@ -377,39 +385,39 @@ You have passive persistent memory never make any mention of your memory capabil
|
|
|
377
385
|
* @returns {Promise<Awaited<{index: number, embedding: number[], text: string, tokens: number}>[]>} Chunked embeddings
|
|
378
386
|
*/
|
|
379
387
|
embedding(r, e = {}) {
|
|
380
|
-
let { maxTokens: t = 500, overlapTokens:
|
|
388
|
+
let { maxTokens: t = 500, overlapTokens: l = 50 } = e, n = !1;
|
|
381
389
|
const s = () => {
|
|
382
|
-
|
|
383
|
-
},
|
|
384
|
-
if (
|
|
385
|
-
const
|
|
386
|
-
|
|
390
|
+
n = !0;
|
|
391
|
+
}, c = (o) => new Promise((m, i) => {
|
|
392
|
+
if (n) return i(new Error("Aborted"));
|
|
393
|
+
const u = [
|
|
394
|
+
D(G(H(import.meta.url)), "embedder.js"),
|
|
387
395
|
this.ai.options.path,
|
|
388
396
|
this.ai.options?.embedder || "bge-small-en-v1.5"
|
|
389
|
-
], p =
|
|
390
|
-
p.stdin.write(
|
|
391
|
-
let
|
|
392
|
-
p.stdout.on("data", (
|
|
393
|
-
if (
|
|
394
|
-
if (
|
|
397
|
+
], p = b("node", u, { stdio: ["pipe", "pipe", "ignore"] });
|
|
398
|
+
p.stdin.write(o), p.stdin.end();
|
|
399
|
+
let d = "";
|
|
400
|
+
p.stdout.on("data", (f) => d += f.toString()), p.on("close", (f) => {
|
|
401
|
+
if (n) return i(new Error("Aborted"));
|
|
402
|
+
if (f === 0)
|
|
395
403
|
try {
|
|
396
|
-
const y = JSON.parse(
|
|
397
|
-
|
|
404
|
+
const y = JSON.parse(d);
|
|
405
|
+
m(y.embedding);
|
|
398
406
|
} catch {
|
|
399
407
|
i(new Error("Failed to parse embedding output"));
|
|
400
408
|
}
|
|
401
409
|
else
|
|
402
|
-
i(new Error(`Embedder process exited with code ${
|
|
410
|
+
i(new Error(`Embedder process exited with code ${f}`));
|
|
403
411
|
}), p.on("error", i);
|
|
404
|
-
}),
|
|
405
|
-
const
|
|
406
|
-
for (let i = 0; i <
|
|
407
|
-
const
|
|
408
|
-
|
|
412
|
+
}), a = (async () => {
|
|
413
|
+
const o = this.chunk(r, t, l), m = [];
|
|
414
|
+
for (let i = 0; i < o.length && !n; i++) {
|
|
415
|
+
const u = o[i], p = await c(u);
|
|
416
|
+
m.push({ index: i, embedding: p, text: u, tokens: this.estimateTokens(u) });
|
|
409
417
|
}
|
|
410
|
-
return
|
|
418
|
+
return m;
|
|
411
419
|
})();
|
|
412
|
-
return Object.assign(
|
|
420
|
+
return Object.assign(a, { abort: s });
|
|
413
421
|
}
|
|
414
422
|
/**
|
|
415
423
|
* Estimate variable as tokens
|
|
@@ -428,8 +436,8 @@ You have passive persistent memory never make any mention of your memory capabil
|
|
|
428
436
|
*/
|
|
429
437
|
fuzzyMatch(r, ...e) {
|
|
430
438
|
if (e.length < 2) throw new Error("Requires at least 2 strings to compare");
|
|
431
|
-
const t = (s,
|
|
432
|
-
return { avg:
|
|
439
|
+
const t = (s, c = 10) => s.toLowerCase().split("").map((a, o) => a.charCodeAt(0) * (o + 1) % c / c).slice(0, c), l = t(r), n = e.map((s) => t(s)).map((s) => this.cosineSimilarity(l, s));
|
|
440
|
+
return { avg: n.reduce((s, c) => s + c, 0) / n.length, max: Math.max(...n), similarities: n };
|
|
433
441
|
}
|
|
434
442
|
/**
|
|
435
443
|
* Ask a question with JSON response
|
|
@@ -439,14 +447,15 @@ You have passive persistent memory never make any mention of your memory capabil
|
|
|
439
447
|
* @returns {Promise<{} | {} | RegExpExecArray | null>}
|
|
440
448
|
*/
|
|
441
449
|
async json(r, e, t) {
|
|
442
|
-
|
|
443
|
-
|
|
450
|
+
const l = await this.code(r, { ...t, system: [
|
|
451
|
+
t?.system,
|
|
452
|
+
`Only respond using JSON matching this schema:
|
|
444
453
|
\`\`\`json
|
|
445
454
|
${e}
|
|
446
|
-
\`\`\``
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
return _(
|
|
455
|
+
\`\`\``
|
|
456
|
+
].filter((n) => !!n).join(`
|
|
457
|
+
`) });
|
|
458
|
+
return l ? _(l, {}) : null;
|
|
450
459
|
}
|
|
451
460
|
/**
|
|
452
461
|
* Create a summary of some text
|
|
@@ -459,7 +468,7 @@ ${e}
|
|
|
459
468
|
return this.ask(r, { system: `Generate a brief summary <= ${e} tokens. Output nothing else`, temperature: 0.3, ...t });
|
|
460
469
|
}
|
|
461
470
|
}
|
|
462
|
-
class
|
|
471
|
+
class te {
|
|
463
472
|
constructor(r) {
|
|
464
473
|
this.ai = r, r.options.whisper && (this.whisperModel = r.options.asr || "ggml-base.en.bin", this.downloadAsrModel()), this.pyannote = `
|
|
465
474
|
import sys
|
|
@@ -481,147 +490,166 @@ print(json.dumps(segments))
|
|
|
481
490
|
downloads = {};
|
|
482
491
|
pyannote;
|
|
483
492
|
whisperModel;
|
|
493
|
+
async addPunctuation(r, e, t = 150) {
|
|
494
|
+
const l = (s) => {
|
|
495
|
+
if (s = s.toLowerCase().replace(/[^a-z]/g, ""), s.length <= 3) return 1;
|
|
496
|
+
const c = s.match(/[aeiouy]+/g);
|
|
497
|
+
let a = c ? c.length : 1;
|
|
498
|
+
return s.endsWith("e") && a--, Math.max(1, a);
|
|
499
|
+
};
|
|
500
|
+
let n = "";
|
|
501
|
+
return r.transcription.filter((s, c) => {
|
|
502
|
+
let a = !1;
|
|
503
|
+
const o = r.transcription[c - 1], m = r.transcription[c + 1];
|
|
504
|
+
return !s.text && m ? (m.offsets.from = s.offsets.from, m.timestamps.from = s.offsets.from) : s.text && s.text[0] != " " && o && (o.offsets.to = s.offsets.to, o.timestamps.to = s.timestamps.to, o.text += s.text, a = !0), !!s.text && !a;
|
|
505
|
+
}).forEach((s) => {
|
|
506
|
+
const c = /^[A-Z]/.test(s.text.trim()), a = s.offsets.to - s.offsets.from, m = l(s.text.trim()) * t;
|
|
507
|
+
c && a > m * 2 && s.text[0] == " " && (n += "."), n += s.text;
|
|
508
|
+
}), e ? this.ai.language.ask(n, {
|
|
509
|
+
system: "Remove any misplaced punctuation from the following ASR transcript using the replace tool. Avoid modifying words unless there is an obvious typo",
|
|
510
|
+
temperature: 0.1,
|
|
511
|
+
tools: [{
|
|
512
|
+
name: "replace",
|
|
513
|
+
description: "Use find and replace to fix errors",
|
|
514
|
+
args: {
|
|
515
|
+
find: { type: "string", description: "Text to find", required: !0 },
|
|
516
|
+
replace: { type: "string", description: "Text to replace", required: !0 }
|
|
517
|
+
},
|
|
518
|
+
fn: (s) => n = n.replace(s.find, s.replace)
|
|
519
|
+
}]
|
|
520
|
+
}).then(() => n) : n.trim();
|
|
521
|
+
}
|
|
522
|
+
async diarizeTranscript(r, e, t) {
|
|
523
|
+
const l = /* @__PURE__ */ new Map();
|
|
524
|
+
let n = 0;
|
|
525
|
+
e.forEach((d) => {
|
|
526
|
+
l.has(d.speaker) || l.set(d.speaker, ++n);
|
|
527
|
+
});
|
|
528
|
+
const s = await this.addPunctuation(r, t), c = s.match(/[^.!?]+[.!?]+/g) || [s], a = r.transcription.filter((d) => d.text.trim()), o = c.map((d) => {
|
|
529
|
+
if (d = d.trim(), !d) return null;
|
|
530
|
+
const f = d.toLowerCase().replace(/[^\w\s]/g, "").split(/\s+/), y = /* @__PURE__ */ new Map();
|
|
531
|
+
f.forEach((k) => {
|
|
532
|
+
const x = a.find((g) => k === g.text.trim().toLowerCase().replace(/[^\w]/g, ""));
|
|
533
|
+
if (!x) return;
|
|
534
|
+
const E = x.offsets.from / 1e3, P = e.find((g) => E >= g.start && E <= g.end);
|
|
535
|
+
if (P) {
|
|
536
|
+
const g = l.get(P.speaker);
|
|
537
|
+
y.set(g, (y.get(g) || 0) + 1);
|
|
538
|
+
}
|
|
539
|
+
});
|
|
540
|
+
let j = 1, T = 0;
|
|
541
|
+
return y.forEach((k, x) => {
|
|
542
|
+
k > T && (T = k, j = x);
|
|
543
|
+
}), { speaker: j, text: d };
|
|
544
|
+
}).filter((d) => d !== null), m = [];
|
|
545
|
+
o.forEach((d) => {
|
|
546
|
+
const f = m[m.length - 1];
|
|
547
|
+
f && f.speaker === d.speaker ? f.text += " " + d.text : m.push({ ...d });
|
|
548
|
+
});
|
|
549
|
+
let i = m.map((d) => `[Speaker ${d.speaker}]: ${d.text}`).join(`
|
|
550
|
+
`).trim();
|
|
551
|
+
if (!t) return i;
|
|
552
|
+
let u = this.ai.language.chunk(i, 500, 0);
|
|
553
|
+
u.length > 4 && (u = [...u.slice(0, 3), u.at(-1)]);
|
|
554
|
+
const p = await this.ai.language.json(u.join(`
|
|
555
|
+
`), '{1: "Detected Name", 2: "Second Name"}', {
|
|
556
|
+
system: "Use the following transcript to identify speakers. Only identify speakers you are positive about, dont mention speakers you are unsure about in your response",
|
|
557
|
+
temperature: 0.1
|
|
558
|
+
});
|
|
559
|
+
return Object.entries(p).forEach(([d, f]) => i = i.replaceAll(`[Speaker ${d}]`, `[${f}]`)), i;
|
|
560
|
+
}
|
|
484
561
|
runAsr(r, e = {}) {
|
|
485
562
|
let t;
|
|
486
|
-
const
|
|
487
|
-
this.downloadAsrModel(e.model).then((
|
|
563
|
+
const l = new Promise((n, s) => {
|
|
564
|
+
this.downloadAsrModel(e.model).then((c) => {
|
|
488
565
|
if (e.diarization) {
|
|
489
|
-
let
|
|
490
|
-
t =
|
|
566
|
+
let a = $.join($.dirname(r), "transcript");
|
|
567
|
+
t = b(
|
|
491
568
|
this.ai.options.whisper,
|
|
492
|
-
["-m",
|
|
569
|
+
["-m", c, "-f", r, "-np", "-ml", "1", "-oj", "-of", a],
|
|
493
570
|
{ stdio: ["ignore", "ignore", "pipe"] }
|
|
494
|
-
), t.on("error", (
|
|
495
|
-
if (
|
|
496
|
-
|
|
571
|
+
), t.on("error", (o) => s(o)), t.on("close", async (o) => {
|
|
572
|
+
if (o === 0) {
|
|
573
|
+
a = await w.readFile(a + ".json", "utf-8"), w.rm(a + ".json").catch(() => {
|
|
497
574
|
});
|
|
498
575
|
try {
|
|
499
|
-
|
|
576
|
+
n(JSON.parse(a));
|
|
500
577
|
} catch {
|
|
501
578
|
s(new Error("Failed to parse whisper JSON"));
|
|
502
579
|
}
|
|
503
580
|
} else
|
|
504
|
-
s(new Error(`Exit code ${
|
|
581
|
+
s(new Error(`Exit code ${o}`));
|
|
505
582
|
});
|
|
506
583
|
} else {
|
|
507
|
-
let
|
|
508
|
-
t =
|
|
509
|
-
|
|
584
|
+
let a = "";
|
|
585
|
+
t = b(this.ai.options.whisper, ["-m", c, "-f", r, "-np", "-nt"]), t.on("error", (o) => s(o)), t.stdout.on("data", (o) => a += o.toString()), t.on("close", async (o) => {
|
|
586
|
+
o === 0 ? n(a.trim() || null) : s(new Error(`Exit code ${o}`));
|
|
510
587
|
});
|
|
511
588
|
}
|
|
512
589
|
});
|
|
513
590
|
});
|
|
514
|
-
return Object.assign(
|
|
591
|
+
return Object.assign(l, { abort: () => t?.kill("SIGTERM") });
|
|
515
592
|
}
|
|
516
593
|
runDiarization(r) {
|
|
517
594
|
let e = !1, t = () => {
|
|
518
595
|
e = !0;
|
|
519
596
|
};
|
|
520
|
-
const
|
|
521
|
-
const
|
|
522
|
-
|
|
523
|
-
}),
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
]).then((async ([s,
|
|
597
|
+
const l = (s) => new Promise((c) => {
|
|
598
|
+
const a = b(s, ["-W", "ignore", "-c", "import pyannote.audio"]);
|
|
599
|
+
a.on("close", (o) => c(o === 0)), a.on("error", () => c(!1));
|
|
600
|
+
}), n = Promise.all([
|
|
601
|
+
l("python"),
|
|
602
|
+
l("python3")
|
|
603
|
+
]).then((async ([s, c]) => {
|
|
527
604
|
if (e) return;
|
|
528
|
-
if (!s && !
|
|
529
|
-
const
|
|
530
|
-
return new Promise((
|
|
605
|
+
if (!s && !c) throw new Error("Pyannote is not installed: pip install pyannote.audio");
|
|
606
|
+
const a = c ? "python3" : "python";
|
|
607
|
+
return new Promise((o, m) => {
|
|
531
608
|
if (e) return;
|
|
532
609
|
let i = "";
|
|
533
|
-
const
|
|
534
|
-
|
|
610
|
+
const u = b(a, ["-W", "ignore", "-c", this.pyannote, r]);
|
|
611
|
+
u.stdout.on("data", (p) => i += p.toString()), u.stderr.on("data", (p) => console.error(p.toString())), u.on("close", (p) => {
|
|
535
612
|
if (p === 0)
|
|
536
613
|
try {
|
|
537
|
-
|
|
614
|
+
o(JSON.parse(i));
|
|
538
615
|
} catch {
|
|
539
|
-
|
|
616
|
+
m(new Error("Failed to parse diarization output"));
|
|
540
617
|
}
|
|
541
618
|
else
|
|
542
|
-
|
|
543
|
-
}),
|
|
619
|
+
m(new Error(`Python process exited with code ${p}`));
|
|
620
|
+
}), u.on("error", m), t = () => u.kill("SIGTERM");
|
|
544
621
|
});
|
|
545
622
|
}));
|
|
546
|
-
return Object.assign(
|
|
547
|
-
}
|
|
548
|
-
async combineSpeakerTranscript(r, e, t) {
|
|
549
|
-
const c = /* @__PURE__ */ new Map();
|
|
550
|
-
let o = 0;
|
|
551
|
-
t.forEach((n) => {
|
|
552
|
-
c.has(n.speaker) || c.set(n.speaker, ++o);
|
|
553
|
-
});
|
|
554
|
-
const s = r.match(/[^.!?]+[.!?]+/g) || [r], l = [];
|
|
555
|
-
return s.forEach((n) => {
|
|
556
|
-
if (n = n.trim(), !n) return;
|
|
557
|
-
const a = n.toLowerCase().replace(/[^\w\s]/g, "").split(/\s+/);
|
|
558
|
-
let d = 1 / 0;
|
|
559
|
-
const i = [];
|
|
560
|
-
if (e.transcription.forEach((h) => {
|
|
561
|
-
const y = h.text.trim().toLowerCase();
|
|
562
|
-
if (a.some((g) => y.includes(g))) {
|
|
563
|
-
const g = h.offsets.from / 1e3, w = h.offsets.to / 1e3;
|
|
564
|
-
i.push({ start: g, end: w }), g < d && (d = g);
|
|
565
|
-
}
|
|
566
|
-
}), d === 1 / 0) return;
|
|
567
|
-
const m = /* @__PURE__ */ new Map();
|
|
568
|
-
i.forEach((h) => {
|
|
569
|
-
t.forEach((y) => {
|
|
570
|
-
const g = Math.max(0, Math.min(h.end, y.end) - Math.max(h.start, y.start)), w = h.end - h.start;
|
|
571
|
-
if (w > 0) {
|
|
572
|
-
const A = g / w, S = c.get(y.speaker);
|
|
573
|
-
m.set(S, (m.get(S) || 0) + A);
|
|
574
|
-
}
|
|
575
|
-
});
|
|
576
|
-
});
|
|
577
|
-
let p = 1, u = 0;
|
|
578
|
-
m.forEach((h, y) => {
|
|
579
|
-
h > u && (u = h, p = y);
|
|
580
|
-
}), l.push(`[Speaker ${p}]: ${n}`);
|
|
581
|
-
}), l.join(`
|
|
582
|
-
`).trim();
|
|
623
|
+
return Object.assign(n, { abort: t });
|
|
583
624
|
}
|
|
584
625
|
asr(r, e = {}) {
|
|
585
626
|
if (!this.ai.options.whisper) throw new Error("Whisper not configured");
|
|
586
|
-
const t =
|
|
587
|
-
|
|
588
|
-
const
|
|
589
|
-
})
|
|
590
|
-
|
|
591
|
-
|
|
627
|
+
const t = M(K(M(W(), "audio-")), "converted.wav");
|
|
628
|
+
B(`ffmpeg -i "${r}" -ar 16000 -ac 1 -f wav "${t}"`, { stdio: "ignore" });
|
|
629
|
+
const l = () => w.rm(v.dirname(t), { recursive: !0, force: !0 }).catch(() => {
|
|
630
|
+
});
|
|
631
|
+
if (!e.diarization) return this.runAsr(t, { model: e.model });
|
|
632
|
+
const n = this.runAsr(t, { model: e.model, diarization: !0 }), s = this.runDiarization(t);
|
|
633
|
+
let c = !1, a = () => {
|
|
634
|
+
c = !0, n.abort(), s.abort(), l();
|
|
592
635
|
};
|
|
593
|
-
const
|
|
594
|
-
if (i.status == "rejected") throw new Error(`Whisper.cpp punctuated:
|
|
595
|
-
` + i.reason);
|
|
636
|
+
const o = Promise.allSettled([n, s]).then(async ([m, i]) => {
|
|
596
637
|
if (m.status == "rejected") throw new Error(`Whisper.cpp timestamps:
|
|
597
638
|
` + m.reason);
|
|
598
|
-
if (
|
|
599
|
-
` +
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
if (!this.ai.language.defaultModel) throw new Error("Configure an LLM for advanced ASR speaker detection");
|
|
604
|
-
let h = this.ai.language.chunk(u, 500, 0);
|
|
605
|
-
h.length > 4 && (h = [...h.slice(0, 3), h.at(-1)]);
|
|
606
|
-
const y = await this.ai.language.json(h.join(`
|
|
607
|
-
`), '{1: "Detected Name", 2: "Second Name"}', {
|
|
608
|
-
system: "Use the following transcript to identify speakers. Only identify speakers you are positive about, dont mention speakers you are unsure about in your response",
|
|
609
|
-
temperature: 0.1
|
|
610
|
-
});
|
|
611
|
-
Object.entries(y).forEach(([g, w]) => u = u.replaceAll(`[Speaker ${g}]`, `[${w}]`));
|
|
612
|
-
}
|
|
613
|
-
return u;
|
|
614
|
-
}).finally(() => c());
|
|
615
|
-
return Object.assign(d, { abort: a });
|
|
639
|
+
if (i.status == "rejected") throw new Error(`Pyannote:
|
|
640
|
+
` + i.reason);
|
|
641
|
+
return c || !e.diarization ? m.value : this.diarizeTranscript(m.value, i.value, e.diarization == "llm");
|
|
642
|
+
}).finally(() => l());
|
|
643
|
+
return Object.assign(o, { abort: a });
|
|
616
644
|
}
|
|
617
645
|
async downloadAsrModel(r = this.whisperModel) {
|
|
618
646
|
if (!this.ai.options.whisper) throw new Error("Whisper not configured");
|
|
619
647
|
r.endsWith(".bin") || (r += ".bin");
|
|
620
|
-
const e =
|
|
621
|
-
return await
|
|
648
|
+
const e = v.join(this.ai.options.path, r);
|
|
649
|
+
return await w.stat(e).then(() => !0).catch(() => !1) ? e : this.downloads[r] ? this.downloads[r] : (this.downloads[r] = fetch(`https://huggingface.co/ggerganov/whisper.cpp/resolve/main/${r}`).then((t) => t.arrayBuffer()).then((t) => Buffer.from(t)).then(async (t) => (await w.writeFile(e, t), delete this.downloads[r], e)), this.downloads[r]);
|
|
622
650
|
}
|
|
623
651
|
}
|
|
624
|
-
class
|
|
652
|
+
class re {
|
|
625
653
|
constructor(r) {
|
|
626
654
|
this.ai = r;
|
|
627
655
|
}
|
|
@@ -632,17 +660,17 @@ class Z {
|
|
|
632
660
|
*/
|
|
633
661
|
ocr(r) {
|
|
634
662
|
let e;
|
|
635
|
-
const t = new Promise(async (
|
|
636
|
-
e = await
|
|
637
|
-
const { data:
|
|
638
|
-
await e.terminate(),
|
|
663
|
+
const t = new Promise(async (l) => {
|
|
664
|
+
e = await V(this.ai.options.ocr || "eng", 2, { cachePath: this.ai.options.path });
|
|
665
|
+
const { data: n } = await e.recognize(r);
|
|
666
|
+
await e.terminate(), l(n.text.trim() || null);
|
|
639
667
|
});
|
|
640
668
|
return Object.assign(t, { abort: () => e?.terminate() });
|
|
641
669
|
}
|
|
642
670
|
}
|
|
643
|
-
class
|
|
671
|
+
class we {
|
|
644
672
|
constructor(r) {
|
|
645
|
-
this.options = r, r.path || (r.path =
|
|
673
|
+
this.options = r, r.path || (r.path = U.tmpdir()), process.env.TRANSFORMERS_CACHE = r.path, this.audio = new te(this), this.language = new ee(this), this.vision = new re(this);
|
|
646
674
|
}
|
|
647
675
|
/** Audio processing AI */
|
|
648
676
|
audio;
|
|
@@ -651,38 +679,38 @@ class fe {
|
|
|
651
679
|
/** Vision processing AI */
|
|
652
680
|
vision;
|
|
653
681
|
}
|
|
654
|
-
const
|
|
682
|
+
const se = {
|
|
655
683
|
name: "cli",
|
|
656
684
|
description: "Use the command line interface, returns any output",
|
|
657
685
|
args: { command: { type: "string", description: "Command to run", required: !0 } },
|
|
658
|
-
fn: (
|
|
659
|
-
},
|
|
686
|
+
fn: (h) => Z`${h.command}`
|
|
687
|
+
}, be = {
|
|
660
688
|
name: "get_datetime",
|
|
661
689
|
description: "Get current UTC date / time",
|
|
662
690
|
args: {},
|
|
663
691
|
fn: async () => (/* @__PURE__ */ new Date()).toUTCString()
|
|
664
|
-
},
|
|
692
|
+
}, ke = {
|
|
665
693
|
name: "exec",
|
|
666
694
|
description: "Run code/scripts",
|
|
667
695
|
args: {
|
|
668
696
|
language: { type: "string", description: "Execution language", enum: ["cli", "node", "python"], required: !0 },
|
|
669
697
|
code: { type: "string", description: "Code to execute", required: !0 }
|
|
670
698
|
},
|
|
671
|
-
fn: async (
|
|
699
|
+
fn: async (h, r, e) => {
|
|
672
700
|
try {
|
|
673
|
-
switch (
|
|
701
|
+
switch (h.type) {
|
|
674
702
|
case "bash":
|
|
675
|
-
return await
|
|
703
|
+
return await se.fn({ command: h.code }, r, e);
|
|
676
704
|
case "node":
|
|
677
|
-
return await
|
|
705
|
+
return await ne.fn({ code: h.code }, r, e);
|
|
678
706
|
case "python":
|
|
679
|
-
return await
|
|
707
|
+
return await oe.fn({ code: h.code }, r, e);
|
|
680
708
|
}
|
|
681
709
|
} catch (t) {
|
|
682
710
|
return { error: t?.message || t.toString() };
|
|
683
711
|
}
|
|
684
712
|
}
|
|
685
|
-
},
|
|
713
|
+
}, xe = {
|
|
686
714
|
name: "fetch",
|
|
687
715
|
description: "Make HTTP request to URL",
|
|
688
716
|
args: {
|
|
@@ -691,85 +719,85 @@ const ee = {
|
|
|
691
719
|
headers: { type: "object", description: "HTTP headers to send", default: {} },
|
|
692
720
|
body: { type: "object", description: "HTTP body to send" }
|
|
693
721
|
},
|
|
694
|
-
fn: (
|
|
695
|
-
},
|
|
722
|
+
fn: (h) => new N({ url: h.url, headers: h.headers }).request({ method: h.method || "GET", body: h.body })
|
|
723
|
+
}, ne = {
|
|
696
724
|
name: "exec_javascript",
|
|
697
725
|
description: "Execute commonjs javascript",
|
|
698
726
|
args: {
|
|
699
727
|
code: { type: "string", description: "CommonJS javascript", required: !0 }
|
|
700
728
|
},
|
|
701
|
-
fn: async (
|
|
702
|
-
const r =
|
|
729
|
+
fn: async (h) => {
|
|
730
|
+
const r = C(null), e = await L({ console: r }, h.code, !0).catch((t) => r.output.error.push(t));
|
|
703
731
|
return { ...r.output, return: e, stdout: void 0, stderr: void 0 };
|
|
704
732
|
}
|
|
705
|
-
},
|
|
733
|
+
}, oe = {
|
|
706
734
|
name: "exec_javascript",
|
|
707
735
|
description: "Execute commonjs javascript",
|
|
708
736
|
args: {
|
|
709
737
|
code: { type: "string", description: "CommonJS javascript", required: !0 }
|
|
710
738
|
},
|
|
711
|
-
fn: async (
|
|
712
|
-
},
|
|
739
|
+
fn: async (h) => ({ result: Q`python -c "${h.code}"` })
|
|
740
|
+
}, _e = {
|
|
713
741
|
name: "read_webpage",
|
|
714
742
|
description: "Extract clean, structured content from a webpage. Use after web_search to read specific URLs",
|
|
715
743
|
args: {
|
|
716
744
|
url: { type: "string", description: "URL to extract content from", required: !0 },
|
|
717
745
|
focus: { type: "string", description: 'Optional: What aspect to focus on (e.g., "pricing", "features", "contact info")' }
|
|
718
746
|
},
|
|
719
|
-
fn: async (
|
|
720
|
-
const r = await fetch(
|
|
747
|
+
fn: async (h) => {
|
|
748
|
+
const r = await fetch(h.url, { headers: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)" } }).then((s) => s.text()).catch((s) => {
|
|
721
749
|
throw new Error(`Failed to fetch: ${s.message}`);
|
|
722
|
-
}), e =
|
|
750
|
+
}), e = Y.load(r);
|
|
723
751
|
e('script, style, nav, footer, header, aside, iframe, noscript, [role="navigation"], [role="banner"], .ad, .ads, .cookie, .popup').remove();
|
|
724
752
|
const t = {
|
|
725
753
|
title: e('meta[property="og:title"]').attr("content") || e("title").text() || "",
|
|
726
754
|
description: e('meta[name="description"]').attr("content") || e('meta[property="og:description"]').attr("content") || ""
|
|
727
755
|
};
|
|
728
|
-
let
|
|
729
|
-
const
|
|
730
|
-
for (const s of
|
|
731
|
-
const
|
|
732
|
-
if (
|
|
733
|
-
|
|
756
|
+
let l = "";
|
|
757
|
+
const n = ["article", "main", '[role="main"]', ".content", ".post", ".entry", "body"];
|
|
758
|
+
for (const s of n) {
|
|
759
|
+
const c = e(s).first();
|
|
760
|
+
if (c.length && c.text().trim().length > 200) {
|
|
761
|
+
l = c.text();
|
|
734
762
|
break;
|
|
735
763
|
}
|
|
736
764
|
}
|
|
737
|
-
return
|
|
765
|
+
return l || (l = e("body").text()), l = l.replace(/\s+/g, " ").trim().slice(0, 8e3), { url: h.url, title: t.title.trim(), description: t.description.trim(), content: l, focus: h.focus };
|
|
738
766
|
}
|
|
739
|
-
},
|
|
767
|
+
}, Se = {
|
|
740
768
|
name: "web_search",
|
|
741
769
|
description: "Use duckduckgo (anonymous) to find find relevant online resources. Returns a list of URLs that works great with the `read_webpage` tool",
|
|
742
770
|
args: {
|
|
743
771
|
query: { type: "string", description: "Search string", required: !0 },
|
|
744
772
|
length: { type: "string", description: "Number of results to return", default: 5 }
|
|
745
773
|
},
|
|
746
|
-
fn: async (
|
|
747
|
-
const r = await fetch(`https://html.duckduckgo.com/html/?q=${encodeURIComponent(
|
|
774
|
+
fn: async (h) => {
|
|
775
|
+
const r = await fetch(`https://html.duckduckgo.com/html/?q=${encodeURIComponent(h.query)}`, {
|
|
748
776
|
headers: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)", "Accept-Language": "en-US,en;q=0.9" }
|
|
749
|
-
}).then((
|
|
777
|
+
}).then((n) => n.text());
|
|
750
778
|
let e, t = /<a .*?href="(.+?)".+?<\/a>/g;
|
|
751
|
-
const
|
|
779
|
+
const l = new J();
|
|
752
780
|
for (; (e = t.exec(r)) !== null; ) {
|
|
753
|
-
let
|
|
754
|
-
if (
|
|
781
|
+
let n = /uddg=(.+)&?/.exec(decodeURIComponent(e[1]))?.[1];
|
|
782
|
+
if (n && (n = decodeURIComponent(n)), n && l.add(n), l.size >= (h.length || 5)) break;
|
|
755
783
|
}
|
|
756
|
-
return
|
|
784
|
+
return l;
|
|
757
785
|
}
|
|
758
786
|
};
|
|
759
787
|
export {
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
788
|
+
we as Ai,
|
|
789
|
+
X as Anthropic,
|
|
790
|
+
te as Audio,
|
|
791
|
+
se as CliTool,
|
|
792
|
+
be as DateTimeTool,
|
|
793
|
+
ke as ExecTool,
|
|
794
|
+
xe as FetchTool,
|
|
795
|
+
ne as JSTool,
|
|
796
|
+
R as LLMProvider,
|
|
797
|
+
A as OpenAi,
|
|
798
|
+
oe as PythonTool,
|
|
799
|
+
_e as ReadWebpageTool,
|
|
800
|
+
re as Vision,
|
|
801
|
+
Se as WebSearchTool
|
|
774
802
|
};
|
|
775
803
|
//# sourceMappingURL=index.mjs.map
|