@ztimson/ai-utils 0.7.11 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -7
- package/dist/index.js +23 -20
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +346 -307
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
<br />
|
|
4
4
|
|
|
5
5
|
<!-- Logo -->
|
|
6
|
-
<img
|
|
6
|
+
<img alt="Logo" width="200" height="200" src="https://git.zakscode.com/repo-avatars/a82d423674763e7a0c1c945bdbb07e249b2bb786d3c9beae76d5b196a10f5c0f">
|
|
7
7
|
|
|
8
8
|
<!-- Title -->
|
|
9
9
|
### @ztimson/ai-utils
|
|
@@ -53,13 +53,15 @@ A TypeScript library that provides a unified interface for working with multiple
|
|
|
53
53
|
- **Provider Abstraction**: Switch between AI providers without changing your code
|
|
54
54
|
|
|
55
55
|
### Built With
|
|
56
|
-
[](https://anthropic.com/)
|
|
57
|
+
[](https://github.com/ggml-org/llama.cpp)
|
|
58
|
+
[](https://openai.com/)
|
|
59
|
+
[](https://github.com/pyannote)
|
|
60
|
+
[](https://tensorflow.org/)
|
|
61
|
+
[](https://tesseract-ocr.github.io/)
|
|
62
|
+
[](https://huggingface.co/docs/transformers.js/en/index)
|
|
61
63
|
[](https://typescriptlang.org/)
|
|
62
|
-
[](https://github.com/ggerganov/whisper.cpp)
|
|
63
65
|
|
|
64
66
|
## Setup
|
|
65
67
|
|
|
@@ -88,6 +90,8 @@ A TypeScript library that provides a unified interface for working with multiple
|
|
|
88
90
|
|
|
89
91
|
#### Prerequisites
|
|
90
92
|
- [Node.js](https://nodejs.org/en/download)
|
|
93
|
+
- _[Whisper.cpp](https://github.com/ggml-org/whisper.cpp/releases/tag) (ASR)_
|
|
94
|
+
- _[Pyannote](https://github.com/pyannote) (ASR Diarization):_ `pip install pyannote.audio`
|
|
91
95
|
|
|
92
96
|
#### Instructions
|
|
93
97
|
1. Install the dependencies: `npm i`
|
package/dist/index.js
CHANGED
|
@@ -1,32 +1,32 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const
|
|
1
|
+
"use strict";Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const M=require("node:os"),g=require("@ztimson/utils"),C=require("@anthropic-ai/sdk"),J=require("openai"),W=require("url"),q=require("path"),k=require("node:child_process"),D=require("node:fs"),_=require("node:fs/promises"),S=require("node:path"),I=require("tesseract.js"),F=require("cheerio"),v=require("@ztimson/node-utils");var x=typeof document<"u"?document.currentScript:null;function T(h){const r=Object.create(null,{[Symbol.toStringTag]:{value:"Module"}});if(h){for(const e in h)if(e!=="default"){const t=Object.getOwnPropertyDescriptor(h,e);Object.defineProperty(r,e,t.get?t:{enumerable:!0,get:()=>h[e]})}}return r.default=h,Object.freeze(r)}const H=T(M),O=T(S),B=T(F);class E{}class A extends E{constructor(r,e,t){super(),this.ai=r,this.apiToken=e,this.model=t,this.client=new C.Anthropic({apiKey:e})}client;toStandard(r){const e=Date.now(),t=[];for(let c of r)if(typeof c.content=="string")t.push({timestamp:e,...c});else{const o=c.content?.filter(s=>s.type=="text").map(s=>s.text).join(`
|
|
2
2
|
|
|
3
|
-
`);
|
|
3
|
+
`);o&&t.push({timestamp:e,role:c.role,content:o}),c.content.forEach(s=>{if(s.type=="tool_use")t.push({timestamp:e,role:"tool",id:s.id,name:s.name,args:s.input,content:void 0});else if(s.type=="tool_result"){const l=t.findLast(n=>n.id==s.tool_use_id);l&&(l[s.is_error?"error":"content"]=s.content)}})}return t}fromStandard(r){for(let e=0;e<r.length;e++)if(r[e].role=="tool"){const t=r[e];r.splice(e,1,{role:"assistant",content:[{type:"tool_use",id:t.id,name:t.name,input:t.args}]},{role:"user",content:[{type:"tool_result",tool_use_id:t.id,is_error:!!t.error,content:t.error||t.content}]}),e++}return r.map(({timestamp:e,...t})=>t)}ask(r,e={}){const t=new AbortController;return Object.assign(new Promise(async c=>{let o=this.fromStandard([...e.history||[],{role:"user",content:r,timestamp:Date.now()}]);const s=e.tools||this.ai.options.llm?.tools||[],l={model:e.model||this.model,max_tokens:e.max_tokens||this.ai.options.llm?.max_tokens||4096,system:e.system||this.ai.options.llm?.system||"",temperature:e.temperature||this.ai.options.llm?.temperature||.7,tools:s.map(d=>({name:d.name,description:d.description,input_schema:{type:"object",properties:d.args?g.objectMap(d.args,(i,m)=>({...m,required:void 0})):{},required:d.args?Object.entries(d.args).filter(i=>i[1].required).map(i=>i[0]):[]},fn:void 0})),messages:o,stream:!!e.stream};let n,a=!0;do{if(n=await this.client.messages.create(l).catch(i=>{throw i.message+=`
|
|
4
4
|
|
|
5
5
|
Messages:
|
|
6
|
-
${JSON.stringify(
|
|
6
|
+
${JSON.stringify(o,null,2)}`,i}),e.stream){a?a=!1:e.stream({text:`
|
|
7
7
|
|
|
8
|
-
`}),
|
|
8
|
+
`}),n.content=[];for await(const i of n){if(t.signal.aborted)break;if(i.type==="content_block_start")i.content_block.type==="text"?n.content.push({type:"text",text:""}):i.content_block.type==="tool_use"&&n.content.push({type:"tool_use",id:i.content_block.id,name:i.content_block.name,input:""});else if(i.type==="content_block_delta")if(i.delta.type==="text_delta"){const m=i.delta.text;n.content.at(-1).text+=m,e.stream({text:m})}else i.delta.type==="input_json_delta"&&(n.content.at(-1).input+=i.delta.partial_json);else if(i.type==="content_block_stop"){const m=n.content.at(-1);m.input!=null&&(m.input=m.input?g.JSONAttemptParse(m.input,{}):{})}else if(i.type==="message_stop")break}}const d=n.content.filter(i=>i.type==="tool_use");if(d.length&&!t.signal.aborted){o.push({role:"assistant",content:n.content});const i=await Promise.all(d.map(async m=>{const p=s.find(g.findByProp("name",m.name));if(e.stream&&e.stream({tool:m.name}),!p)return{tool_use_id:m.id,is_error:!0,content:"Tool not found"};try{const u=await p.fn(m.input,e?.stream,this.ai);return{type:"tool_result",tool_use_id:m.id,content:g.JSONSanitize(u)}}catch(u){return{type:"tool_result",tool_use_id:m.id,is_error:!0,content:u?.message||u?.toString()||"Unknown"}}}));o.push({role:"user",content:i}),l.messages=o}}while(!t.signal.aborted&&n.content.some(d=>d.type==="tool_use"));o.push({role:"assistant",content:n.content.filter(d=>d.type=="text").map(d=>d.text).join(`
|
|
9
9
|
|
|
10
|
-
`)}),
|
|
10
|
+
`)}),o=this.toStandard(o),e.stream&&e.stream({done:!0}),e.history&&e.history.splice(0,e.history.length,...o),c(o.at(-1)?.content)}),{abort:()=>t.abort()})}}class j extends E{constructor(r,e,t,c){super(),this.ai=r,this.host=e,this.token=t,this.model=c,this.client=new J.OpenAI(g.clean({baseURL:e,apiKey:t}))}client;toStandard(r){for(let e=0;e<r.length;e++){const t=r[e];if(t.role==="assistant"&&t.tool_calls){const c=t.tool_calls.map(o=>({role:"tool",id:o.id,name:o.function.name,args:g.JSONAttemptParse(o.function.arguments,{}),timestamp:t.timestamp}));r.splice(e,1,...c),e+=c.length-1}else if(t.role==="tool"&&t.content){const c=r.find(o=>t.tool_call_id==o.id);c&&(t.content.includes('"error":')?c.error=t.content:c.content=t.content),r.splice(e,1),e--}r[e]?.timestamp||(r[e].timestamp=Date.now())}return r}fromStandard(r){return r.reduce((e,t)=>{if(t.role==="tool")e.push({role:"assistant",content:null,tool_calls:[{id:t.id,type:"function",function:{name:t.name,arguments:JSON.stringify(t.args)}}],refusal:null,annotations:[]},{role:"tool",tool_call_id:t.id,content:t.error||t.content});else{const{timestamp:c,...o}=t;e.push(o)}return e},[])}ask(r,e={}){const t=new AbortController;return Object.assign(new Promise(async(c,o)=>{e.system&&e.history?.[0]?.role!="system"&&e.history?.splice(0,0,{role:"system",content:e.system,timestamp:Date.now()});let s=this.fromStandard([...e.history||[],{role:"user",content:r,timestamp:Date.now()}]);const l=e.tools||this.ai.options.llm?.tools||[],n={model:e.model||this.model,messages:s,stream:!!e.stream,max_tokens:e.max_tokens||this.ai.options.llm?.max_tokens||4096,temperature:e.temperature||this.ai.options.llm?.temperature||.7,tools:l.map(i=>({type:"function",function:{name:i.name,description:i.description,parameters:{type:"object",properties:i.args?g.objectMap(i.args,(m,p)=>({...p,required:void 0})):{},required:i.args?Object.entries(i.args).filter(m=>m[1].required).map(m=>m[0]):[]}}}))};let a,d=!0;do{if(a=await this.client.chat.completions.create(n).catch(m=>{throw m.message+=`
|
|
11
11
|
|
|
12
12
|
Messages:
|
|
13
|
-
${JSON.stringify(s,null,2)}`,
|
|
13
|
+
${JSON.stringify(s,null,2)}`,m}),e.stream){d?d=!1:e.stream({text:`
|
|
14
14
|
|
|
15
|
-
`}),
|
|
15
|
+
`}),a.choices=[{message:{content:"",tool_calls:[]}}];for await(const m of a){if(t.signal.aborted)break;m.choices[0].delta.content&&(a.choices[0].message.content+=m.choices[0].delta.content,e.stream({text:m.choices[0].delta.content})),m.choices[0].delta.tool_calls&&(a.choices[0].message.tool_calls=m.choices[0].delta.tool_calls)}}const i=a.choices[0].message.tool_calls||[];if(i.length&&!t.signal.aborted){s.push(a.choices[0].message);const m=await Promise.all(i.map(async p=>{const u=l?.find(g.findByProp("name",p.function.name));if(e.stream&&e.stream({tool:p.function.name}),!u)return{role:"tool",tool_call_id:p.id,content:'{"error": "Tool not found"}'};try{const f=g.JSONAttemptParse(p.function.arguments,{}),y=await u.fn(f,e.stream,this.ai);return{role:"tool",tool_call_id:p.id,content:g.JSONSanitize(y)}}catch(f){return{role:"tool",tool_call_id:p.id,content:g.JSONSanitize({error:f?.message||f?.toString()||"Unknown"})}}}));s.push(...m),n.messages=s}}while(!t.signal.aborted&&a.choices?.[0]?.message?.tool_calls?.length);s.push({role:"assistant",content:a.choices[0].message.content||""}),s=this.toStandard(s),e.stream&&e.stream({done:!0}),e.history&&e.history.splice(0,e.history.length,...s),c(s.at(-1)?.content)}),{abort:()=>t.abort()})}}class G{constructor(r){this.ai=r,r.options.llm?.models&&Object.entries(r.options.llm.models).forEach(([e,t])=>{this.defaultModel||(this.defaultModel=e),t.proto=="anthropic"?this.models[e]=new A(this.ai,t.token,e):t.proto=="ollama"?this.models[e]=new j(this.ai,t.host,"not-needed",e):t.proto=="openai"&&(this.models[e]=new j(this.ai,t.host||null,t.token,e))})}defaultModel;models={};ask(r,e={}){const t=e.model||this.defaultModel;if(!this.models[t])throw new Error(`Model does not exist: ${t}`);let c=()=>{};return Object.assign(new Promise(async o=>{if(e.history||(e.history=[]),e.memory){e.system=(e.system||"")+`
|
|
16
16
|
You have passive persistent memory never make any mention of your memory capabilities and what you can/cannot remember
|
|
17
|
-
`;const
|
|
18
|
-
`+
|
|
19
|
-
`)}),e.tools=[...e.tools||[],{name:"read_memory",description:"Check your long-term memory for more information",args:{subject:{type:"string",description:"Find information by a subject topic, can be used with or without query argument"},query:{type:"string",description:"Search memory based on a query, can be used with or without subject argument"},limit:{type:"number",description:"Result limit, default 5"}},fn:
|
|
20
|
-
|
|
21
|
-
`),"{summary: string, facts: [[subject, fact]]}",{system:"Create the smallest summary possible, no more than 500 tokens. Create a list of NEW facts (split by subject [pro]noun and fact) about what you learned from this conversation that you didn't already know or get from a tool call or system prompt. Focus only on new information about people, topics, or facts. Avoid generating facts about the AI.",model:
|
|
22
|
-
`)).flatMap(
|
|
23
|
-
`]),
|
|
24
|
-
`))>e&&
|
|
25
|
-
`).trim();
|
|
17
|
+
`;const l=async(a,d,i=50)=>{const[m,p]=await Promise.all([d?this.embedding(d):Promise.resolve(null),a?this.embedding(a):Promise.resolve(null)]);return(e.memory||[]).map(u=>({...u,score:m?this.cosineSimilarity(u.embeddings[0],m[0].embedding):1})).filter(u=>u.score>=.8).map(u=>({...u,score:p?this.cosineSimilarity(u.embeddings[1],p[0].embedding):u.score})).filter(u=>u.score>=.2).toSorted((u,f)=>u.score-f.score).slice(0,i)},n=await l(r);n.length&&e.history.push({role:"assistant",content:`Things I remembered:
|
|
18
|
+
`+n.map(a=>`${a.owner}: ${a.fact}`).join(`
|
|
19
|
+
`)}),e.tools=[...e.tools||[],{name:"read_memory",description:"Check your long-term memory for more information",args:{subject:{type:"string",description:"Find information by a subject topic, can be used with or without query argument"},query:{type:"string",description:"Search memory based on a query, can be used with or without subject argument"},limit:{type:"number",description:"Result limit, default 5"}},fn:a=>{if(!a.subject&&!a.query)throw new Error("Either a subject or query argument is required");return l(a.query,a.subject,a.limit||5)}}]}const s=await this.models[t].ask(r,e);if(e.memory){const l=e.history?.findIndex(n=>n.role=="assistant"&&n.content.startsWith("Things I remembered:"));l!=null&&l>=0&&e.history?.splice(l,1)}if(e.compress||e.memory){let l=null;if(e.compress)l=await this.ai.language.compressHistory(e.history,e.compress.max,e.compress.min,e),e.history.splice(0,e.history.length,...l.history);else{const n=e.history?.findLastIndex(a=>a.role=="user")??-1;l=await this.ai.language.compressHistory(n!=-1?e.history.slice(n):e.history,0,0,e)}if(e.memory){const n=e.memory.filter(a=>!l.memory.some(d=>this.cosineSimilarity(a.embeddings[1],d.embeddings[1])>.8)).concat(l.memory);e.memory.splice(0,e.memory.length,...n)}}return o(s)}),{abort:c})}async compressHistory(r,e,t,c){if(this.estimateTokens(r)<e)return{history:r,memory:[]};let o=0,s=0;for(let u of r.toReversed())if(s+=this.estimateTokens(u.content),s<t)o++;else break;if(r.length<=o)return{history:r,memory:[]};const l=r[0].role=="system"?r[0]:null,n=o==0?[]:r.slice(-o),a=(o==0?r:r.slice(0,-o)).filter(u=>u.role==="assistant"||u.role==="user"),d=await this.json(a.map(u=>`${u.role}: ${u.content}`).join(`
|
|
20
|
+
|
|
21
|
+
`),"{summary: string, facts: [[subject, fact]]}",{system:"Create the smallest summary possible, no more than 500 tokens. Create a list of NEW facts (split by subject [pro]noun and fact) about what you learned from this conversation that you didn't already know or get from a tool call or system prompt. Focus only on new information about people, topics, or facts. Avoid generating facts about the AI.",model:c?.model,temperature:c?.temperature||.3}),i=new Date,m=await Promise.all((d?.facts||[])?.map(async([u,f])=>{const y=await Promise.all([this.embedding(u),this.embedding(`${u}: ${f}`)]);return{owner:u,fact:f,embeddings:[y[0][0].embedding,y[1][0].embedding],timestamp:i}})),p=[{role:"assistant",content:`Conversation Summary: ${d?.summary}`,timestamp:Date.now()},...n];return l&&p.splice(0,0,l),{history:p,memory:m}}cosineSimilarity(r,e){if(r.length!==e.length)throw new Error("Vectors must be same length");let t=0,c=0,o=0;for(let l=0;l<r.length;l++)t+=r[l]*e[l],c+=r[l]*r[l],o+=e[l]*e[l];const s=Math.sqrt(c)*Math.sqrt(o);return s===0?0:t/s}chunk(r,e=500,t=50){const c=(n,a="")=>n?Object.entries(n).flatMap(([d,i])=>{const m=a?`${a}${isNaN(+d)?`.${d}`:`[${d}]`}`:d;return typeof i=="object"&&!Array.isArray(i)?c(i,m):`${m}: ${Array.isArray(i)?i.join(", "):i}`}):[],s=(typeof r=="object"?c(r):r.split(`
|
|
22
|
+
`)).flatMap(n=>[...n.split(/\s+/).filter(Boolean),`
|
|
23
|
+
`]),l=[];for(let n=0;n<s.length;){let a="",d=n;for(;d<s.length;){const m=a+(a?" ":"")+s[d];if(this.estimateTokens(m.replace(/\s*\n\s*/g,`
|
|
24
|
+
`))>e&&a)break;a=m,d++}const i=a.replace(/\s*\n\s*/g,`
|
|
25
|
+
`).trim();i&&l.push(i),n=Math.max(d-t,d===n?n+1:d)}return l}embedding(r,e={}){let{maxTokens:t=500,overlapTokens:c=50}=e,o=!1;const s=()=>{o=!0},l=a=>new Promise((d,i)=>{if(o)return i(new Error("Aborted"));const m=[q.join(q.dirname(W.fileURLToPath(typeof document>"u"?require("url").pathToFileURL(__filename).href:x&&x.tagName.toUpperCase()==="SCRIPT"&&x.src||new URL("index.js",document.baseURI).href)),"embedder.js"),this.ai.options.path,this.ai.options?.embedder||"bge-small-en-v1.5"],p=k.spawn("node",m,{stdio:["pipe","pipe","ignore"]});p.stdin.write(a),p.stdin.end();let u="";p.stdout.on("data",f=>u+=f.toString()),p.on("close",f=>{if(o)return i(new Error("Aborted"));if(f===0)try{const y=JSON.parse(u);d(y.embedding)}catch{i(new Error("Failed to parse embedding output"))}else i(new Error(`Embedder process exited with code ${f}`))}),p.on("error",i)}),n=(async()=>{const a=this.chunk(r,t,c),d=[];for(let i=0;i<a.length&&!o;i++){const m=a[i],p=await l(m);d.push({index:i,embedding:p,text:m,tokens:this.estimateTokens(m)})}return d})();return Object.assign(n,{abort:s})}estimateTokens(r){const e=JSON.stringify(r);return Math.ceil(e.length/4*1.2)}fuzzyMatch(r,...e){if(e.length<2)throw new Error("Requires at least 2 strings to compare");const t=(s,l=10)=>s.toLowerCase().split("").map((n,a)=>n.charCodeAt(0)*(a+1)%l/l).slice(0,l),c=t(r),o=e.map(s=>t(s)).map(s=>this.cosineSimilarity(c,s));return{avg:o.reduce((s,l)=>s+l,0)/o.length,max:Math.max(...o),similarities:o}}async json(r,e,t){let c=await this.ask(r,{...t,system:(t?.system?`${t.system}
|
|
26
26
|
`:"")+`Only respond using a JSON code block matching this schema:
|
|
27
27
|
\`\`\`json
|
|
28
28
|
${e}
|
|
29
|
-
\`\`\``});if(!
|
|
29
|
+
\`\`\``});if(!c)return{};const o=/```(?:.+)?\s*([\s\S]*?)```/.exec(c),s=o?o[1].trim():c;return g.JSONAttemptParse(s,{})}summarize(r,e,t){return this.ask(r,{system:`Generate a brief summary <= ${e} tokens. Output nothing else`,temperature:.3,...t})}}class ${constructor(r){this.ai=r,r.options.whisper&&(this.whisperModel=r.options.asr||"ggml-base.en.bin",this.downloadAsrModel()),this.pyannote=`
|
|
30
30
|
import sys
|
|
31
31
|
import json
|
|
32
32
|
import os
|
|
@@ -41,7 +41,10 @@ for turn, speaker in output.speaker_diarization:
|
|
|
41
41
|
segments.append({"start": turn.start, "end": turn.end, "speaker": speaker})
|
|
42
42
|
|
|
43
43
|
print(json.dumps(segments))
|
|
44
|
-
`}downloads={};pyannote;whisperModel;runAsr(r,e={}){let t;const
|
|
45
|
-
`)}asr(r,e={}){if(!this.ai.options.whisper)throw new Error("Whisper not configured");const t=
|
|
46
|
-
|
|
44
|
+
`}downloads={};pyannote;whisperModel;runAsr(r,e={}){let t;const c=new Promise((o,s)=>{this.downloadAsrModel(e.model).then(l=>{if(e.diarization){let n=O.join(O.dirname(r),"transcript");t=k.spawn(this.ai.options.whisper,["-m",l,"-f",r,"-np","-ml","1","-oj","-of",n],{stdio:["ignore","ignore","pipe"]}),t.on("error",a=>s(a)),t.on("close",async a=>{if(a===0){n=await _.readFile(n+".json","utf-8"),_.rm(n+".json").catch(()=>{});try{o(JSON.parse(n))}catch{s(new Error("Failed to parse whisper JSON"))}}else s(new Error(`Exit code ${a}`))})}else{let n="";t=k.spawn(this.ai.options.whisper,["-m",l,"-f",r,"-np","-nt"]),t.on("error",a=>s(a)),t.stdout.on("data",a=>n+=a.toString()),t.on("close",async a=>{a===0?o(n.trim()||null):s(new Error(`Exit code ${a}`))})}})});return Object.assign(c,{abort:()=>t?.kill("SIGTERM")})}runDiarization(r){let e=!1,t=()=>{e=!0};const c=s=>new Promise(l=>{const n=k.spawn(s,["-W","ignore","-c","import pyannote.audio"]);n.on("close",a=>l(a===0)),n.on("error",()=>l(!1))}),o=Promise.all([c("python"),c("python3")]).then((async([s,l])=>{if(e)return;if(!s&&!l)throw new Error("Pyannote is not installed: pip install pyannote.audio");const n=l?"python3":"python";return new Promise((a,d)=>{if(e)return;let i="";const m=k.spawn(n,["-W","ignore","-c",this.pyannote,r]);m.stdout.on("data",p=>i+=p.toString()),m.stderr.on("data",p=>console.error(p.toString())),m.on("close",p=>{if(p===0)try{a(JSON.parse(i))}catch{d(new Error("Failed to parse diarization output"))}else d(new Error(`Python process exited with code ${p}`))}),m.on("error",d),t=()=>m.kill("SIGTERM")})}));return Object.assign(o,{abort:t})}async combineSpeakerTranscript(r,e,t){const c=new Map;let o=0;t.forEach(n=>{c.has(n.speaker)||c.set(n.speaker,++o)});const s=r.match(/[^.!?]+[.!?]+/g)||[r],l=[];return s.forEach(n=>{if(n=n.trim(),!n)return;const a=n.toLowerCase().replace(/[^\w\s]/g,"").split(/\s+/);let d=1/0;const i=[];if(e.transcription.forEach(f=>{const y=f.text.trim().toLowerCase();if(a.some(w=>y.includes(w))){const w=f.offsets.from/1e3,b=f.offsets.to/1e3;i.push({start:w,end:b}),w<d&&(d=w)}}),d===1/0)return;const m=new Map;i.forEach(f=>{t.forEach(y=>{const w=Math.max(0,Math.min(f.end,y.end)-Math.max(f.start,y.start)),b=f.end-f.start;if(b>0){const L=w/b,P=c.get(y.speaker);m.set(P,(m.get(P)||0)+L)}})});let p=1,u=0;m.forEach((f,y)=>{f>u&&(u=f,p=y)}),l.push(`[Speaker ${p}]: ${n}`)}),l.join(`
|
|
45
|
+
`).trim()}asr(r,e={}){if(!this.ai.options.whisper)throw new Error("Whisper not configured");const t=S.join(D.mkdtempSync(S.join(M.tmpdir(),"audio-")),"converted.wav");k.execSync(`ffmpeg -i "${r}" -ar 16000 -ac 1 -f wav "${t}"`,{stdio:"ignore"});const c=()=>_.rm(S.dirname(t),{recursive:!0,force:!0}).catch(()=>{}),o=this.runAsr(t,{model:e.model,diarization:!1}),s=e.diarization?this.runAsr(t,{model:e.model,diarization:!0}):Promise.resolve(null),l=e.diarization?this.runDiarization(t):Promise.resolve(null);let n=!1,a=()=>{n=!0,o.abort(),s?.abort?.(),l?.abort?.(),c()};const d=Promise.allSettled([o,s,l]).then(async([i,m,p])=>{if(i.status=="rejected")throw new Error(`Whisper.cpp punctuated:
|
|
46
|
+
`+i.reason);if(m.status=="rejected")throw new Error(`Whisper.cpp timestamps:
|
|
47
|
+
`+m.reason);if(p.status=="rejected")throw new Error(`Pyannote:
|
|
48
|
+
`+p.reason);if(n||!e.diarization)return i.value;let u=await this.combineSpeakerTranscript(i.value,m.value,p.value);if(!n&&e.diarization==="id"){if(!this.ai.language.defaultModel)throw new Error("Configure an LLM for advanced ASR speaker detection");let f=this.ai.language.chunk(u,500,0);f.length>4&&(f=[...f.slice(0,3),f.at(-1)]);const y=await this.ai.language.json(f.join(`
|
|
49
|
+
`),'{1: "Detected Name", 2: "Second Name"}',{system:"Use the following transcript to identify speakers. Only identify speakers you are positive about, dont mention speakers you are unsure about in your response",temperature:.1});Object.entries(y).forEach(([w,b])=>u=u.replaceAll(`[Speaker ${w}]`,`[${b}]`))}return u}).finally(()=>c());return Object.assign(d,{abort:a})}async downloadAsrModel(r=this.whisperModel){if(!this.ai.options.whisper)throw new Error("Whisper not configured");r.endsWith(".bin")||(r+=".bin");const e=S.join(this.ai.options.path,r);return await _.stat(e).then(()=>!0).catch(()=>!1)?e:this.downloads[r]?this.downloads[r]:(this.downloads[r]=fetch(`https://huggingface.co/ggerganov/whisper.cpp/resolve/main/${r}`).then(t=>t.arrayBuffer()).then(t=>Buffer.from(t)).then(async t=>(await _.writeFile(e,t),delete this.downloads[r],e)),this.downloads[r])}}class N{constructor(r){this.ai=r}ocr(r){let e;const t=new Promise(async c=>{e=await I.createWorker(this.ai.options.ocr||"eng",2,{cachePath:this.ai.options.path});const{data:o}=await e.recognize(r);await e.terminate(),c(o.text.trim()||null)});return Object.assign(t,{abort:()=>e?.terminate()})}}class V{constructor(r){this.options=r,r.path||(r.path=H.tmpdir()),process.env.TRANSFORMERS_CACHE=r.path,this.audio=new $(this),this.language=new G(this),this.vision=new N(this)}audio;language;vision}const R={name:"cli",description:"Use the command line interface, returns any output",args:{command:{type:"string",description:"Command to run",required:!0}},fn:h=>v.$`${h.command}`},K={name:"get_datetime",description:"Get current UTC date / time",args:{},fn:async()=>new Date().toUTCString()},Y={name:"exec",description:"Run code/scripts",args:{language:{type:"string",description:"Execution language",enum:["cli","node","python"],required:!0},code:{type:"string",description:"Code to execute",required:!0}},fn:async(h,r,e)=>{try{switch(h.type){case"bash":return await R.fn({command:h.code},r,e);case"node":return await U.fn({code:h.code},r,e);case"python":return await z.fn({code:h.code},r,e)}}catch(t){return{error:t?.message||t.toString()}}}},Q={name:"fetch",description:"Make HTTP request to URL",args:{url:{type:"string",description:"URL to fetch",required:!0},method:{type:"string",description:"HTTP method to use",enum:["GET","POST","PUT","DELETE"],default:"GET"},headers:{type:"object",description:"HTTP headers to send",default:{}},body:{type:"object",description:"HTTP body to send"}},fn:h=>new g.Http({url:h.url,headers:h.headers}).request({method:h.method||"GET",body:h.body})},U={name:"exec_javascript",description:"Execute commonjs javascript",args:{code:{type:"string",description:"CommonJS javascript",required:!0}},fn:async h=>{const r=g.consoleInterceptor(null),e=await g.fn({console:r},h.code,!0).catch(t=>r.output.error.push(t));return{...r.output,return:e,stdout:void 0,stderr:void 0}}},z={name:"exec_javascript",description:"Execute commonjs javascript",args:{code:{type:"string",description:"CommonJS javascript",required:!0}},fn:async h=>({result:v.$Sync`python -c "${h.code}"`})},X={name:"read_webpage",description:"Extract clean, structured content from a webpage. Use after web_search to read specific URLs",args:{url:{type:"string",description:"URL to extract content from",required:!0},focus:{type:"string",description:'Optional: What aspect to focus on (e.g., "pricing", "features", "contact info")'}},fn:async h=>{const r=await fetch(h.url,{headers:{"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"}}).then(s=>s.text()).catch(s=>{throw new Error(`Failed to fetch: ${s.message}`)}),e=B.load(r);e('script, style, nav, footer, header, aside, iframe, noscript, [role="navigation"], [role="banner"], .ad, .ads, .cookie, .popup').remove();const t={title:e('meta[property="og:title"]').attr("content")||e("title").text()||"",description:e('meta[name="description"]').attr("content")||e('meta[property="og:description"]').attr("content")||""};let c="";const o=["article","main",'[role="main"]',".content",".post",".entry","body"];for(const s of o){const l=e(s).first();if(l.length&&l.text().trim().length>200){c=l.text();break}}return c||(c=e("body").text()),c=c.replace(/\s+/g," ").trim().slice(0,8e3),{url:h.url,title:t.title.trim(),description:t.description.trim(),content:c,focus:h.focus}}},Z={name:"web_search",description:"Use duckduckgo (anonymous) to find find relevant online resources. Returns a list of URLs that works great with the `read_webpage` tool",args:{query:{type:"string",description:"Search string",required:!0},length:{type:"string",description:"Number of results to return",default:5}},fn:async h=>{const r=await fetch(`https://html.duckduckgo.com/html/?q=${encodeURIComponent(h.query)}`,{headers:{"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64)","Accept-Language":"en-US,en;q=0.9"}}).then(o=>o.text());let e,t=/<a .*?href="(.+?)".+?<\/a>/g;const c=new g.ASet;for(;(e=t.exec(r))!==null;){let o=/uddg=(.+)&?/.exec(decodeURIComponent(e[1]))?.[1];if(o&&(o=decodeURIComponent(o)),o&&c.add(o),c.size>=(h.length||5))break}return c}};exports.Ai=V;exports.Anthropic=A;exports.Audio=$;exports.CliTool=R;exports.DateTimeTool=K;exports.ExecTool=Y;exports.FetchTool=Q;exports.JSTool=U;exports.LLMProvider=E;exports.OpenAi=j;exports.PythonTool=z;exports.ReadWebpageTool=X;exports.Vision=N;exports.WebSearchTool=Z;
|
|
47
50
|
//# sourceMappingURL=index.js.map
|