@push.rocks/smartai 0.13.3 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist_ts/00_commitinfo_data.js +3 -3
- package/dist_ts/index.d.ts +6 -11
- package/dist_ts/index.js +6 -12
- package/dist_ts/plugins.d.ts +10 -15
- package/dist_ts/plugins.js +13 -19
- package/dist_ts/smartai.classes.smartai.d.ts +7 -0
- package/dist_ts/smartai.classes.smartai.js +51 -0
- package/dist_ts/smartai.interfaces.d.ts +41 -0
- package/dist_ts/smartai.interfaces.js +2 -0
- package/dist_ts/smartai.middleware.anthropic.d.ts +7 -0
- package/dist_ts/smartai.middleware.anthropic.js +36 -0
- package/dist_ts/smartai.provider.ollama.d.ts +8 -0
- package/dist_ts/smartai.provider.ollama.js +378 -0
- package/dist_ts_audio/index.d.ts +9 -0
- package/dist_ts_audio/index.js +15 -0
- package/dist_ts_audio/plugins.d.ts +2 -0
- package/dist_ts_audio/plugins.js +3 -0
- package/dist_ts_document/index.d.ts +11 -0
- package/dist_ts_document/index.js +45 -0
- package/dist_ts_document/plugins.d.ts +3 -0
- package/dist_ts_document/plugins.js +4 -0
- package/dist_ts_image/index.d.ts +46 -0
- package/dist_ts_image/index.js +110 -0
- package/dist_ts_image/plugins.d.ts +3 -0
- package/dist_ts_image/plugins.js +4 -0
- package/dist_ts_research/index.d.ts +19 -0
- package/dist_ts_research/index.js +98 -0
- package/dist_ts_research/plugins.d.ts +2 -0
- package/dist_ts_research/plugins.js +3 -0
- package/dist_ts_vision/index.d.ts +8 -0
- package/dist_ts_vision/index.js +21 -0
- package/dist_ts_vision/plugins.d.ts +2 -0
- package/dist_ts_vision/plugins.js +3 -0
- package/package.json +50 -22
- package/readme.hints.md +34 -88
- package/readme.md +284 -547
- package/ts/00_commitinfo_data.ts +2 -2
- package/ts/index.ts +8 -11
- package/ts/plugins.ts +19 -35
- package/ts/smartai.classes.smartai.ts +51 -0
- package/ts/smartai.interfaces.ts +53 -0
- package/ts/smartai.middleware.anthropic.ts +38 -0
- package/ts/smartai.provider.ollama.ts +426 -0
- package/ts_audio/index.ts +24 -0
- package/ts_audio/plugins.ts +2 -0
- package/ts_document/index.ts +61 -0
- package/ts_document/plugins.ts +3 -0
- package/ts_image/index.ts +147 -0
- package/ts_image/plugins.ts +3 -0
- package/ts_research/index.ts +120 -0
- package/ts_research/plugins.ts +2 -0
- package/ts_vision/index.ts +29 -0
- package/ts_vision/plugins.ts +2 -0
- package/dist_ts/abstract.classes.multimodal.d.ts +0 -212
- package/dist_ts/abstract.classes.multimodal.js +0 -43
- package/dist_ts/classes.conversation.d.ts +0 -31
- package/dist_ts/classes.conversation.js +0 -150
- package/dist_ts/classes.smartai.d.ts +0 -59
- package/dist_ts/classes.smartai.js +0 -139
- package/dist_ts/classes.tts.d.ts +0 -6
- package/dist_ts/classes.tts.js +0 -10
- package/dist_ts/interfaces.d.ts +0 -1
- package/dist_ts/interfaces.js +0 -2
- package/dist_ts/paths.d.ts +0 -2
- package/dist_ts/paths.js +0 -4
- package/dist_ts/provider.anthropic.d.ts +0 -48
- package/dist_ts/provider.anthropic.js +0 -369
- package/dist_ts/provider.elevenlabs.d.ts +0 -43
- package/dist_ts/provider.elevenlabs.js +0 -64
- package/dist_ts/provider.exo.d.ts +0 -40
- package/dist_ts/provider.exo.js +0 -116
- package/dist_ts/provider.groq.d.ts +0 -39
- package/dist_ts/provider.groq.js +0 -178
- package/dist_ts/provider.mistral.d.ts +0 -61
- package/dist_ts/provider.mistral.js +0 -288
- package/dist_ts/provider.ollama.d.ts +0 -141
- package/dist_ts/provider.ollama.js +0 -529
- package/dist_ts/provider.openai.d.ts +0 -62
- package/dist_ts/provider.openai.js +0 -403
- package/dist_ts/provider.perplexity.d.ts +0 -37
- package/dist_ts/provider.perplexity.js +0 -215
- package/dist_ts/provider.xai.d.ts +0 -52
- package/dist_ts/provider.xai.js +0 -160
- package/ts/abstract.classes.multimodal.ts +0 -240
- package/ts/classes.conversation.ts +0 -176
- package/ts/classes.smartai.ts +0 -187
- package/ts/classes.tts.ts +0 -15
- package/ts/interfaces.ts +0 -0
- package/ts/paths.ts +0 -4
- package/ts/provider.anthropic.ts +0 -446
- package/ts/provider.elevenlabs.ts +0 -116
- package/ts/provider.exo.ts +0 -155
- package/ts/provider.groq.ts +0 -219
- package/ts/provider.mistral.ts +0 -352
- package/ts/provider.ollama.ts +0 -705
- package/ts/provider.openai.ts +0 -462
- package/ts/provider.perplexity.ts +0 -259
- package/ts/provider.xai.ts +0 -214
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import * as plugins from './plugins.js';
|
|
2
|
+
import type { LanguageModelV3 } from '@ai-sdk/provider';
|
|
3
|
+
import type { ModelMessage } from 'ai';
|
|
4
|
+
|
|
5
|
+
let smartpdfInstance: InstanceType<typeof plugins.smartpdf.SmartPdf> | null = null;
|
|
6
|
+
|
|
7
|
+
async function ensureSmartpdf(): Promise<InstanceType<typeof plugins.smartpdf.SmartPdf>> {
|
|
8
|
+
if (!smartpdfInstance) {
|
|
9
|
+
smartpdfInstance = new plugins.smartpdf.SmartPdf();
|
|
10
|
+
await smartpdfInstance.start();
|
|
11
|
+
}
|
|
12
|
+
return smartpdfInstance;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export interface IDocumentOptions {
|
|
16
|
+
model: LanguageModelV3;
|
|
17
|
+
systemMessage?: string;
|
|
18
|
+
userMessage: string;
|
|
19
|
+
pdfDocuments: Uint8Array[];
|
|
20
|
+
messageHistory?: ModelMessage[];
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export async function analyzeDocuments(options: IDocumentOptions): Promise<string> {
|
|
24
|
+
const pdf = await ensureSmartpdf();
|
|
25
|
+
|
|
26
|
+
const imagePages: Uint8Array[] = [];
|
|
27
|
+
for (const doc of options.pdfDocuments) {
|
|
28
|
+
const pages = await pdf.convertPDFToPngBytes(doc);
|
|
29
|
+
imagePages.push(...pages);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
// Filter out empty buffers
|
|
33
|
+
const validPages = imagePages.filter(page => page && page.length > 0);
|
|
34
|
+
|
|
35
|
+
const result = await plugins.generateText({
|
|
36
|
+
model: options.model,
|
|
37
|
+
system: options.systemMessage,
|
|
38
|
+
messages: [
|
|
39
|
+
...(options.messageHistory ?? []),
|
|
40
|
+
{
|
|
41
|
+
role: 'user',
|
|
42
|
+
content: [
|
|
43
|
+
{ type: 'text', text: options.userMessage },
|
|
44
|
+
...validPages.map(page => ({
|
|
45
|
+
type: 'image' as const,
|
|
46
|
+
image: page,
|
|
47
|
+
mimeType: 'image/png' as const,
|
|
48
|
+
})),
|
|
49
|
+
],
|
|
50
|
+
},
|
|
51
|
+
],
|
|
52
|
+
});
|
|
53
|
+
return result.text;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export async function stopSmartpdf(): Promise<void> {
|
|
57
|
+
if (smartpdfInstance) {
|
|
58
|
+
await smartpdfInstance.stop();
|
|
59
|
+
smartpdfInstance = null;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import * as plugins from './plugins.js';
|
|
2
|
+
|
|
3
|
+
export interface IImageGenerateOptions {
|
|
4
|
+
apiKey: string;
|
|
5
|
+
prompt: string;
|
|
6
|
+
model?: 'gpt-image-1' | 'dall-e-3' | 'dall-e-2';
|
|
7
|
+
quality?: 'low' | 'medium' | 'high' | 'standard' | 'hd' | 'auto';
|
|
8
|
+
size?: '256x256' | '512x512' | '1024x1024' | '1536x1024' | '1024x1536' | '1792x1024' | '1024x1792' | 'auto';
|
|
9
|
+
style?: 'vivid' | 'natural';
|
|
10
|
+
background?: 'transparent' | 'opaque' | 'auto';
|
|
11
|
+
outputFormat?: 'png' | 'jpeg' | 'webp';
|
|
12
|
+
outputCompression?: number;
|
|
13
|
+
moderation?: 'low' | 'auto';
|
|
14
|
+
n?: number;
|
|
15
|
+
stream?: boolean;
|
|
16
|
+
partialImages?: number;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface IImageEditOptions {
|
|
20
|
+
apiKey: string;
|
|
21
|
+
image: Buffer;
|
|
22
|
+
prompt: string;
|
|
23
|
+
mask?: Buffer;
|
|
24
|
+
model?: 'gpt-image-1' | 'dall-e-2';
|
|
25
|
+
quality?: 'low' | 'medium' | 'high' | 'standard' | 'auto';
|
|
26
|
+
size?: '256x256' | '512x512' | '1024x1024' | '1536x1024' | '1024x1536' | 'auto';
|
|
27
|
+
background?: 'transparent' | 'opaque' | 'auto';
|
|
28
|
+
outputFormat?: 'png' | 'jpeg' | 'webp';
|
|
29
|
+
outputCompression?: number;
|
|
30
|
+
n?: number;
|
|
31
|
+
stream?: boolean;
|
|
32
|
+
partialImages?: number;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export interface IImageResponse {
|
|
36
|
+
images: Array<{
|
|
37
|
+
b64_json?: string;
|
|
38
|
+
url?: string;
|
|
39
|
+
revisedPrompt?: string;
|
|
40
|
+
}>;
|
|
41
|
+
metadata?: {
|
|
42
|
+
model: string;
|
|
43
|
+
quality?: string;
|
|
44
|
+
size?: string;
|
|
45
|
+
outputFormat?: string;
|
|
46
|
+
tokensUsed?: number;
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export async function generateImage(options: IImageGenerateOptions): Promise<IImageResponse> {
|
|
51
|
+
const client = new plugins.OpenAI({ apiKey: options.apiKey });
|
|
52
|
+
const model = options.model || 'gpt-image-1';
|
|
53
|
+
|
|
54
|
+
const requestParams: Record<string, unknown> = {
|
|
55
|
+
model,
|
|
56
|
+
prompt: options.prompt,
|
|
57
|
+
n: options.n || 1,
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
if (model === 'gpt-image-1') {
|
|
61
|
+
if (options.quality) requestParams.quality = options.quality;
|
|
62
|
+
if (options.size) requestParams.size = options.size;
|
|
63
|
+
if (options.background) requestParams.background = options.background;
|
|
64
|
+
if (options.outputFormat) requestParams.output_format = options.outputFormat;
|
|
65
|
+
if (options.outputCompression !== undefined) requestParams.output_compression = options.outputCompression;
|
|
66
|
+
if (options.moderation) requestParams.moderation = options.moderation;
|
|
67
|
+
if (options.stream !== undefined) requestParams.stream = options.stream;
|
|
68
|
+
if (options.partialImages !== undefined) requestParams.partial_images = options.partialImages;
|
|
69
|
+
} else if (model === 'dall-e-3') {
|
|
70
|
+
if (options.quality) requestParams.quality = options.quality;
|
|
71
|
+
if (options.size) requestParams.size = options.size;
|
|
72
|
+
if (options.style) requestParams.style = options.style;
|
|
73
|
+
requestParams.response_format = 'b64_json';
|
|
74
|
+
} else if (model === 'dall-e-2') {
|
|
75
|
+
if (options.size) requestParams.size = options.size;
|
|
76
|
+
requestParams.response_format = 'b64_json';
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const result: any = await client.images.generate(requestParams as any);
|
|
80
|
+
|
|
81
|
+
const images = (result.data || []).map((img: any) => ({
|
|
82
|
+
b64_json: img.b64_json,
|
|
83
|
+
url: img.url,
|
|
84
|
+
revisedPrompt: img.revised_prompt,
|
|
85
|
+
}));
|
|
86
|
+
|
|
87
|
+
return {
|
|
88
|
+
images,
|
|
89
|
+
metadata: {
|
|
90
|
+
model,
|
|
91
|
+
quality: result.quality,
|
|
92
|
+
size: result.size,
|
|
93
|
+
outputFormat: result.output_format,
|
|
94
|
+
tokensUsed: result.usage?.total_tokens,
|
|
95
|
+
},
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export async function editImage(options: IImageEditOptions): Promise<IImageResponse> {
|
|
100
|
+
const client = new plugins.OpenAI({ apiKey: options.apiKey });
|
|
101
|
+
const model = options.model || 'gpt-image-1';
|
|
102
|
+
|
|
103
|
+
const imageFile = await plugins.toFile(options.image, 'image.png', { type: 'image/png' });
|
|
104
|
+
|
|
105
|
+
const requestParams: Record<string, unknown> = {
|
|
106
|
+
model,
|
|
107
|
+
image: imageFile,
|
|
108
|
+
prompt: options.prompt,
|
|
109
|
+
n: options.n || 1,
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
if (options.mask) {
|
|
113
|
+
requestParams.mask = await plugins.toFile(options.mask, 'mask.png', { type: 'image/png' });
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
if (model === 'gpt-image-1') {
|
|
117
|
+
if (options.quality) requestParams.quality = options.quality;
|
|
118
|
+
if (options.size) requestParams.size = options.size;
|
|
119
|
+
if (options.background) requestParams.background = options.background;
|
|
120
|
+
if (options.outputFormat) requestParams.output_format = options.outputFormat;
|
|
121
|
+
if (options.outputCompression !== undefined) requestParams.output_compression = options.outputCompression;
|
|
122
|
+
if (options.stream !== undefined) requestParams.stream = options.stream;
|
|
123
|
+
if (options.partialImages !== undefined) requestParams.partial_images = options.partialImages;
|
|
124
|
+
} else if (model === 'dall-e-2') {
|
|
125
|
+
if (options.size) requestParams.size = options.size;
|
|
126
|
+
requestParams.response_format = 'b64_json';
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const result: any = await client.images.edit(requestParams as any);
|
|
130
|
+
|
|
131
|
+
const images = (result.data || []).map((img: any) => ({
|
|
132
|
+
b64_json: img.b64_json,
|
|
133
|
+
url: img.url,
|
|
134
|
+
revisedPrompt: img.revised_prompt,
|
|
135
|
+
}));
|
|
136
|
+
|
|
137
|
+
return {
|
|
138
|
+
images,
|
|
139
|
+
metadata: {
|
|
140
|
+
model,
|
|
141
|
+
quality: result.quality,
|
|
142
|
+
size: result.size,
|
|
143
|
+
outputFormat: result.output_format,
|
|
144
|
+
tokensUsed: result.usage?.total_tokens,
|
|
145
|
+
},
|
|
146
|
+
};
|
|
147
|
+
}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import * as plugins from './plugins.js';
|
|
2
|
+
|
|
3
|
+
export interface IResearchOptions {
|
|
4
|
+
apiKey: string;
|
|
5
|
+
query: string;
|
|
6
|
+
searchDepth?: 'basic' | 'advanced' | 'deep';
|
|
7
|
+
maxSources?: number;
|
|
8
|
+
allowedDomains?: string[];
|
|
9
|
+
blockedDomains?: string[];
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export interface IResearchResponse {
|
|
13
|
+
answer: string;
|
|
14
|
+
sources: Array<{ url: string; title: string; snippet: string }>;
|
|
15
|
+
searchQueries?: string[];
|
|
16
|
+
metadata?: Record<string, unknown>;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export async function research(options: IResearchOptions): Promise<IResearchResponse> {
|
|
20
|
+
const client = new plugins.Anthropic({ apiKey: options.apiKey });
|
|
21
|
+
|
|
22
|
+
const systemMessage = `You are a research assistant with web search capabilities.
|
|
23
|
+
Provide comprehensive, well-researched answers with citations and sources.
|
|
24
|
+
When searching the web, be thorough and cite your sources accurately.`;
|
|
25
|
+
|
|
26
|
+
// Build web search tool config
|
|
27
|
+
const webSearchTool: any = {
|
|
28
|
+
type: 'web_search_20250305',
|
|
29
|
+
name: 'web_search',
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
if (options.maxSources) {
|
|
33
|
+
webSearchTool.max_uses = options.maxSources;
|
|
34
|
+
}
|
|
35
|
+
if (options.allowedDomains?.length) {
|
|
36
|
+
webSearchTool.allowed_domains = options.allowedDomains;
|
|
37
|
+
} else if (options.blockedDomains?.length) {
|
|
38
|
+
webSearchTool.blocked_domains = options.blockedDomains;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const result = await client.messages.create({
|
|
42
|
+
model: 'claude-sonnet-4-5-20250929',
|
|
43
|
+
system: systemMessage,
|
|
44
|
+
messages: [
|
|
45
|
+
{ role: 'user' as const, content: options.query },
|
|
46
|
+
],
|
|
47
|
+
max_tokens: 20000,
|
|
48
|
+
temperature: 0.7,
|
|
49
|
+
tools: [webSearchTool],
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
// Extract answer, sources, and search queries
|
|
53
|
+
let answer = '';
|
|
54
|
+
const sources: Array<{ url: string; title: string; snippet: string }> = [];
|
|
55
|
+
const searchQueries: string[] = [];
|
|
56
|
+
|
|
57
|
+
for (const block of result.content) {
|
|
58
|
+
const b: any = block;
|
|
59
|
+
if ('text' in b) {
|
|
60
|
+
answer += b.text;
|
|
61
|
+
|
|
62
|
+
// Extract citations if present
|
|
63
|
+
if (b.citations && Array.isArray(b.citations)) {
|
|
64
|
+
for (const citation of b.citations) {
|
|
65
|
+
if (citation.type === 'web_search_result_location') {
|
|
66
|
+
sources.push({
|
|
67
|
+
title: citation.title || '',
|
|
68
|
+
url: citation.url || '',
|
|
69
|
+
snippet: citation.cited_text || '',
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
} else if (b.type === 'server_tool_use') {
|
|
75
|
+
if (b.name === 'web_search' && b.input?.query) {
|
|
76
|
+
searchQueries.push(b.input.query);
|
|
77
|
+
}
|
|
78
|
+
} else if (b.type === 'web_search_tool_result') {
|
|
79
|
+
if (Array.isArray(b.content)) {
|
|
80
|
+
for (const item of b.content) {
|
|
81
|
+
if (item.type === 'web_search_result') {
|
|
82
|
+
if (!sources.some(s => s.url === item.url)) {
|
|
83
|
+
sources.push({
|
|
84
|
+
title: item.title || '',
|
|
85
|
+
url: item.url || '',
|
|
86
|
+
snippet: '',
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// Fallback: parse markdown links if no citations found
|
|
96
|
+
if (sources.length === 0) {
|
|
97
|
+
const urlRegex = /\[([^\]]+)\]\(([^)]+)\)/g;
|
|
98
|
+
let match: RegExpExecArray | null;
|
|
99
|
+
while ((match = urlRegex.exec(answer)) !== null) {
|
|
100
|
+
sources.push({
|
|
101
|
+
title: match[1],
|
|
102
|
+
url: match[2],
|
|
103
|
+
snippet: '',
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
const usage: any = result.usage;
|
|
109
|
+
return {
|
|
110
|
+
answer,
|
|
111
|
+
sources,
|
|
112
|
+
searchQueries: searchQueries.length > 0 ? searchQueries : undefined,
|
|
113
|
+
metadata: {
|
|
114
|
+
model: 'claude-sonnet-4-5-20250929',
|
|
115
|
+
searchDepth: options.searchDepth || 'basic',
|
|
116
|
+
tokensUsed: usage?.output_tokens,
|
|
117
|
+
webSearchesPerformed: usage?.server_tool_use?.web_search_requests ?? 0,
|
|
118
|
+
},
|
|
119
|
+
};
|
|
120
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import * as plugins from './plugins.js';
|
|
2
|
+
import type { LanguageModelV3 } from '@ai-sdk/provider';
|
|
3
|
+
|
|
4
|
+
export interface IVisionOptions {
|
|
5
|
+
model: LanguageModelV3;
|
|
6
|
+
image: Buffer | Uint8Array;
|
|
7
|
+
prompt: string;
|
|
8
|
+
mediaType?: 'image/jpeg' | 'image/png' | 'image/webp' | 'image/gif';
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export async function analyzeImage(options: IVisionOptions): Promise<string> {
|
|
12
|
+
const result = await plugins.generateText({
|
|
13
|
+
model: options.model,
|
|
14
|
+
messages: [
|
|
15
|
+
{
|
|
16
|
+
role: 'user',
|
|
17
|
+
content: [
|
|
18
|
+
{ type: 'text', text: options.prompt },
|
|
19
|
+
{
|
|
20
|
+
type: 'image',
|
|
21
|
+
image: options.image,
|
|
22
|
+
mediaType: options.mediaType ?? 'image/jpeg',
|
|
23
|
+
},
|
|
24
|
+
],
|
|
25
|
+
},
|
|
26
|
+
],
|
|
27
|
+
});
|
|
28
|
+
return result.text;
|
|
29
|
+
}
|
|
@@ -1,212 +0,0 @@
|
|
|
1
|
-
import * as plugins from './plugins.js';
|
|
2
|
-
/**
|
|
3
|
-
* Message format for chat interactions
|
|
4
|
-
*/
|
|
5
|
-
export interface ChatMessage {
|
|
6
|
-
role: 'assistant' | 'user' | 'system';
|
|
7
|
-
content: string;
|
|
8
|
-
/** Base64-encoded images for vision-capable models */
|
|
9
|
-
images?: string[];
|
|
10
|
-
/** Chain-of-thought reasoning for GPT-OSS models (e.g., Ollama) */
|
|
11
|
-
reasoning?: string;
|
|
12
|
-
}
|
|
13
|
-
/**
|
|
14
|
-
* Options for chat interactions
|
|
15
|
-
*/
|
|
16
|
-
export interface ChatOptions {
|
|
17
|
-
systemMessage: string;
|
|
18
|
-
userMessage: string;
|
|
19
|
-
messageHistory: ChatMessage[];
|
|
20
|
-
/** Base64-encoded images for the current message (vision-capable models) */
|
|
21
|
-
images?: string[];
|
|
22
|
-
}
|
|
23
|
-
/**
|
|
24
|
-
* Options for streaming chat interactions
|
|
25
|
-
*/
|
|
26
|
-
export interface StreamingChatOptions extends ChatOptions {
|
|
27
|
-
/** Callback fired for each token during generation */
|
|
28
|
-
onToken?: (token: string) => void;
|
|
29
|
-
}
|
|
30
|
-
/**
|
|
31
|
-
* Response format for chat interactions
|
|
32
|
-
*/
|
|
33
|
-
export interface ChatResponse {
|
|
34
|
-
role: 'assistant';
|
|
35
|
-
message: string;
|
|
36
|
-
/** Chain-of-thought reasoning from reasoning models */
|
|
37
|
-
reasoning?: string;
|
|
38
|
-
}
|
|
39
|
-
/**
|
|
40
|
-
* Options for research interactions
|
|
41
|
-
*/
|
|
42
|
-
export interface ResearchOptions {
|
|
43
|
-
query: string;
|
|
44
|
-
searchDepth?: 'basic' | 'advanced' | 'deep';
|
|
45
|
-
maxSources?: number;
|
|
46
|
-
includeWebSearch?: boolean;
|
|
47
|
-
background?: boolean;
|
|
48
|
-
}
|
|
49
|
-
/**
|
|
50
|
-
* Response format for research interactions
|
|
51
|
-
*/
|
|
52
|
-
export interface ResearchResponse {
|
|
53
|
-
answer: string;
|
|
54
|
-
sources: Array<{
|
|
55
|
-
url: string;
|
|
56
|
-
title: string;
|
|
57
|
-
snippet: string;
|
|
58
|
-
}>;
|
|
59
|
-
searchQueries?: string[];
|
|
60
|
-
metadata?: any;
|
|
61
|
-
}
|
|
62
|
-
/**
|
|
63
|
-
* Options for image generation
|
|
64
|
-
*/
|
|
65
|
-
export interface ImageGenerateOptions {
|
|
66
|
-
prompt: string;
|
|
67
|
-
model?: 'gpt-image-1' | 'dall-e-3' | 'dall-e-2';
|
|
68
|
-
quality?: 'low' | 'medium' | 'high' | 'standard' | 'hd' | 'auto';
|
|
69
|
-
size?: '256x256' | '512x512' | '1024x1024' | '1536x1024' | '1024x1536' | '1792x1024' | '1024x1792' | 'auto';
|
|
70
|
-
style?: 'vivid' | 'natural';
|
|
71
|
-
background?: 'transparent' | 'opaque' | 'auto';
|
|
72
|
-
outputFormat?: 'png' | 'jpeg' | 'webp';
|
|
73
|
-
outputCompression?: number;
|
|
74
|
-
moderation?: 'low' | 'auto';
|
|
75
|
-
n?: number;
|
|
76
|
-
stream?: boolean;
|
|
77
|
-
partialImages?: number;
|
|
78
|
-
}
|
|
79
|
-
/**
|
|
80
|
-
* Options for image editing
|
|
81
|
-
*/
|
|
82
|
-
export interface ImageEditOptions {
|
|
83
|
-
image: Buffer;
|
|
84
|
-
prompt: string;
|
|
85
|
-
mask?: Buffer;
|
|
86
|
-
model?: 'gpt-image-1' | 'dall-e-2';
|
|
87
|
-
quality?: 'low' | 'medium' | 'high' | 'standard' | 'auto';
|
|
88
|
-
size?: '256x256' | '512x512' | '1024x1024' | '1536x1024' | '1024x1536' | 'auto';
|
|
89
|
-
background?: 'transparent' | 'opaque' | 'auto';
|
|
90
|
-
outputFormat?: 'png' | 'jpeg' | 'webp';
|
|
91
|
-
outputCompression?: number;
|
|
92
|
-
n?: number;
|
|
93
|
-
stream?: boolean;
|
|
94
|
-
partialImages?: number;
|
|
95
|
-
}
|
|
96
|
-
/**
|
|
97
|
-
* Response format for image operations
|
|
98
|
-
*/
|
|
99
|
-
export interface ImageResponse {
|
|
100
|
-
images: Array<{
|
|
101
|
-
b64_json?: string;
|
|
102
|
-
url?: string;
|
|
103
|
-
revisedPrompt?: string;
|
|
104
|
-
}>;
|
|
105
|
-
metadata?: {
|
|
106
|
-
model: string;
|
|
107
|
-
quality?: string;
|
|
108
|
-
size?: string;
|
|
109
|
-
outputFormat?: string;
|
|
110
|
-
tokensUsed?: number;
|
|
111
|
-
};
|
|
112
|
-
}
|
|
113
|
-
/**
|
|
114
|
-
* Abstract base class for multi-modal AI models.
|
|
115
|
-
* Provides a common interface for different AI providers (OpenAI, Anthropic, Perplexity, Ollama)
|
|
116
|
-
*/
|
|
117
|
-
export declare abstract class MultiModalModel {
|
|
118
|
-
/**
|
|
119
|
-
* SmartPdf instance for document processing
|
|
120
|
-
* Lazy-loaded only when PDF processing is needed to avoid starting browser unnecessarily
|
|
121
|
-
*/
|
|
122
|
-
protected smartpdfInstance: plugins.smartpdf.SmartPdf | null;
|
|
123
|
-
/**
|
|
124
|
-
* Ensures SmartPdf instance is initialized and ready
|
|
125
|
-
* Call this before using smartpdfInstance in document processing methods
|
|
126
|
-
*/
|
|
127
|
-
protected ensureSmartpdfReady(): Promise<void>;
|
|
128
|
-
/**
|
|
129
|
-
* Initializes the model and any necessary resources
|
|
130
|
-
* Should be called before using any other methods
|
|
131
|
-
*/
|
|
132
|
-
start(): Promise<void>;
|
|
133
|
-
/**
|
|
134
|
-
* Cleans up any resources used by the model
|
|
135
|
-
* Should be called when the model is no longer needed
|
|
136
|
-
*/
|
|
137
|
-
stop(): Promise<void>;
|
|
138
|
-
/**
|
|
139
|
-
* Synchronous chat interaction with the model
|
|
140
|
-
* @param optionsArg Options containing system message, user message, and message history
|
|
141
|
-
* @returns Promise resolving to the assistant's response
|
|
142
|
-
*/
|
|
143
|
-
abstract chat(optionsArg: ChatOptions): Promise<ChatResponse>;
|
|
144
|
-
/**
|
|
145
|
-
* Streaming interface for chat interactions
|
|
146
|
-
* Allows for real-time responses from the model
|
|
147
|
-
* @param input Stream of user messages
|
|
148
|
-
* @returns Stream of model responses
|
|
149
|
-
*/
|
|
150
|
-
abstract chatStream(input: ReadableStream<Uint8Array>): Promise<ReadableStream<string>>;
|
|
151
|
-
/**
|
|
152
|
-
* Streaming chat with token callback
|
|
153
|
-
* Calls onToken for each token generated, returns final response
|
|
154
|
-
* @param optionsArg Options containing system message, user message, message history, and onToken callback
|
|
155
|
-
* @returns Promise resolving to the assistant's response
|
|
156
|
-
*/
|
|
157
|
-
chatStreaming?(optionsArg: StreamingChatOptions): Promise<ChatResponse>;
|
|
158
|
-
/**
|
|
159
|
-
* Text-to-speech conversion
|
|
160
|
-
* @param optionsArg Options containing the message to convert to speech
|
|
161
|
-
* @returns Promise resolving to a readable stream of audio data
|
|
162
|
-
* @throws Error if the provider doesn't support audio generation
|
|
163
|
-
*/
|
|
164
|
-
abstract audio(optionsArg: {
|
|
165
|
-
message: string;
|
|
166
|
-
}): Promise<NodeJS.ReadableStream>;
|
|
167
|
-
/**
|
|
168
|
-
* Vision-language processing
|
|
169
|
-
* @param optionsArg Options containing the image and prompt for analysis
|
|
170
|
-
* @returns Promise resolving to the model's description or analysis of the image
|
|
171
|
-
* @throws Error if the provider doesn't support vision tasks
|
|
172
|
-
*/
|
|
173
|
-
abstract vision(optionsArg: {
|
|
174
|
-
image: Buffer;
|
|
175
|
-
prompt: string;
|
|
176
|
-
}): Promise<string>;
|
|
177
|
-
/**
|
|
178
|
-
* Document analysis and processing
|
|
179
|
-
* @param optionsArg Options containing system message, user message, PDF documents, and message history
|
|
180
|
-
* @returns Promise resolving to the model's analysis of the documents
|
|
181
|
-
* @throws Error if the provider doesn't support document processing
|
|
182
|
-
*/
|
|
183
|
-
abstract document(optionsArg: {
|
|
184
|
-
systemMessage: string;
|
|
185
|
-
userMessage: string;
|
|
186
|
-
pdfDocuments: Uint8Array[];
|
|
187
|
-
messageHistory: ChatMessage[];
|
|
188
|
-
}): Promise<{
|
|
189
|
-
message: any;
|
|
190
|
-
}>;
|
|
191
|
-
/**
|
|
192
|
-
* Research and web search capabilities
|
|
193
|
-
* @param optionsArg Options containing the research query and configuration
|
|
194
|
-
* @returns Promise resolving to the research results with sources
|
|
195
|
-
* @throws Error if the provider doesn't support research capabilities
|
|
196
|
-
*/
|
|
197
|
-
abstract research(optionsArg: ResearchOptions): Promise<ResearchResponse>;
|
|
198
|
-
/**
|
|
199
|
-
* Image generation from text prompts
|
|
200
|
-
* @param optionsArg Options containing the prompt and generation parameters
|
|
201
|
-
* @returns Promise resolving to the generated image(s)
|
|
202
|
-
* @throws Error if the provider doesn't support image generation
|
|
203
|
-
*/
|
|
204
|
-
abstract imageGenerate(optionsArg: ImageGenerateOptions): Promise<ImageResponse>;
|
|
205
|
-
/**
|
|
206
|
-
* Image editing and inpainting
|
|
207
|
-
* @param optionsArg Options containing the image, prompt, and editing parameters
|
|
208
|
-
* @returns Promise resolving to the edited image(s)
|
|
209
|
-
* @throws Error if the provider doesn't support image editing
|
|
210
|
-
*/
|
|
211
|
-
abstract imageEdit(optionsArg: ImageEditOptions): Promise<ImageResponse>;
|
|
212
|
-
}
|
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
import * as plugins from './plugins.js';
|
|
2
|
-
/**
|
|
3
|
-
* Abstract base class for multi-modal AI models.
|
|
4
|
-
* Provides a common interface for different AI providers (OpenAI, Anthropic, Perplexity, Ollama)
|
|
5
|
-
*/
|
|
6
|
-
export class MultiModalModel {
|
|
7
|
-
constructor() {
|
|
8
|
-
/**
|
|
9
|
-
* SmartPdf instance for document processing
|
|
10
|
-
* Lazy-loaded only when PDF processing is needed to avoid starting browser unnecessarily
|
|
11
|
-
*/
|
|
12
|
-
this.smartpdfInstance = null;
|
|
13
|
-
}
|
|
14
|
-
/**
|
|
15
|
-
* Ensures SmartPdf instance is initialized and ready
|
|
16
|
-
* Call this before using smartpdfInstance in document processing methods
|
|
17
|
-
*/
|
|
18
|
-
async ensureSmartpdfReady() {
|
|
19
|
-
if (!this.smartpdfInstance) {
|
|
20
|
-
this.smartpdfInstance = new plugins.smartpdf.SmartPdf();
|
|
21
|
-
await this.smartpdfInstance.start();
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
/**
|
|
25
|
-
* Initializes the model and any necessary resources
|
|
26
|
-
* Should be called before using any other methods
|
|
27
|
-
*/
|
|
28
|
-
async start() {
|
|
29
|
-
// SmartPdf is now lazy-loaded only when needed for PDF processing
|
|
30
|
-
// This avoids starting a browser unless document() method is actually used
|
|
31
|
-
}
|
|
32
|
-
/**
|
|
33
|
-
* Cleans up any resources used by the model
|
|
34
|
-
* Should be called when the model is no longer needed
|
|
35
|
-
*/
|
|
36
|
-
async stop() {
|
|
37
|
-
if (this.smartpdfInstance) {
|
|
38
|
-
await this.smartpdfInstance.stop();
|
|
39
|
-
this.smartpdfInstance = null;
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiYWJzdHJhY3QuY2xhc3Nlcy5tdWx0aW1vZGFsLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vdHMvYWJzdHJhY3QuY2xhc3Nlcy5tdWx0aW1vZGFsLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sS0FBSyxPQUFPLE1BQU0sY0FBYyxDQUFDO0FBMEh4Qzs7O0dBR0c7QUFDSCxNQUFNLE9BQWdCLGVBQWU7SUFBckM7UUFDRTs7O1dBR0c7UUFDTyxxQkFBZ0IsR0FBcUMsSUFBSSxDQUFDO0lBNEd0RSxDQUFDO0lBMUdDOzs7T0FHRztJQUNPLEtBQUssQ0FBQyxtQkFBbUI7UUFDakMsSUFBSSxDQUFDLElBQUksQ0FBQyxnQkFBZ0IsRUFBRSxDQUFDO1lBQzNCLElBQUksQ0FBQyxnQkFBZ0IsR0FBRyxJQUFJLE9BQU8sQ0FBQyxRQUFRLENBQUMsUUFBUSxFQUFFLENBQUM7WUFDeEQsTUFBTSxJQUFJLENBQUMsZ0JBQWdCLENBQUMsS0FBSyxFQUFFLENBQUM7UUFDdEMsQ0FBQztJQUNILENBQUM7SUFFRDs7O09BR0c7SUFDSSxLQUFLLENBQUMsS0FBSztRQUNoQixrRUFBa0U7UUFDbEUsMkVBQTJFO0lBQzdFLENBQUM7SUFFRDs7O09BR0c7SUFDSSxLQUFLLENBQUMsSUFBSTtRQUNmLElBQUksSUFBSSxDQUFDLGdCQUFnQixFQUFFLENBQUM7WUFDMUIsTUFBTSxJQUFJLENBQUMsZ0JBQWdCLENBQUMsSUFBSSxFQUFFLENBQUM7WUFDbkMsSUFBSSxDQUFDLGdCQUFnQixHQUFHLElBQUksQ0FBQztRQUMvQixDQUFDO0lBQ0gsQ0FBQztDQTZFRiJ9
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
import type { SmartAi } from "./classes.smartai.js";
|
|
2
|
-
type TProcessFunction = (input: string) => Promise<string>;
|
|
3
|
-
export interface IConversationOptions {
|
|
4
|
-
processFunction: TProcessFunction;
|
|
5
|
-
}
|
|
6
|
-
/**
|
|
7
|
-
* a conversation
|
|
8
|
-
*/
|
|
9
|
-
export declare class Conversation {
|
|
10
|
-
static createWithOpenAi(smartaiRefArg: SmartAi): Promise<Conversation>;
|
|
11
|
-
static createWithAnthropic(smartaiRefArg: SmartAi): Promise<Conversation>;
|
|
12
|
-
static createWithPerplexity(smartaiRefArg: SmartAi): Promise<Conversation>;
|
|
13
|
-
static createWithExo(smartaiRefArg: SmartAi): Promise<Conversation>;
|
|
14
|
-
static createWithOllama(smartaiRefArg: SmartAi): Promise<Conversation>;
|
|
15
|
-
static createWithGroq(smartaiRefArg: SmartAi): Promise<Conversation>;
|
|
16
|
-
static createWithMistral(smartaiRefArg: SmartAi): Promise<Conversation>;
|
|
17
|
-
static createWithXai(smartaiRefArg: SmartAi): Promise<Conversation>;
|
|
18
|
-
static createWithElevenlabs(smartaiRefArg: SmartAi): Promise<Conversation>;
|
|
19
|
-
smartaiRef: SmartAi;
|
|
20
|
-
private systemMessage;
|
|
21
|
-
private processFunction;
|
|
22
|
-
private inputStreamWriter;
|
|
23
|
-
private outputStreamController;
|
|
24
|
-
constructor(smartairefArg: SmartAi, options: IConversationOptions);
|
|
25
|
-
setSystemMessage(systemMessageArg: string): Promise<void>;
|
|
26
|
-
private setupOutputStream;
|
|
27
|
-
private setupInputStream;
|
|
28
|
-
getInputStreamWriter(): WritableStreamDefaultWriter<string>;
|
|
29
|
-
getOutputStream(): ReadableStream<string>;
|
|
30
|
-
}
|
|
31
|
-
export {};
|