@axium/ai 0.0.0-stage → 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +157 -0
- package/README.md +2 -2
- package/db.json +114 -0
- package/dist/.locale.d.ts +300 -0
- package/dist/.locale.js +1 -0
- package/dist/client/cli.d.ts +1 -0
- package/dist/client/cli.js +224 -0
- package/dist/client/config.d.ts +196 -0
- package/dist/client/config.js +55 -0
- package/dist/client/hooks.d.ts +3 -0
- package/dist/client/hooks.js +32 -0
- package/dist/client/local.d.ts +17 -0
- package/dist/client/local.js +112 -0
- package/dist/common.d.ts +1318 -0
- package/dist/common.js +569 -0
- package/dist/hooks.d.ts +13 -0
- package/dist/hooks.js +34 -0
- package/dist/hub.d.ts +4 -0
- package/dist/hub.js +75 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/integrations/notes.d.ts +1 -0
- package/dist/integrations/notes.js +104 -0
- package/dist/integrations/tasks.d.ts +1 -0
- package/dist/integrations/tasks.js +157 -0
- package/dist/loop.d.ts +75 -0
- package/dist/loop.js +238 -0
- package/dist/memory.d.ts +107 -0
- package/dist/memory.js +193 -0
- package/dist/node/chat.d.ts +50 -0
- package/dist/node/chat.js +60 -0
- package/dist/node/llama.d.ts +115 -0
- package/dist/node/llama.js +385 -0
- package/dist/node/markdown.d.ts +6 -0
- package/dist/node/markdown.js +285 -0
- package/dist/node/models.d.ts +34 -0
- package/dist/node/models.js +287 -0
- package/dist/node/web.d.ts +1 -0
- package/dist/node/web.js +203 -0
- package/dist/server/api/conversations.d.ts +1 -0
- package/dist/server/api/conversations.js +216 -0
- package/dist/server/api/gateway.d.ts +1 -0
- package/dist/server/api/gateway.js +113 -0
- package/dist/server/api/memories.d.ts +1 -0
- package/dist/server/api/memories.js +129 -0
- package/dist/server/api/models.d.ts +1 -0
- package/dist/server/api/models.js +161 -0
- package/dist/server/api/temporary.d.ts +1 -0
- package/dist/server/api/temporary.js +50 -0
- package/dist/server/api/timers.d.ts +1 -0
- package/dist/server/api/timers.js +27 -0
- package/dist/server/api/voice.d.ts +1 -0
- package/dist/server/api/voice.js +188 -0
- package/dist/server/api.d.ts +1 -0
- package/dist/server/api.js +157 -0
- package/dist/server/generation.d.ts +38 -0
- package/dist/server/generation.js +299 -0
- package/dist/server/memory.d.ts +7 -0
- package/dist/server/memory.js +218 -0
- package/dist/server/models.d.ts +13 -0
- package/dist/server/models.js +59 -0
- package/dist/server/preferences.d.ts +3 -0
- package/dist/server/preferences.js +12 -0
- package/dist/server/timers.d.ts +7 -0
- package/dist/server/timers.js +66 -0
- package/dist/speech.d.ts +17 -0
- package/dist/speech.js +69 -0
- package/dist/timers.d.ts +1 -0
- package/dist/timers.js +67 -0
- package/dist/tools.d.ts +70 -0
- package/dist/tools.js +84 -0
- package/lib/ChatMessage.svelte +331 -0
- package/lib/Composer.svelte +234 -0
- package/lib/Conversation.svelte +433 -0
- package/lib/ConversationMenu.svelte +18 -0
- package/lib/ModelActions.svelte +71 -0
- package/lib/ModelOptions.svelte +101 -0
- package/lib/ModelStatus.svelte +49 -0
- package/lib/ToolCall.svelte +171 -0
- package/lib/VoiceMode.svelte +113 -0
- package/lib/actions.ts +21 -0
- package/lib/index.ts +7 -0
- package/lib/live.svelte.ts +113 -0
- package/lib/markdown.ts +40 -0
- package/lib/preferences.ts +13 -0
- package/lib/transition.ts +24 -0
- package/lib/tsconfig.json +14 -0
- package/lib/voice.svelte.ts +476 -0
- package/locales/en.json +316 -0
- package/package.json +139 -5
- package/routes/ai/+layout.svelte +177 -0
- package/routes/ai/+layout.ts +26 -0
- package/routes/ai/+page.svelte +109 -0
- package/routes/ai/TimerAlerts.svelte +51 -0
- package/routes/ai/Timers.svelte +121 -0
- package/routes/ai/[id]/+page.svelte +18 -0
- package/routes/ai/[id]/+page.ts +14 -0
- package/routes/ai/memory/+page.svelte +313 -0
- package/routes/ai/memory/+page.ts +10 -0
- package/routes/ai/memory/MemoryForm.svelte +111 -0
- package/routes/ai/models/+page.svelte +199 -0
- package/routes/ai/models/+page.ts +5 -0
- package/routes/ai/models/AddModel.svelte +240 -0
- package/routes/ai/models/MemoryMeter.svelte +120 -0
- package/routes/ai/models/Roles.svelte +159 -0
- package/routes/ai/models/[id]/+page.svelte +331 -0
- package/routes/ai/models/[id]/+page.ts +6 -0
- package/routes/ai/timers.svelte.ts +94 -0
- package/routes/tsconfig.json +14 -0
package/dist/memory.d.ts
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import * as z from 'zod';
|
|
2
|
+
/**
|
|
3
|
+
* Where a memory came from: added by the user on the memory page (`user`), saved when the user asked (`explicit`),
|
|
4
|
+
* saved because the model judged it worth keeping (`inferred`), or found by reviewing conversations (`periodic`)
|
|
5
|
+
*/
|
|
6
|
+
export declare const Source: z.ZodLiteral<"user" | "explicit" | "inferred" | "periodic">;
|
|
7
|
+
export type Source = z.infer<typeof Source>;
|
|
8
|
+
/** Unique for each user, in `kebab-case` */
|
|
9
|
+
export declare const Id: z.ZodString;
|
|
10
|
+
/** One line saying what a memory is about, shown to the model in every conversation */
|
|
11
|
+
export declare const Description: z.ZodString;
|
|
12
|
+
/** Details the model reads only when relevant; limited further by the `max_memory_length` config */
|
|
13
|
+
export declare const Content: z.ZodString;
|
|
14
|
+
/** A memory */
|
|
15
|
+
export declare const Node: z.ZodObject<{
|
|
16
|
+
id: z.ZodString;
|
|
17
|
+
userId: z.ZodUUID;
|
|
18
|
+
description: z.ZodString;
|
|
19
|
+
content: z.ZodString;
|
|
20
|
+
source: z.ZodLiteral<"user" | "explicit" | "inferred" | "periodic">;
|
|
21
|
+
conversationId: z.ZodNullable<z.ZodUUID>;
|
|
22
|
+
created: z.ZodCoercedDate<unknown>;
|
|
23
|
+
modified: z.ZodCoercedDate<unknown>;
|
|
24
|
+
}, z.core.$strip>;
|
|
25
|
+
export interface Node extends z.infer<typeof Node> {
|
|
26
|
+
}
|
|
27
|
+
export declare const Init: z.ZodObject<{
|
|
28
|
+
id: z.ZodString;
|
|
29
|
+
description: z.ZodString;
|
|
30
|
+
content: z.ZodDefault<z.ZodString>;
|
|
31
|
+
source: z.ZodDefault<z.ZodLiteral<"user" | "explicit" | "inferred" | "periodic">>;
|
|
32
|
+
conversationId: z.ZodOptional<z.ZodNullable<z.ZodUUID>>;
|
|
33
|
+
}, z.core.$strip>;
|
|
34
|
+
export interface Init extends z.infer<typeof Init> {
|
|
35
|
+
}
|
|
36
|
+
export declare const Update: z.ZodObject<{
|
|
37
|
+
id: z.ZodOptional<z.ZodString>;
|
|
38
|
+
description: z.ZodOptional<z.ZodString>;
|
|
39
|
+
content: z.ZodOptional<z.ZodString>;
|
|
40
|
+
}, z.core.$strip>;
|
|
41
|
+
export interface Update extends z.infer<typeof Update> {
|
|
42
|
+
}
|
|
43
|
+
/** A user's memories with the limits on them */
|
|
44
|
+
export declare const List: z.ZodObject<{
|
|
45
|
+
memories: z.ZodArray<z.ZodObject<{
|
|
46
|
+
id: z.ZodString;
|
|
47
|
+
userId: z.ZodUUID;
|
|
48
|
+
description: z.ZodString;
|
|
49
|
+
content: z.ZodString;
|
|
50
|
+
source: z.ZodLiteral<"user" | "explicit" | "inferred" | "periodic">;
|
|
51
|
+
conversationId: z.ZodNullable<z.ZodUUID>;
|
|
52
|
+
created: z.ZodCoercedDate<unknown>;
|
|
53
|
+
modified: z.ZodCoercedDate<unknown>;
|
|
54
|
+
}, z.core.$strip>>;
|
|
55
|
+
limit: z.ZodInt;
|
|
56
|
+
maxLength: z.ZodInt;
|
|
57
|
+
}, z.core.$strip>;
|
|
58
|
+
export interface List extends z.infer<typeof List> {
|
|
59
|
+
}
|
|
60
|
+
export declare const UpdateFrequency: z.ZodLiteral<"off" | "daily" | "weekly">;
|
|
61
|
+
export type UpdateFrequency = z.infer<typeof UpdateFrequency>;
|
|
62
|
+
/** IDs of the memories an update changed */
|
|
63
|
+
export declare const ChangeIds: z.ZodObject<{
|
|
64
|
+
added: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
65
|
+
updated: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
66
|
+
deleted: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
67
|
+
}, z.core.$strip>;
|
|
68
|
+
export interface ChangeIds extends z.infer<typeof ChangeIds> {
|
|
69
|
+
}
|
|
70
|
+
/** The state of a user's periodic memory updates */
|
|
71
|
+
export declare const Updates: z.ZodObject<{
|
|
72
|
+
enabled: z.ZodBoolean;
|
|
73
|
+
frequency: z.ZodLiteral<"off" | "daily" | "weekly">;
|
|
74
|
+
ran: z.ZodNullable<z.ZodCoercedDate<unknown>>;
|
|
75
|
+
summary: z.ZodNullable<z.ZodString>;
|
|
76
|
+
changes: z.ZodNullable<z.ZodObject<{
|
|
77
|
+
added: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
78
|
+
updated: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
79
|
+
deleted: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
80
|
+
}, z.core.$strip>>;
|
|
81
|
+
next: z.ZodNullable<z.ZodCoercedDate<unknown>>;
|
|
82
|
+
running: z.ZodBoolean;
|
|
83
|
+
}, z.core.$strip>;
|
|
84
|
+
export interface Updates extends z.infer<typeof Updates> {
|
|
85
|
+
}
|
|
86
|
+
/** The changes a model proposes when updating memories, as structured output */
|
|
87
|
+
export declare const Changes: z.ZodObject<{
|
|
88
|
+
add: z.ZodArray<z.ZodObject<{
|
|
89
|
+
id: z.ZodString;
|
|
90
|
+
description: z.ZodString;
|
|
91
|
+
content: z.ZodString;
|
|
92
|
+
}, z.core.$strip>>;
|
|
93
|
+
update: z.ZodArray<z.ZodObject<{
|
|
94
|
+
id: z.ZodString;
|
|
95
|
+
description: z.ZodOptional<z.ZodString>;
|
|
96
|
+
content: z.ZodOptional<z.ZodString>;
|
|
97
|
+
}, z.core.$strip>>;
|
|
98
|
+
delete: z.ZodArray<z.ZodString>;
|
|
99
|
+
summary: z.ZodString;
|
|
100
|
+
}, z.core.$strip>;
|
|
101
|
+
export interface Changes extends z.infer<typeof Changes> {
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* The system prompt section about memory. Only descriptions are included; the model reads a memory's content with `read_memory`.
|
|
105
|
+
* @param writable Whether the tools that change memories are offered
|
|
106
|
+
*/
|
|
107
|
+
export declare function prompt(memories: Node[], writable: boolean): string | undefined;
|
package/dist/memory.js
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
import { pick } from 'utilium';
|
|
2
|
+
import * as z from 'zod';
|
|
3
|
+
import * as tools from './tools.js';
|
|
4
|
+
/**
|
|
5
|
+
* Where a memory came from: added by the user on the memory page (`user`), saved when the user asked (`explicit`),
|
|
6
|
+
* saved because the model judged it worth keeping (`inferred`), or found by reviewing conversations (`periodic`)
|
|
7
|
+
*/
|
|
8
|
+
export const Source = z.literal(['user', 'explicit', 'inferred', 'periodic']);
|
|
9
|
+
/** Unique for each user, in `kebab-case` */
|
|
10
|
+
export const Id = z
|
|
11
|
+
.string()
|
|
12
|
+
.min(1)
|
|
13
|
+
.max(64)
|
|
14
|
+
.regex(/^[a-z0-9]+(-[a-z0-9]+)*$/, 'Use lowercase words separated by hyphens, e.g. "coffee-preferences"');
|
|
15
|
+
/** One line saying what a memory is about, shown to the model in every conversation */
|
|
16
|
+
export const Description = z.string().trim().min(1).max(200);
|
|
17
|
+
/** Details the model reads only when relevant; limited further by the `max_memory_length` config */
|
|
18
|
+
export const Content = z.string().trim().max(10_000);
|
|
19
|
+
/** A memory */
|
|
20
|
+
export const Node = z.object({
|
|
21
|
+
id: Id,
|
|
22
|
+
userId: z.uuid(),
|
|
23
|
+
description: z.string(),
|
|
24
|
+
content: z.string(),
|
|
25
|
+
source: Source,
|
|
26
|
+
/** The conversation the memory was saved in, while it exists */
|
|
27
|
+
conversationId: z.uuid().nullable(),
|
|
28
|
+
created: z.coerce.date(),
|
|
29
|
+
modified: z.coerce.date(),
|
|
30
|
+
});
|
|
31
|
+
export const Init = z.object({
|
|
32
|
+
id: Id,
|
|
33
|
+
description: Description,
|
|
34
|
+
content: Content.default(''),
|
|
35
|
+
source: Source.default('user'),
|
|
36
|
+
conversationId: z.uuid().nullish(),
|
|
37
|
+
});
|
|
38
|
+
export const Update = z.object({
|
|
39
|
+
id: Id.optional(),
|
|
40
|
+
description: Description.optional(),
|
|
41
|
+
content: Content.optional(),
|
|
42
|
+
});
|
|
43
|
+
/** A user's memories with the limits on them */
|
|
44
|
+
export const List = z.object({
|
|
45
|
+
memories: Node.array(),
|
|
46
|
+
/** The most memories a user may have */
|
|
47
|
+
limit: z.int().positive(),
|
|
48
|
+
/** The most characters in a memory's content */
|
|
49
|
+
maxLength: z.int().positive(),
|
|
50
|
+
});
|
|
51
|
+
export const UpdateFrequency = z.literal(['off', 'daily', 'weekly']);
|
|
52
|
+
/** IDs of the memories an update changed */
|
|
53
|
+
export const ChangeIds = z.object({
|
|
54
|
+
added: Id.array().default([]),
|
|
55
|
+
updated: Id.array().default([]),
|
|
56
|
+
deleted: Id.array().default([]),
|
|
57
|
+
});
|
|
58
|
+
/** The state of a user's periodic memory updates */
|
|
59
|
+
export const Updates = z.object({
|
|
60
|
+
/** Whether the server allows them */
|
|
61
|
+
enabled: z.boolean(),
|
|
62
|
+
frequency: UpdateFrequency,
|
|
63
|
+
/** When the last update finished */
|
|
64
|
+
ran: z.coerce.date().nullable(),
|
|
65
|
+
/** What the last update changed, as the model described it */
|
|
66
|
+
summary: z.string().nullable(),
|
|
67
|
+
changes: ChangeIds.nullable(),
|
|
68
|
+
/** When the next update starts */
|
|
69
|
+
next: z.coerce.date().nullable(),
|
|
70
|
+
running: z.boolean(),
|
|
71
|
+
});
|
|
72
|
+
/** The changes a model proposes when updating memories, as structured output */
|
|
73
|
+
export const Changes = z.object({
|
|
74
|
+
add: z.object({ id: Id, description: Description, content: Content }).array().max(20).describe('New memories'),
|
|
75
|
+
update: z
|
|
76
|
+
.object({ id: Id, description: Description.optional(), content: Content.optional() })
|
|
77
|
+
.array()
|
|
78
|
+
.max(20)
|
|
79
|
+
.describe('Changes to existing memories; only the fields that change'),
|
|
80
|
+
delete: Id.array().max(20).describe('IDs of memories to delete'),
|
|
81
|
+
summary: z.string().max(300).describe('One sentence for the user saying what changed, or that nothing did'),
|
|
82
|
+
});
|
|
83
|
+
const writeGuidance = 'You have a long-term memory of the user. When they share something lasting about themselves that would help in future conversations, ' +
|
|
84
|
+
"such as preferences, facts about their life or ongoing projects, save it with add_memory, even if they didn't ask. " +
|
|
85
|
+
"Don't save trivia, one-off requests or things that only matter in this conversation. " +
|
|
86
|
+
'Each memory has a kebab-case ID, a one-line description that you see in every conversation, and optional content with details, ' +
|
|
87
|
+
'e.g. ID "coffee", description "How the user takes their coffee", content "- Oat milk, no sugar". ' +
|
|
88
|
+
'Prefer adding to a related memory with update_memory over creating a new one, and update or delete memories that are wrong or outdated.';
|
|
89
|
+
/**
|
|
90
|
+
* The system prompt section about memory. Only descriptions are included; the model reads a memory's content with `read_memory`.
|
|
91
|
+
* @param writable Whether the tools that change memories are offered
|
|
92
|
+
*/
|
|
93
|
+
export function prompt(memories, writable) {
|
|
94
|
+
if (!memories.length && !writable)
|
|
95
|
+
return;
|
|
96
|
+
const sections = [];
|
|
97
|
+
if (writable)
|
|
98
|
+
sections.push(writeGuidance);
|
|
99
|
+
if (!memories.length)
|
|
100
|
+
sections.push("You don't remember anything about the user yet.");
|
|
101
|
+
else {
|
|
102
|
+
sections.push('Your memories of the user from earlier conversations, by ID with a description. ' +
|
|
103
|
+
'Read one with read_memory when its details would help. Use what you remember without mentioning it unless asked:\n' +
|
|
104
|
+
memories.map(memory => `- ${memory.id}: ${memory.description.replace(/\s+/g, ' ')}`).join('\n'));
|
|
105
|
+
}
|
|
106
|
+
return sections.join('\n\n');
|
|
107
|
+
}
|
|
108
|
+
function full(memory) {
|
|
109
|
+
return pick(memory, 'id', 'description', 'content', 'modified');
|
|
110
|
+
}
|
|
111
|
+
const IdInput = Id.describe('The memory\'s ID, in kebab-case, e.g. "coffee-preferences"');
|
|
112
|
+
const DescriptionInput = Description.describe('One line saying what the memory is about');
|
|
113
|
+
const ContentInput = Content.describe('Details, as Markdown, usually a short list of facts about the user written in the third person');
|
|
114
|
+
tools.add({
|
|
115
|
+
name: 'read_memory',
|
|
116
|
+
description: "Reads a memory's content.",
|
|
117
|
+
environment: 'memory',
|
|
118
|
+
access: 'read',
|
|
119
|
+
input: z.object({ id: IdInput }),
|
|
120
|
+
async run({ id }, { api, userId }) {
|
|
121
|
+
return full(await api('GET', 'users/:id/ai/memories/:memory', {}, userId, id));
|
|
122
|
+
},
|
|
123
|
+
});
|
|
124
|
+
/** The first line of `content` without list markers, for memories saved without a description */
|
|
125
|
+
function firstLine(content) {
|
|
126
|
+
const line = content.split('\n').find(line => line.trim()) ?? '';
|
|
127
|
+
return line
|
|
128
|
+
.replace(/^\s*(?:[-*+]|\d+[.)]|#+)\s*/, '')
|
|
129
|
+
.trim()
|
|
130
|
+
.slice(0, 200);
|
|
131
|
+
}
|
|
132
|
+
tools.add({
|
|
133
|
+
name: 'add_memory',
|
|
134
|
+
description: 'Saves something about the user to remember in future conversations. ' +
|
|
135
|
+
'Give an ID and a one-line description; put details in content.',
|
|
136
|
+
environment: 'memory',
|
|
137
|
+
access: 'write',
|
|
138
|
+
approval: 'allow',
|
|
139
|
+
input: z.object({
|
|
140
|
+
id: IdInput.describe('A new kebab-case ID naming what the memory is about, e.g. "coffee-preferences"'),
|
|
141
|
+
description: DescriptionInput.optional().describe('One line saying what the memory is about. Defaults to the first line of content.'),
|
|
142
|
+
content: ContentInput.optional(),
|
|
143
|
+
requested: z.boolean().optional().describe('Whether the user asked you to remember this'),
|
|
144
|
+
}),
|
|
145
|
+
async run({ requested, description, ...init }, { api, userId, conversationId }) {
|
|
146
|
+
description ||= init.content && firstLine(init.content);
|
|
147
|
+
if (!description)
|
|
148
|
+
throw new Error('Give a description, or content to take it from');
|
|
149
|
+
const source = requested ? 'explicit' : 'inferred';
|
|
150
|
+
return full(await api('PUT', 'users/:id/ai/memories', { ...init, description, source, conversationId }, userId));
|
|
151
|
+
},
|
|
152
|
+
});
|
|
153
|
+
tools.add({
|
|
154
|
+
name: 'update_memory',
|
|
155
|
+
description: 'Changes a memory. Use `append` to add to its content, or `content` to replace it; only the given fields change.',
|
|
156
|
+
environment: 'memory',
|
|
157
|
+
access: 'write',
|
|
158
|
+
approval: 'allow',
|
|
159
|
+
input: z.object({
|
|
160
|
+
id: IdInput,
|
|
161
|
+
description: DescriptionInput.optional(),
|
|
162
|
+
content: ContentInput.optional().describe('New content, replacing the old'),
|
|
163
|
+
append: z
|
|
164
|
+
.string()
|
|
165
|
+
.trim()
|
|
166
|
+
.min(1)
|
|
167
|
+
.max(2000)
|
|
168
|
+
.optional()
|
|
169
|
+
.describe('A line to add to the end of the content, e.g. "- Prefers oat milk"'),
|
|
170
|
+
rename: IdInput.optional().describe('A new ID for the memory'),
|
|
171
|
+
}),
|
|
172
|
+
async run({ id, description, content, append, rename }, { api, userId }) {
|
|
173
|
+
if (content !== undefined && append)
|
|
174
|
+
throw new Error('Give either content or append, not both');
|
|
175
|
+
if (append) {
|
|
176
|
+
const existing = await api('GET', 'users/:id/ai/memories/:memory', {}, userId, id);
|
|
177
|
+
content = existing.content ? existing.content + '\n' + append : append;
|
|
178
|
+
}
|
|
179
|
+
return full(await api('PATCH', 'users/:id/ai/memories/:memory', { id: rename, description, content }, userId, id));
|
|
180
|
+
},
|
|
181
|
+
});
|
|
182
|
+
tools.add({
|
|
183
|
+
name: 'delete_memory',
|
|
184
|
+
description: 'Deletes a memory, e.g. when the user asks you to forget something.',
|
|
185
|
+
environment: 'memory',
|
|
186
|
+
access: 'write',
|
|
187
|
+
approval: 'allow',
|
|
188
|
+
input: z.object({ id: IdInput }),
|
|
189
|
+
async run({ id }, { api, userId }) {
|
|
190
|
+
const memory = await api('DELETE', 'users/:id/ai/memories/:memory', {}, userId, id);
|
|
191
|
+
return { deleted: { id: memory.id, description: memory.description } };
|
|
192
|
+
},
|
|
193
|
+
});
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import type * as ai from '../common.js';
|
|
2
|
+
/** A message in the OpenAI chat completions format */
|
|
3
|
+
export type ChatMessage = {
|
|
4
|
+
role: 'system' | 'user';
|
|
5
|
+
content: string;
|
|
6
|
+
} | {
|
|
7
|
+
role: 'assistant';
|
|
8
|
+
content: string | null;
|
|
9
|
+
tool_calls?: {
|
|
10
|
+
id: string;
|
|
11
|
+
type: 'function';
|
|
12
|
+
function: {
|
|
13
|
+
name: string;
|
|
14
|
+
arguments: string;
|
|
15
|
+
};
|
|
16
|
+
}[];
|
|
17
|
+
} | {
|
|
18
|
+
role: 'tool';
|
|
19
|
+
tool_call_id: string;
|
|
20
|
+
content: string;
|
|
21
|
+
};
|
|
22
|
+
export interface ChatRequest {
|
|
23
|
+
model: string;
|
|
24
|
+
messages: ChatMessage[];
|
|
25
|
+
/** Sent as `reasoning_effort`, except for `default` */
|
|
26
|
+
reasoning: string;
|
|
27
|
+
maxTokens?: number;
|
|
28
|
+
/** In the OpenAI chat completions format */
|
|
29
|
+
tools?: object[];
|
|
30
|
+
/** A JSON schema the response must follow */
|
|
31
|
+
schema?: object;
|
|
32
|
+
signal?: AbortSignal;
|
|
33
|
+
}
|
|
34
|
+
/** Text the model streamed */
|
|
35
|
+
export interface ChatDelta {
|
|
36
|
+
field: 'content' | 'reasoning';
|
|
37
|
+
text: string;
|
|
38
|
+
}
|
|
39
|
+
export interface ChatResult {
|
|
40
|
+
content: string;
|
|
41
|
+
reasoning: string;
|
|
42
|
+
toolCalls: ai.ToolCall[];
|
|
43
|
+
finishReason: string | null;
|
|
44
|
+
usage: ai.Usage | null;
|
|
45
|
+
}
|
|
46
|
+
/** Yields text as the model streams it, returning the complete result */
|
|
47
|
+
export type ChatStream = AsyncGenerator<ChatDelta, ChatResult, void>;
|
|
48
|
+
/** Runs a chat completion, ignoring the text as it streams */
|
|
49
|
+
export declare function chat(request: ChatRequest): Promise<ChatResult>;
|
|
50
|
+
export declare function streamChat(request: ChatRequest): ChatStream;
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { request as llamaRequest, serverSentEvents } from './llama.js';
|
|
2
|
+
/** Runs a chat completion, ignoring the text as it streams */
|
|
3
|
+
export async function chat(request) {
|
|
4
|
+
const stream = streamChat(request);
|
|
5
|
+
let chunk = await stream.next();
|
|
6
|
+
while (!chunk.done)
|
|
7
|
+
chunk = await stream.next();
|
|
8
|
+
return chunk.value;
|
|
9
|
+
}
|
|
10
|
+
export async function* streamChat(request) {
|
|
11
|
+
const response = await llamaRequest('v1/chat/completions', {
|
|
12
|
+
method: 'POST',
|
|
13
|
+
signal: request.signal,
|
|
14
|
+
body: JSON.stringify({
|
|
15
|
+
model: request.model,
|
|
16
|
+
messages: request.messages,
|
|
17
|
+
reasoning_effort: request.reasoning == 'default' ? undefined : request.reasoning,
|
|
18
|
+
max_tokens: request.maxTokens,
|
|
19
|
+
tools: request.tools?.length ? request.tools : undefined,
|
|
20
|
+
response_format: request.schema && { type: 'json_schema', json_schema: { name: 'response', schema: request.schema } },
|
|
21
|
+
stream: true,
|
|
22
|
+
stream_options: { include_usage: true },
|
|
23
|
+
}),
|
|
24
|
+
});
|
|
25
|
+
const result = { content: '', reasoning: '', toolCalls: [], finishReason: null, usage: null };
|
|
26
|
+
for await (const event of serverSentEvents(response)) {
|
|
27
|
+
if (event.usage) {
|
|
28
|
+
result.usage = {
|
|
29
|
+
promptTokens: event.usage.prompt_tokens,
|
|
30
|
+
completionTokens: event.usage.completion_tokens,
|
|
31
|
+
promptSpeed: event.timings?.prompt_per_second,
|
|
32
|
+
generationSpeed: event.timings?.predicted_per_second,
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
const choice = event.choices?.[0];
|
|
36
|
+
if (!choice)
|
|
37
|
+
continue;
|
|
38
|
+
if (choice.finish_reason)
|
|
39
|
+
result.finishReason = choice.finish_reason;
|
|
40
|
+
const delta = choice.delta ?? {};
|
|
41
|
+
if (delta.reasoning_content) {
|
|
42
|
+
result.reasoning += delta.reasoning_content;
|
|
43
|
+
yield { field: 'reasoning', text: delta.reasoning_content };
|
|
44
|
+
}
|
|
45
|
+
if (delta.content) {
|
|
46
|
+
result.content += delta.content;
|
|
47
|
+
yield { field: 'content', text: delta.content };
|
|
48
|
+
}
|
|
49
|
+
for (const part of delta.tool_calls ?? []) {
|
|
50
|
+
const call = (result.toolCalls[part.index] ??= { id: '', name: '', arguments: '' });
|
|
51
|
+
if (part.id)
|
|
52
|
+
call.id = part.id;
|
|
53
|
+
if (part.function?.name)
|
|
54
|
+
call.name += part.function.name;
|
|
55
|
+
if (part.function?.arguments)
|
|
56
|
+
call.arguments += part.function.arguments;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
return result;
|
|
60
|
+
}
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { EventEmitter } from 'node:events';
|
|
2
|
+
import type * as ai from '../common.js';
|
|
3
|
+
/** A model as llama-server's router lists it */
|
|
4
|
+
export interface RouterModel {
|
|
5
|
+
id: string;
|
|
6
|
+
source?: 'cache' | 'preset' | string;
|
|
7
|
+
status: {
|
|
8
|
+
value: 'unloaded' | 'loading' | 'loaded' | 'sleeping' | 'downloading' | 'incomplete';
|
|
9
|
+
failed?: boolean;
|
|
10
|
+
progress?: Record<string, {
|
|
11
|
+
done: number;
|
|
12
|
+
total: number;
|
|
13
|
+
}>;
|
|
14
|
+
};
|
|
15
|
+
architecture?: {
|
|
16
|
+
input_modalities?: string[];
|
|
17
|
+
output_modalities?: string[];
|
|
18
|
+
};
|
|
19
|
+
/** Tokens; `null` when the model's files can't be read */
|
|
20
|
+
context_length?: number | null;
|
|
21
|
+
memory?: RouterMemory;
|
|
22
|
+
}
|
|
23
|
+
export interface RouterMemoryDevice {
|
|
24
|
+
kind: 'device' | 'host';
|
|
25
|
+
name: string;
|
|
26
|
+
/** Bytes */
|
|
27
|
+
model: number;
|
|
28
|
+
/** Bytes */
|
|
29
|
+
context: number;
|
|
30
|
+
/** Bytes */
|
|
31
|
+
compute: number;
|
|
32
|
+
/** Bytes */
|
|
33
|
+
total?: number;
|
|
34
|
+
/** Bytes */
|
|
35
|
+
free?: number;
|
|
36
|
+
}
|
|
37
|
+
export interface RouterMeasurement {
|
|
38
|
+
n_ctx: number;
|
|
39
|
+
devices: RouterMemoryDevice[];
|
|
40
|
+
}
|
|
41
|
+
export interface RouterMemory {
|
|
42
|
+
status: 'unmeasured' | 'measuring' | 'measured' | 'failed';
|
|
43
|
+
min?: RouterMeasurement;
|
|
44
|
+
max?: RouterMeasurement;
|
|
45
|
+
effective?: RouterMeasurement | null;
|
|
46
|
+
}
|
|
47
|
+
export interface RouterEvent {
|
|
48
|
+
model: string;
|
|
49
|
+
event: string;
|
|
50
|
+
data?: any;
|
|
51
|
+
}
|
|
52
|
+
/** A model to put in the presets file */
|
|
53
|
+
export interface PresetModel {
|
|
54
|
+
id: string;
|
|
55
|
+
source: string;
|
|
56
|
+
options: ai.ModelOptions;
|
|
57
|
+
}
|
|
58
|
+
/** Emits `event` with each {@link RouterEvent}, `ready` once llama-server answers, and `down` when it stops answering */
|
|
59
|
+
export declare const events: EventEmitter<{
|
|
60
|
+
event: [RouterEvent];
|
|
61
|
+
ready: [];
|
|
62
|
+
down: [];
|
|
63
|
+
}>;
|
|
64
|
+
/** Where llama-server keeps its files and how it runs */
|
|
65
|
+
export interface Settings {
|
|
66
|
+
data_dir: string;
|
|
67
|
+
llama: ai.LlamaConfig;
|
|
68
|
+
}
|
|
69
|
+
/** Uses settings other than the plugin's config, e.g. a device's own */
|
|
70
|
+
export declare function useSettings(source: () => Settings): void;
|
|
71
|
+
export declare function isRunning(): boolean;
|
|
72
|
+
/** Writes the presets file llama-server loads models from */
|
|
73
|
+
export declare function writePreset(models: PresetModel[]): void;
|
|
74
|
+
/**
|
|
75
|
+
* Starts llama-server, or keeps using one started by a previous Axium process with the same settings.
|
|
76
|
+
* Models are listed from the presets file, so call {@link writePreset} first.
|
|
77
|
+
*/
|
|
78
|
+
export declare function start(): Promise<void>;
|
|
79
|
+
/**
|
|
80
|
+
* Uses a llama-server that another process supervises, e.g. the device daemon, without supervising it.
|
|
81
|
+
* @returns whether one is running with the current settings
|
|
82
|
+
*/
|
|
83
|
+
export declare function attach(): Promise<boolean>;
|
|
84
|
+
/** Stops llama-server, e.g. when its settings change */
|
|
85
|
+
export declare function stop(): Promise<void>;
|
|
86
|
+
/** Whether llama-server needs restarting to apply the current config */
|
|
87
|
+
export declare function settingsChanged(): boolean;
|
|
88
|
+
/** Subscribes to router events, which are yielded as they happen until `signal` aborts */
|
|
89
|
+
export declare function follow(signal?: AbortSignal): Promise<AsyncGenerator<RouterEvent>>;
|
|
90
|
+
export declare function serverSentEvents(response: Response): AsyncGenerator<any>;
|
|
91
|
+
/** Sends a request to llama-server, returning its response whether or not it succeeded */
|
|
92
|
+
export declare function send(path: string, init?: RequestInit): Promise<Response>;
|
|
93
|
+
/** Sends a request to llama-server, throwing its error message on failure */
|
|
94
|
+
export declare function request(path: string, init?: RequestInit): Promise<Response>;
|
|
95
|
+
export declare function listModels(reload?: boolean): Promise<RouterModel[]>;
|
|
96
|
+
/** Makes llama-server read the presets file again */
|
|
97
|
+
export declare function reload(): Promise<void>;
|
|
98
|
+
export declare const load: (model: string) => Promise<void>;
|
|
99
|
+
export declare const unload: (model: string) => Promise<void>;
|
|
100
|
+
/** Starts downloading `<repository>:<quantization>` into the cache, or resumes a paused download */
|
|
101
|
+
export declare const download: (source: string) => Promise<void>;
|
|
102
|
+
/** Deletes a cached model's files, or a paused download's partial files */
|
|
103
|
+
export declare function remove(source: string): Promise<void>;
|
|
104
|
+
/**
|
|
105
|
+
* What a loaded model's chat template supports.
|
|
106
|
+
* Only available while the model is loaded; this never loads it.
|
|
107
|
+
*/
|
|
108
|
+
export declare function getTemplateCaps(model: string, signal?: AbortSignal): Promise<TemplateCaps | undefined>;
|
|
109
|
+
export interface TemplateCaps {
|
|
110
|
+
/** Lowest to highest */
|
|
111
|
+
reasoning_effort_levels?: string[];
|
|
112
|
+
supports_tool_calls?: boolean;
|
|
113
|
+
}
|
|
114
|
+
/** Which of `candidates` a loaded model's chat template uses when no reasoning effort is given, found by comparing rendered prompts */
|
|
115
|
+
export declare function getTemplateReasoning(model: string, candidates: string[], signal?: AbortSignal): Promise<string | undefined>;
|