agentlang 0.10.3 → 0.10.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -9
- package/out/extension/main.cjs +250 -250
- package/out/extension/main.cjs.map +2 -2
- package/out/language/generated/ast.d.ts +7 -7
- package/out/language/generated/ast.d.ts.map +1 -1
- package/out/language/generated/grammar.js +2 -2
- package/out/language/main.cjs +506 -506
- package/out/language/main.cjs.map +3 -3
- package/out/language/parser.d.ts.map +1 -1
- package/out/language/parser.js +9 -2
- package/out/language/parser.js.map +1 -1
- package/out/language/syntax.d.ts +2 -2
- package/out/language/syntax.d.ts.map +1 -1
- package/out/runtime/api.d.ts.map +1 -1
- package/out/runtime/api.js +0 -14
- package/out/runtime/api.js.map +1 -1
- package/out/runtime/defs.d.ts +0 -1
- package/out/runtime/defs.d.ts.map +1 -1
- package/out/runtime/defs.js +1 -2
- package/out/runtime/defs.js.map +1 -1
- package/out/runtime/embeddings/chunker.d.ts +0 -18
- package/out/runtime/embeddings/chunker.d.ts.map +1 -1
- package/out/runtime/embeddings/chunker.js +15 -47
- package/out/runtime/embeddings/chunker.js.map +1 -1
- package/out/runtime/embeddings/openai.d.ts.map +1 -1
- package/out/runtime/embeddings/openai.js +11 -22
- package/out/runtime/embeddings/openai.js.map +1 -1
- package/out/runtime/embeddings/provider.d.ts +0 -1
- package/out/runtime/embeddings/provider.d.ts.map +1 -1
- package/out/runtime/embeddings/provider.js +1 -20
- package/out/runtime/embeddings/provider.js.map +1 -1
- package/out/runtime/exec-graph.js +5 -5
- package/out/runtime/exec-graph.js.map +1 -1
- package/out/runtime/interpreter.d.ts +4 -4
- package/out/runtime/interpreter.d.ts.map +1 -1
- package/out/runtime/interpreter.js +27 -16
- package/out/runtime/interpreter.js.map +1 -1
- package/out/runtime/loader.d.ts.map +1 -1
- package/out/runtime/loader.js +6 -2
- package/out/runtime/loader.js.map +1 -1
- package/out/runtime/logger.d.ts.map +1 -1
- package/out/runtime/logger.js +1 -8
- package/out/runtime/logger.js.map +1 -1
- package/out/runtime/module.d.ts +0 -6
- package/out/runtime/module.d.ts.map +1 -1
- package/out/runtime/module.js +1 -58
- package/out/runtime/module.js.map +1 -1
- package/out/runtime/modules/ai.d.ts +4 -4
- package/out/runtime/modules/ai.d.ts.map +1 -1
- package/out/runtime/modules/ai.js +70 -166
- package/out/runtime/modules/ai.js.map +1 -1
- package/out/runtime/modules/auth.d.ts.map +1 -1
- package/out/runtime/modules/auth.js +6 -4
- package/out/runtime/modules/auth.js.map +1 -1
- package/out/runtime/modules/core.d.ts.map +1 -1
- package/out/runtime/modules/core.js +3 -0
- package/out/runtime/modules/core.js.map +1 -1
- package/out/runtime/modules/messaging.d.ts +10 -0
- package/out/runtime/modules/messaging.d.ts.map +1 -0
- package/out/runtime/modules/messaging.js +210 -0
- package/out/runtime/modules/messaging.js.map +1 -0
- package/out/runtime/monitor.d.ts +2 -1
- package/out/runtime/monitor.d.ts.map +1 -1
- package/out/runtime/monitor.js +5 -1
- package/out/runtime/monitor.js.map +1 -1
- package/out/runtime/resolvers/sqldb/database.d.ts.map +1 -1
- package/out/runtime/resolvers/sqldb/database.js +126 -128
- package/out/runtime/resolvers/sqldb/database.js.map +1 -1
- package/out/runtime/resolvers/sqldb/impl.d.ts +0 -1
- package/out/runtime/resolvers/sqldb/impl.d.ts.map +1 -1
- package/out/runtime/resolvers/sqldb/impl.js +0 -3
- package/out/runtime/resolvers/sqldb/impl.js.map +1 -1
- package/out/runtime/services/documentFetcher.d.ts.map +1 -1
- package/out/runtime/services/documentFetcher.js +6 -21
- package/out/runtime/services/documentFetcher.js.map +1 -1
- package/out/runtime/state.d.ts +0 -14
- package/out/runtime/state.d.ts.map +1 -1
- package/out/runtime/state.js +0 -28
- package/out/runtime/state.js.map +1 -1
- package/package.json +19 -19
- package/src/language/agentlang.langium +2 -2
- package/src/language/generated/ast.ts +7 -7
- package/src/language/generated/grammar.ts +2 -2
- package/src/language/parser.ts +9 -2
- package/src/language/syntax.ts +2 -2
- package/src/runtime/api.ts +0 -15
- package/src/runtime/defs.ts +1 -2
- package/src/runtime/embeddings/chunker.ts +14 -52
- package/src/runtime/embeddings/openai.ts +9 -27
- package/src/runtime/embeddings/provider.ts +1 -22
- package/src/runtime/exec-graph.ts +4 -4
- package/src/runtime/interpreter.ts +32 -17
- package/src/runtime/loader.ts +10 -2
- package/src/runtime/logger.ts +1 -12
- package/src/runtime/module.ts +1 -64
- package/src/runtime/modules/ai.ts +81 -206
- package/src/runtime/modules/auth.ts +6 -4
- package/src/runtime/modules/core.ts +4 -0
- package/src/runtime/modules/messaging.ts +228 -0
- package/src/runtime/monitor.ts +10 -1
- package/src/runtime/resolvers/sqldb/database.ts +130 -142
- package/src/runtime/resolvers/sqldb/impl.ts +0 -4
- package/src/runtime/services/documentFetcher.ts +6 -21
- package/src/runtime/state.ts +0 -29
- package/out/runtime/document-retriever.d.ts +0 -24
- package/out/runtime/document-retriever.d.ts.map +0 -1
- package/out/runtime/document-retriever.js +0 -258
- package/out/runtime/document-retriever.js.map +0 -1
- package/out/runtime/resolvers/vector/lancedb-store.d.ts +0 -16
- package/out/runtime/resolvers/vector/lancedb-store.d.ts.map +0 -1
- package/out/runtime/resolvers/vector/lancedb-store.js +0 -159
- package/out/runtime/resolvers/vector/lancedb-store.js.map +0 -1
- package/out/runtime/resolvers/vector/types.d.ts +0 -32
- package/out/runtime/resolvers/vector/types.d.ts.map +0 -1
- package/out/runtime/resolvers/vector/types.js +0 -2
- package/out/runtime/resolvers/vector/types.js.map +0 -1
- package/src/runtime/document-retriever.ts +0 -311
- package/src/runtime/resolvers/vector/lancedb-store.ts +0 -187
- package/src/runtime/resolvers/vector/types.ts +0 -39
|
@@ -2,6 +2,7 @@ import { default as ai, normalizeGeneratedCode } from './ai.js';
|
|
|
2
2
|
import { default as auth } from './auth.js';
|
|
3
3
|
import { default as files } from './files.js';
|
|
4
4
|
import { default as mcp } from './mcp.js';
|
|
5
|
+
import messaging, { initMessagingModule } from './messaging.js';
|
|
5
6
|
import {
|
|
6
7
|
DefaultModuleName,
|
|
7
8
|
DefaultModules,
|
|
@@ -188,6 +189,7 @@ export function registerCoreModules() {
|
|
|
188
189
|
{ def: ai, name: makeCoreModuleName('ai') },
|
|
189
190
|
{ def: files, name: makeCoreModuleName('files') },
|
|
190
191
|
{ def: mcp, name: mcpn },
|
|
192
|
+
{ def: messaging, name: makeCoreModuleName('messaging') },
|
|
191
193
|
];
|
|
192
194
|
|
|
193
195
|
coreModuleInfo.forEach(({ def, name }) => {
|
|
@@ -639,6 +641,8 @@ async function internPersistentModules() {
|
|
|
639
641
|
}
|
|
640
642
|
|
|
641
643
|
export function initCoreModuleManager() {
|
|
644
|
+
initMessagingModule();
|
|
645
|
+
|
|
642
646
|
const ModuleResolverName = 'agentlang/moduleResolver';
|
|
643
647
|
const ModuleResolver = new GenericResolver(ModuleResolverName, {
|
|
644
648
|
create: createModule,
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
import { makeCoreModuleName } from '../util.js';
|
|
2
|
+
import { Instance } from '../module.js';
|
|
3
|
+
import { GenericResolver, Resolver } from '../resolvers/interface.js';
|
|
4
|
+
import { registerResolver, setResolver } from '../resolvers/registry.js';
|
|
5
|
+
import { logger } from '../logger.js';
|
|
6
|
+
|
|
7
|
+
export const CoreMessagingModuleName = makeCoreModuleName('messaging');
|
|
8
|
+
|
|
9
|
+
function getAgentManagerUrl(): string {
|
|
10
|
+
return (
|
|
11
|
+
(globalThis as any).__agentmanager_url ||
|
|
12
|
+
process.env.AGENTMANAGER_URL ||
|
|
13
|
+
'http://localhost:3001'
|
|
14
|
+
);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export async function pushNotification(_: Resolver, inst: Instance) {
|
|
18
|
+
const inbox = inst.lookup('inbox');
|
|
19
|
+
const subject = inst.lookup('subject');
|
|
20
|
+
const body = inst.lookup('body');
|
|
21
|
+
const fromKind = inst.lookup('fromKind') || 'employee';
|
|
22
|
+
const fromId = inst.lookup('fromId');
|
|
23
|
+
|
|
24
|
+
const url = `${getAgentManagerUrl()}/api/inboxes/${encodeURIComponent(inbox)}/messages`;
|
|
25
|
+
const payload = {
|
|
26
|
+
from: { kind: fromKind, id: fromId },
|
|
27
|
+
subject,
|
|
28
|
+
body,
|
|
29
|
+
kind: 'notification',
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
try {
|
|
33
|
+
const response = await fetch(url, {
|
|
34
|
+
method: 'POST',
|
|
35
|
+
headers: { 'Content-Type': 'application/json' },
|
|
36
|
+
body: JSON.stringify(payload),
|
|
37
|
+
});
|
|
38
|
+
if (!response.ok) {
|
|
39
|
+
const text = await response.text();
|
|
40
|
+
logger.error(
|
|
41
|
+
`Failed to push notification to inbox ${inbox}: HTTP ${response.status} — ${text}`
|
|
42
|
+
);
|
|
43
|
+
}
|
|
44
|
+
return inst;
|
|
45
|
+
} catch (err: any) {
|
|
46
|
+
logger.error(`Failed to push notification to inbox ${inbox}: ${err.message}`);
|
|
47
|
+
return inst;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export async function pushAction(_: Resolver, inst: Instance) {
|
|
52
|
+
const inbox = inst.lookup('inbox');
|
|
53
|
+
const subject = inst.lookup('subject');
|
|
54
|
+
const body = inst.lookup('body');
|
|
55
|
+
const fromKind = inst.lookup('fromKind') || 'employee';
|
|
56
|
+
const fromId = inst.lookup('fromId');
|
|
57
|
+
const responses: string[] = inst.lookup('responses') || [];
|
|
58
|
+
const continuationEvents: string[] = inst.lookup('continuationEvents') || [];
|
|
59
|
+
const inputPrompt: string | undefined = inst.lookup('inputPrompt');
|
|
60
|
+
|
|
61
|
+
// Zip responses and continuationEvents into a continuations map
|
|
62
|
+
const continuations: Record<string, { event: string }> = {};
|
|
63
|
+
for (let i = 0; i < responses.length; i++) {
|
|
64
|
+
const eventName = i < continuationEvents.length ? continuationEvents[i] : '';
|
|
65
|
+
if (eventName) {
|
|
66
|
+
continuations[responses[i]] = { event: eventName };
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const url = `${getAgentManagerUrl()}/api/inboxes/${encodeURIComponent(inbox)}/messages`;
|
|
71
|
+
const payload: any = {
|
|
72
|
+
from: { kind: fromKind, id: fromId },
|
|
73
|
+
subject,
|
|
74
|
+
body,
|
|
75
|
+
kind: 'action',
|
|
76
|
+
action: {
|
|
77
|
+
responses,
|
|
78
|
+
continuations,
|
|
79
|
+
...(inputPrompt ? { inputPrompt } : {}),
|
|
80
|
+
},
|
|
81
|
+
};
|
|
82
|
+
|
|
83
|
+
try {
|
|
84
|
+
const response = await fetch(url, {
|
|
85
|
+
method: 'POST',
|
|
86
|
+
headers: { 'Content-Type': 'application/json' },
|
|
87
|
+
body: JSON.stringify(payload),
|
|
88
|
+
});
|
|
89
|
+
if (!response.ok) {
|
|
90
|
+
const text = await response.text();
|
|
91
|
+
logger.error(`Failed to push action to inbox ${inbox}: HTTP ${response.status} — ${text}`);
|
|
92
|
+
}
|
|
93
|
+
return inst;
|
|
94
|
+
} catch (err: any) {
|
|
95
|
+
logger.error(`Failed to push action to inbox ${inbox}: ${err.message}`);
|
|
96
|
+
return inst;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export async function queryInbox(_: Resolver, inst: Instance, _queryAll: boolean) {
|
|
101
|
+
const inbox = inst.lookup('inbox');
|
|
102
|
+
const status: string | undefined = inst.lookup('status');
|
|
103
|
+
const kind: string | undefined = inst.lookup('kind');
|
|
104
|
+
const from: string | undefined = inst.lookup('from');
|
|
105
|
+
|
|
106
|
+
const params = new URLSearchParams();
|
|
107
|
+
if (status) params.set('status', status);
|
|
108
|
+
if (kind) params.set('kind', kind);
|
|
109
|
+
if (from) params.set('from', from);
|
|
110
|
+
|
|
111
|
+
const qs = params.toString();
|
|
112
|
+
const url = `${getAgentManagerUrl()}/api/inboxes/${encodeURIComponent(inbox)}/messages${qs ? '?' + qs : ''}`;
|
|
113
|
+
|
|
114
|
+
try {
|
|
115
|
+
const response = await fetch(url);
|
|
116
|
+
if (!response.ok) {
|
|
117
|
+
const text = await response.text();
|
|
118
|
+
logger.error(`Failed to query inbox ${inbox}: HTTP ${response.status} — ${text}`);
|
|
119
|
+
return [];
|
|
120
|
+
}
|
|
121
|
+
return await response.json();
|
|
122
|
+
} catch (err: any) {
|
|
123
|
+
logger.error(`Failed to query inbox ${inbox}: ${err.message}`);
|
|
124
|
+
return [];
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export default `module ${CoreMessagingModuleName}
|
|
129
|
+
|
|
130
|
+
record Notification {
|
|
131
|
+
inbox String,
|
|
132
|
+
subject String,
|
|
133
|
+
body String,
|
|
134
|
+
fromKind String @default("employee"),
|
|
135
|
+
fromId String
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
record Action {
|
|
139
|
+
inbox String,
|
|
140
|
+
subject String,
|
|
141
|
+
body String,
|
|
142
|
+
fromKind String @default("employee"),
|
|
143
|
+
fromId String,
|
|
144
|
+
responses String[],
|
|
145
|
+
continuationEvents String[] @optional,
|
|
146
|
+
inputPrompt String @optional
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
record InboxQuery {
|
|
150
|
+
inbox String,
|
|
151
|
+
status String @optional,
|
|
152
|
+
kind String @optional,
|
|
153
|
+
from String @optional
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
@public event SendNotification extends Notification {}
|
|
157
|
+
workflow SendNotification {
|
|
158
|
+
{Notification {inbox SendNotification.inbox,
|
|
159
|
+
subject SendNotification.subject,
|
|
160
|
+
body SendNotification.body,
|
|
161
|
+
fromKind SendNotification.fromKind,
|
|
162
|
+
fromId SendNotification.fromId}}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
@public event SendAction extends Action {}
|
|
166
|
+
workflow SendAction {
|
|
167
|
+
{Action {inbox SendAction.inbox,
|
|
168
|
+
subject SendAction.subject,
|
|
169
|
+
body SendAction.body,
|
|
170
|
+
fromKind SendAction.fromKind,
|
|
171
|
+
fromId SendAction.fromId,
|
|
172
|
+
responses SendAction.responses,
|
|
173
|
+
continuationEvents SendAction.continuationEvents,
|
|
174
|
+
inputPrompt SendAction.inputPrompt}}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
@public event QueryInbox extends InboxQuery {}
|
|
178
|
+
workflow QueryInbox {
|
|
179
|
+
{InboxQuery {inbox? QueryInbox.inbox,
|
|
180
|
+
status? QueryInbox.status,
|
|
181
|
+
kind? QueryInbox.kind,
|
|
182
|
+
from? QueryInbox.from}}
|
|
183
|
+
}
|
|
184
|
+
`;
|
|
185
|
+
|
|
186
|
+
export function initMessagingModule() {
|
|
187
|
+
const resolverName = 'agentlang.messaging/messagingResolver';
|
|
188
|
+
const resolver = new GenericResolver(resolverName, {
|
|
189
|
+
create: pushNotification,
|
|
190
|
+
upsert: undefined,
|
|
191
|
+
update: undefined,
|
|
192
|
+
query: queryInbox,
|
|
193
|
+
delete: undefined,
|
|
194
|
+
startTransaction: undefined,
|
|
195
|
+
commitTransaction: undefined,
|
|
196
|
+
rollbackTransaction: undefined,
|
|
197
|
+
});
|
|
198
|
+
registerResolver(resolverName, () => resolver);
|
|
199
|
+
setResolver('agentlang.messaging/Notification', resolverName);
|
|
200
|
+
|
|
201
|
+
const actionResolverName = 'agentlang.messaging/actionResolver';
|
|
202
|
+
const actionResolver = new GenericResolver(actionResolverName, {
|
|
203
|
+
create: pushAction,
|
|
204
|
+
upsert: undefined,
|
|
205
|
+
update: undefined,
|
|
206
|
+
query: undefined,
|
|
207
|
+
delete: undefined,
|
|
208
|
+
startTransaction: undefined,
|
|
209
|
+
commitTransaction: undefined,
|
|
210
|
+
rollbackTransaction: undefined,
|
|
211
|
+
});
|
|
212
|
+
registerResolver(actionResolverName, () => actionResolver);
|
|
213
|
+
setResolver('agentlang.messaging/Action', actionResolverName);
|
|
214
|
+
|
|
215
|
+
const queryResolverName = 'agentlang.messaging/queryResolver';
|
|
216
|
+
const queryResolver = new GenericResolver(queryResolverName, {
|
|
217
|
+
create: undefined,
|
|
218
|
+
upsert: undefined,
|
|
219
|
+
update: undefined,
|
|
220
|
+
query: queryInbox,
|
|
221
|
+
delete: undefined,
|
|
222
|
+
startTransaction: undefined,
|
|
223
|
+
commitTransaction: undefined,
|
|
224
|
+
rollbackTransaction: undefined,
|
|
225
|
+
});
|
|
226
|
+
registerResolver(queryResolverName, () => queryResolver);
|
|
227
|
+
setResolver('agentlang.messaging/InboxQuery', queryResolverName);
|
|
228
|
+
}
|
package/src/runtime/monitor.ts
CHANGED
|
@@ -179,6 +179,7 @@ export class Monitor {
|
|
|
179
179
|
private id: string;
|
|
180
180
|
private eventInstance: Instance | undefined;
|
|
181
181
|
private user: string | undefined;
|
|
182
|
+
private role: string | undefined;
|
|
182
183
|
private entries: (MonitorEntry | Monitor)[] = new Array<MonitorEntry | Monitor>();
|
|
183
184
|
private parent: Monitor | undefined = undefined;
|
|
184
185
|
private lastEntry: MonitorEntry | undefined = undefined;
|
|
@@ -189,10 +190,15 @@ export class Monitor {
|
|
|
189
190
|
|
|
190
191
|
private static MAX_REGISTRY_SIZE = 25;
|
|
191
192
|
|
|
192
|
-
constructor(
|
|
193
|
+
constructor(
|
|
194
|
+
eventInstance?: Instance | undefined,
|
|
195
|
+
user?: string | undefined,
|
|
196
|
+
role?: string | undefined
|
|
197
|
+
) {
|
|
193
198
|
this.eventInstance = eventInstance;
|
|
194
199
|
this.id = eventInstance ? eventInstance.getId() : crypto.randomUUID();
|
|
195
200
|
this.user = user;
|
|
201
|
+
this.role = role;
|
|
196
202
|
this.timestamp = Date.now();
|
|
197
203
|
while (monitorRegistry.length >= Monitor.MAX_REGISTRY_SIZE) {
|
|
198
204
|
monitorRegistry.shift();
|
|
@@ -405,6 +411,9 @@ export class Monitor {
|
|
|
405
411
|
if (this.user) {
|
|
406
412
|
r.user = this.user;
|
|
407
413
|
}
|
|
414
|
+
if (this.role) {
|
|
415
|
+
r.role = this.role;
|
|
416
|
+
}
|
|
408
417
|
r.timestamp = this.timestamp;
|
|
409
418
|
if (this.flowResult !== undefined) {
|
|
410
419
|
let fr = this.flowResult;
|
|
@@ -46,8 +46,6 @@ import { saveMigration } from '../../modules/core.js';
|
|
|
46
46
|
import { getAppSpec } from '../../loader.js';
|
|
47
47
|
import { WhereClause } from '../interface.js';
|
|
48
48
|
import { AppConfig } from '../../state.js';
|
|
49
|
-
import { createLanceDBStore } from '../vector/lancedb-store.js';
|
|
50
|
-
import type { VectorStore } from '../vector/types.js';
|
|
51
49
|
|
|
52
50
|
export let defaultDataSource: DataSource | undefined;
|
|
53
51
|
|
|
@@ -60,8 +58,57 @@ function isBrowser(): boolean {
|
|
|
60
58
|
);
|
|
61
59
|
}
|
|
62
60
|
|
|
63
|
-
//
|
|
64
|
-
|
|
61
|
+
// SQLite WASM with built-in sqlite-vec for browsers
|
|
62
|
+
// Loaded from CDN - this has sqlite-vec statically compiled in
|
|
63
|
+
let sqliteVecWasmModule: any = null;
|
|
64
|
+
|
|
65
|
+
async function loadSqliteVecWasm(): Promise<any> {
|
|
66
|
+
if (sqliteVecWasmModule) {
|
|
67
|
+
return sqliteVecWasmModule;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
try {
|
|
71
|
+
// Use dynamic import with string to prevent bundlers from analyzing
|
|
72
|
+
const cdnUrl = 'https://cdn.jsdelivr.net/npm/sqlite-vec-wasm-demo@latest/sqlite3.mjs';
|
|
73
|
+
const module = await import(/* @vite-ignore */ cdnUrl);
|
|
74
|
+
sqliteVecWasmModule = await module.default();
|
|
75
|
+
return sqliteVecWasmModule;
|
|
76
|
+
} catch (err) {
|
|
77
|
+
logger.warn('Failed to load sqlite-vec WASM:', err);
|
|
78
|
+
return null;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// Helper to load sqlite-vec based on environment
|
|
83
|
+
async function loadSqliteVec(): Promise<any> {
|
|
84
|
+
// In browser, use WASM version with built-in sqlite-vec
|
|
85
|
+
// In Node.js, use the npm package
|
|
86
|
+
if (isBrowser()) {
|
|
87
|
+
const wasmModule = await loadSqliteVecWasm();
|
|
88
|
+
if (wasmModule) {
|
|
89
|
+
// WASM version has sqlite-vec built-in, no need to call load()
|
|
90
|
+
// Return a compatible interface
|
|
91
|
+
return {
|
|
92
|
+
load: () => {
|
|
93
|
+
// No-op: sqlite-vec is already loaded in WASM version
|
|
94
|
+
logger.info('sqlite-vec WASM loaded (built-in)');
|
|
95
|
+
},
|
|
96
|
+
// Expose the WASM module for direct use if needed
|
|
97
|
+
_wasmModule: wasmModule,
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
return null;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// Node.js: use npm package
|
|
104
|
+
try {
|
|
105
|
+
// Use variable to prevent bundlers from statically analyzing this import
|
|
106
|
+
const moduleName = 'sqlite-vec';
|
|
107
|
+
return await import(/* @vite-ignore */ moduleName);
|
|
108
|
+
} catch {
|
|
109
|
+
return null;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
65
112
|
|
|
66
113
|
export class DbContext {
|
|
67
114
|
txnId: string | undefined;
|
|
@@ -259,27 +306,17 @@ function makeSqliteDataSource(
|
|
|
259
306
|
ds.initialize = async () => {
|
|
260
307
|
const res = await originalInit();
|
|
261
308
|
try {
|
|
309
|
+
const sqliteVec = await loadSqliteVec();
|
|
262
310
|
const driver = ds.driver as any;
|
|
263
311
|
const db = driver.databaseConnection || driver.nativeDatabase;
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
const syncMode = process.env.SQLITE_SYNC_MODE || 'NORMAL';
|
|
269
|
-
const busyTimeout = process.env.SQLITE_BUSY_TIMEOUT || '5000';
|
|
270
|
-
const cacheSize = process.env.SQLITE_CACHE_SIZE || '-20000';
|
|
271
|
-
|
|
272
|
-
db.pragma(`synchronous = ${syncMode}`);
|
|
273
|
-
db.pragma(`busy_timeout = ${busyTimeout}`);
|
|
274
|
-
db.pragma(`cache_size = ${cacheSize}`);
|
|
275
|
-
db.pragma('temp_store = MEMORY');
|
|
276
|
-
|
|
277
|
-
logger.info(
|
|
278
|
-
`SQLite pragmas enabled: WAL mode, synchronous=${syncMode}, busy_timeout=${busyTimeout}, cache_size=${cacheSize}, temp_store=MEMORY`
|
|
279
|
-
);
|
|
312
|
+
if (db && sqliteVec?.load) {
|
|
313
|
+
sqliteVec.load(db);
|
|
314
|
+
logger.info('sqlite-vec extension loaded successfully');
|
|
280
315
|
}
|
|
281
316
|
} catch (err: any) {
|
|
282
|
-
logger.warn(
|
|
317
|
+
logger.warn(
|
|
318
|
+
`Failed to load sqlite-vec extension: ${err.message}. Vector operations may not be available.`
|
|
319
|
+
);
|
|
283
320
|
}
|
|
284
321
|
return res;
|
|
285
322
|
};
|
|
@@ -375,20 +412,6 @@ function forceGetDbType(config: DatabaseConfig | undefined): string {
|
|
|
375
412
|
|
|
376
413
|
let DbType: string | undefined;
|
|
377
414
|
|
|
378
|
-
function getVectorStoreType(): string {
|
|
379
|
-
// Check explicit vectorStore config first
|
|
380
|
-
const vectorStoreConfig = AppConfig?.vectorStore;
|
|
381
|
-
if (vectorStoreConfig?.type) {
|
|
382
|
-
if (vectorStoreConfig.type === 'pgvector') return 'postgres';
|
|
383
|
-
if (vectorStoreConfig.type === 'lancedb') return 'lancedb';
|
|
384
|
-
}
|
|
385
|
-
|
|
386
|
-
// Fallback to main store type
|
|
387
|
-
const dbType = getDbType(AppConfig?.store);
|
|
388
|
-
if (dbType === 'postgres') return 'postgres';
|
|
389
|
-
return 'lancedb';
|
|
390
|
-
}
|
|
391
|
-
|
|
392
415
|
function getDbType(config: DatabaseConfig | undefined): string {
|
|
393
416
|
if (DbType === undefined) DbType = forceGetDbType(config);
|
|
394
417
|
return DbType;
|
|
@@ -422,9 +445,12 @@ export function isUsingSqljs(): boolean {
|
|
|
422
445
|
}
|
|
423
446
|
|
|
424
447
|
export async function isVectorStoreSupported(): Promise<boolean> {
|
|
425
|
-
const
|
|
426
|
-
if (
|
|
427
|
-
if (
|
|
448
|
+
const dbType = getDbType(AppConfig?.store);
|
|
449
|
+
if (dbType === 'postgres') return true;
|
|
450
|
+
if (dbType === 'sqlite') {
|
|
451
|
+
const sqliteVecModule = await loadSqliteVec();
|
|
452
|
+
return !!sqliteVecModule;
|
|
453
|
+
}
|
|
428
454
|
return false;
|
|
429
455
|
}
|
|
430
456
|
|
|
@@ -483,14 +509,9 @@ async function insertRowsHelper(
|
|
|
483
509
|
ctx: DbContext,
|
|
484
510
|
doUpsert: boolean
|
|
485
511
|
): Promise<void> {
|
|
486
|
-
const
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
if (doUpsert) {
|
|
490
|
-
await repo.save(rows);
|
|
491
|
-
} else {
|
|
492
|
-
await repo.insert(rows);
|
|
493
|
-
}
|
|
512
|
+
const repo = getDatasourceForTransaction(ctx.txnId).getRepository(tableName);
|
|
513
|
+
if (doUpsert) await repo.save(rows);
|
|
514
|
+
else await repo.insert(rows);
|
|
494
515
|
}
|
|
495
516
|
|
|
496
517
|
export async function addRowForFullTextSearch(
|
|
@@ -499,45 +520,28 @@ export async function addRowForFullTextSearch(
|
|
|
499
520
|
vect: number[],
|
|
500
521
|
ctx: DbContext
|
|
501
522
|
) {
|
|
502
|
-
if (!(await isVectorStoreSupported()))
|
|
503
|
-
logger.warn(`[VECTOR] Vector store not supported, skipping save for ${id}`);
|
|
504
|
-
return;
|
|
505
|
-
}
|
|
523
|
+
if (!(await isVectorStoreSupported())) return;
|
|
506
524
|
try {
|
|
507
525
|
const vecTableName = tableName + VectorSuffix;
|
|
508
|
-
|
|
509
|
-
`[VECTOR] Saving embedding to ${vecTableName} for ${id} (${vect.length} dimensions)`
|
|
510
|
-
);
|
|
511
|
-
const dbType = getVectorStoreType();
|
|
526
|
+
const qb = getDatasourceForTransaction(ctx.txnId).createQueryBuilder();
|
|
512
527
|
const tenantId = await ctx.getTenantId();
|
|
513
|
-
|
|
514
|
-
if (dbType === '
|
|
515
|
-
let store = lanceDBStores.get(tableName);
|
|
516
|
-
if (!store) {
|
|
517
|
-
store = createLanceDBStore({
|
|
518
|
-
moduleName: tableName,
|
|
519
|
-
vectorDimension: vect.length,
|
|
520
|
-
});
|
|
521
|
-
await store.init();
|
|
522
|
-
lanceDBStores.set(tableName, store);
|
|
523
|
-
}
|
|
524
|
-
await store.addEmbedding({
|
|
525
|
-
id,
|
|
526
|
-
embedding: Array.from(vect),
|
|
527
|
-
tenantId,
|
|
528
|
-
});
|
|
529
|
-
} else if (dbType === 'postgres') {
|
|
530
|
-
const qb = getDatasourceForTransaction(ctx.txnId).createQueryBuilder();
|
|
528
|
+
const dbType = getDbType(AppConfig?.store);
|
|
529
|
+
if (dbType === 'postgres') {
|
|
531
530
|
const { default: pgvector } = await import('pgvector');
|
|
532
531
|
await qb
|
|
533
532
|
.insert()
|
|
534
533
|
.into(vecTableName)
|
|
535
|
-
.values([{ id: id, embedding: pgvector.toSql(vect),
|
|
534
|
+
.values([{ id: id, embedding: pgvector.toSql(vect), __tenant__: tenantId }])
|
|
535
|
+
.execute();
|
|
536
|
+
} else {
|
|
537
|
+
await qb
|
|
538
|
+
.insert()
|
|
539
|
+
.into(vecTableName)
|
|
540
|
+
.values([{ id: id, embedding: new Float32Array(vect) }])
|
|
536
541
|
.execute();
|
|
537
542
|
}
|
|
538
|
-
logger.info(`[VECTOR] Successfully saved embedding to ${vecTableName} for ${id}`);
|
|
539
543
|
} catch (err: any) {
|
|
540
|
-
logger.error(`
|
|
544
|
+
logger.error(`Failed to add row to vector store - ${err}`);
|
|
541
545
|
}
|
|
542
546
|
}
|
|
543
547
|
|
|
@@ -546,21 +550,11 @@ export async function initVectorStore(tableNames: string[], ctx: DbContext) {
|
|
|
546
550
|
logger.info(`Vector store not supported for ${getDbType(AppConfig?.store)}, skipping init...`);
|
|
547
551
|
return;
|
|
548
552
|
}
|
|
549
|
-
const dbType =
|
|
553
|
+
const dbType = getDbType(AppConfig?.store);
|
|
550
554
|
let notInited = true;
|
|
551
555
|
for (const vecTableName of tableNames) {
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
const store = createLanceDBStore({
|
|
555
|
-
moduleName: vecTableName,
|
|
556
|
-
vectorDimension: DefaultVectorDimension,
|
|
557
|
-
});
|
|
558
|
-
await store.init();
|
|
559
|
-
lanceDBStores.set(vecTableName, store);
|
|
560
|
-
logger.info(`[VECTOR] Initialized LanceDB store for ${vecTableName}`);
|
|
561
|
-
}
|
|
562
|
-
} else if (dbType === 'postgres') {
|
|
563
|
-
const vecRepo = getDatasourceForTransaction(ctx.txnId).getRepository(vecTableName);
|
|
556
|
+
const vecRepo = getDatasourceForTransaction(ctx.txnId).getRepository(vecTableName);
|
|
557
|
+
if (dbType === 'postgres') {
|
|
564
558
|
if (notInited) {
|
|
565
559
|
let failure = false;
|
|
566
560
|
try {
|
|
@@ -574,11 +568,18 @@ export async function initVectorStore(tableNames: string[], ctx: DbContext) {
|
|
|
574
568
|
}
|
|
575
569
|
await vecRepo.query(
|
|
576
570
|
`CREATE TABLE IF NOT EXISTS ${vecTableName} (
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
571
|
+
id varchar PRIMARY KEY,
|
|
572
|
+
embedding vector(${DefaultVectorDimension}),
|
|
573
|
+
${TenantAttributeName} varchar,
|
|
574
|
+
__is_deleted__ boolean default false
|
|
575
|
+
)`
|
|
576
|
+
);
|
|
577
|
+
} else {
|
|
578
|
+
// sqlite-vec - vec0 doesn't support type declarations for metadata columns
|
|
579
|
+
await vecRepo.query(
|
|
580
|
+
`CREATE VIRTUAL TABLE IF NOT EXISTS ${vecTableName} USING vec0(
|
|
581
|
+
id TEXT PRIMARY KEY,
|
|
582
|
+
embedding FLOAT[${DefaultVectorDimension}])`
|
|
582
583
|
);
|
|
583
584
|
}
|
|
584
585
|
}
|
|
@@ -595,21 +596,6 @@ export async function vectorStoreSearch(
|
|
|
595
596
|
return [];
|
|
596
597
|
}
|
|
597
598
|
try {
|
|
598
|
-
const dbType = getVectorStoreType();
|
|
599
|
-
const tenantId = await ctx.getTenantId();
|
|
600
|
-
|
|
601
|
-
if (dbType === 'lancedb') {
|
|
602
|
-
const store = lanceDBStores.get(tableName);
|
|
603
|
-
if (!store) {
|
|
604
|
-
logger.warn(`[VECTOR] LanceDB store not found for ${tableName}`);
|
|
605
|
-
return [];
|
|
606
|
-
}
|
|
607
|
-
// Extract agentId from resourceFqName for agent-level filtering
|
|
608
|
-
const agentId = ctx.resourceFqName || undefined;
|
|
609
|
-
const results = await store.search(searchVec, tenantId, agentId, limit);
|
|
610
|
-
return results.map(r => ({ id: r.id }));
|
|
611
|
-
}
|
|
612
|
-
|
|
613
599
|
let hasGlobalPerms = ctx.isPermitted();
|
|
614
600
|
if (!hasGlobalPerms) {
|
|
615
601
|
const userId = ctx.getUserId();
|
|
@@ -619,6 +605,8 @@ export async function vectorStoreSearch(
|
|
|
619
605
|
}
|
|
620
606
|
const vecTableName = tableName + VectorSuffix;
|
|
621
607
|
const qb = getDatasourceForTransaction(ctx.txnId).getRepository(tableName).manager;
|
|
608
|
+
const dbType = getDbType(AppConfig?.store);
|
|
609
|
+
const tenantId = await ctx.getTenantId();
|
|
622
610
|
let ownersJoinCond: string = '';
|
|
623
611
|
if (!hasGlobalPerms) {
|
|
624
612
|
const ot = ownersTable(tableName);
|
|
@@ -631,6 +619,16 @@ export async function vectorStoreSearch(
|
|
|
631
619
|
const sql = `select ${vecTableName}.id from ${vecTableName} ${ownersJoinCond} order by embedding <-> $1 LIMIT ${limit}`;
|
|
632
620
|
const args = pgvector.toSql(searchVec);
|
|
633
621
|
return await qb.query(sql, [args]);
|
|
622
|
+
} else {
|
|
623
|
+
// sqlite-vec - join with main table to filter by tenant
|
|
624
|
+
const alias = tableName.toLowerCase();
|
|
625
|
+
const sql = `SELECT ${vecTableName}.id FROM ${vecTableName}
|
|
626
|
+
INNER JOIN ${tableName} ${alias} ON ${alias}.${PathAttributeName} = ${vecTableName}.id
|
|
627
|
+
${ownersJoinCond}
|
|
628
|
+
WHERE ${alias}.${TenantAttributeName} = '${tenantId}' AND ${alias}.${DeletedFlagAttributeName} = false AND ${vecTableName}.embedding MATCH $1
|
|
629
|
+
LIMIT ${limit}`;
|
|
630
|
+
const args = new Float32Array(searchVec);
|
|
631
|
+
return await qb.query(sql, [args]);
|
|
634
632
|
}
|
|
635
633
|
} catch (err: any) {
|
|
636
634
|
logger.error(`Vector store search failed - ${err}`);
|
|
@@ -645,19 +643,9 @@ export async function vectorStoreSearchEntryExists(
|
|
|
645
643
|
): Promise<boolean> {
|
|
646
644
|
if (!(await isVectorStoreSupported())) return false;
|
|
647
645
|
try {
|
|
648
|
-
const dbType = getVectorStoreType();
|
|
649
|
-
|
|
650
|
-
if (dbType === 'lancedb') {
|
|
651
|
-
const store = lanceDBStores.get(tableName);
|
|
652
|
-
if (!store) {
|
|
653
|
-
logger.warn(`[VECTOR] LanceDB store not found for ${tableName}`);
|
|
654
|
-
return false;
|
|
655
|
-
}
|
|
656
|
-
return await store.exists(id);
|
|
657
|
-
}
|
|
658
|
-
|
|
659
646
|
const qb = getDatasourceForTransaction(ctx.txnId).getRepository(tableName).manager;
|
|
660
647
|
const vecTableName = tableName + VectorSuffix;
|
|
648
|
+
const dbType = getDbType(AppConfig?.store);
|
|
661
649
|
const tenantId = await ctx.getTenantId();
|
|
662
650
|
|
|
663
651
|
if (dbType === 'postgres') {
|
|
@@ -666,6 +654,16 @@ export async function vectorStoreSearchEntryExists(
|
|
|
666
654
|
[id]
|
|
667
655
|
);
|
|
668
656
|
return result !== null && result.length > 0;
|
|
657
|
+
} else {
|
|
658
|
+
// sqlite-vec - join with main table to verify tenant
|
|
659
|
+
const alias = tableName.toLowerCase();
|
|
660
|
+
const result: any[] = await qb.query(
|
|
661
|
+
`SELECT ${vecTableName}.id FROM ${vecTableName}
|
|
662
|
+
INNER JOIN ${tableName} ${alias} ON ${alias}.${PathAttributeName} = ${vecTableName}.id
|
|
663
|
+
WHERE ${vecTableName}.id = $1 AND ${alias}.${TenantAttributeName} = '${tenantId}'`,
|
|
664
|
+
[id]
|
|
665
|
+
);
|
|
666
|
+
return result !== null && result.length > 0;
|
|
669
667
|
}
|
|
670
668
|
} catch (err: any) {
|
|
671
669
|
logger.error(`Vector store search failed - ${err}`);
|
|
@@ -676,20 +674,9 @@ export async function vectorStoreSearchEntryExists(
|
|
|
676
674
|
export async function deleteFullTextSearchEntry(tableName: string, id: string, ctx: DbContext) {
|
|
677
675
|
if (!(await isVectorStoreSupported())) return;
|
|
678
676
|
try {
|
|
679
|
-
const dbType = getVectorStoreType();
|
|
680
|
-
|
|
681
|
-
if (dbType === 'lancedb') {
|
|
682
|
-
const store = lanceDBStores.get(tableName);
|
|
683
|
-
if (!store) {
|
|
684
|
-
logger.warn(`[VECTOR] LanceDB store not found for ${tableName}`);
|
|
685
|
-
return;
|
|
686
|
-
}
|
|
687
|
-
await store.delete(id);
|
|
688
|
-
return;
|
|
689
|
-
}
|
|
690
|
-
|
|
691
677
|
const qb = getDatasourceForTransaction(ctx.txnId).getRepository(tableName).manager;
|
|
692
678
|
const vecTableName = tableName + VectorSuffix;
|
|
679
|
+
const dbType = getDbType(AppConfig?.store);
|
|
693
680
|
const tenantId = await ctx.getTenantId();
|
|
694
681
|
|
|
695
682
|
if (dbType === 'postgres') {
|
|
@@ -697,6 +684,9 @@ export async function deleteFullTextSearchEntry(tableName: string, id: string, c
|
|
|
697
684
|
`delete from ${vecTableName} where id = $1 and ${TenantAttributeName} = '${tenantId}'`,
|
|
698
685
|
[id]
|
|
699
686
|
);
|
|
687
|
+
} else {
|
|
688
|
+
// sqlite-vec - delete just by id (ownership verified by caller)
|
|
689
|
+
await qb.query(`delete from ${vecTableName} where id = $1`, [id]);
|
|
700
690
|
}
|
|
701
691
|
} catch (err: any) {
|
|
702
692
|
logger.error(`Vector store delete failed - ${err}`);
|
|
@@ -872,7 +862,7 @@ async function createLimitedOwnership(
|
|
|
872
862
|
r: perms.has(RbacPermissionFlag.READ),
|
|
873
863
|
d: perms.has(RbacPermissionFlag.DELETE),
|
|
874
864
|
u: perms.has(RbacPermissionFlag.UPDATE),
|
|
875
|
-
|
|
865
|
+
__tenant__: tenantId,
|
|
876
866
|
});
|
|
877
867
|
});
|
|
878
868
|
const tname = ownersTable(tableName);
|
|
@@ -1374,16 +1364,14 @@ async function endTransaction(txnId: string, commit: boolean): Promise<void> {
|
|
|
1374
1364
|
const qr: QueryRunner | undefined = transactionsDb.get(txnId);
|
|
1375
1365
|
if (qr && qr.isTransactionActive) {
|
|
1376
1366
|
try {
|
|
1377
|
-
if (commit)
|
|
1378
|
-
await qr.commitTransaction()
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
);
|
|
1386
|
-
throw err;
|
|
1367
|
+
if (commit)
|
|
1368
|
+
await qr.commitTransaction().catch((reason: any) => {
|
|
1369
|
+
logger.error(`failed to commit transaction ${txnId} - ${reason}`);
|
|
1370
|
+
});
|
|
1371
|
+
else
|
|
1372
|
+
await qr.rollbackTransaction().catch((reason: any) => {
|
|
1373
|
+
logger.error(`failed to rollback transaction ${txnId} - ${reason}`);
|
|
1374
|
+
});
|
|
1387
1375
|
} finally {
|
|
1388
1376
|
await qr.release();
|
|
1389
1377
|
transactionsDb.delete(txnId);
|