@adminforth/agent 1.37.0 → 1.38.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/agent/languageDetect.ts +0 -8
  2. package/agent/simpleAgent.ts +5 -5
  3. package/agent/systemPrompt.ts +35 -4
  4. package/agent/toolCallEvents.ts +31 -2
  5. package/agent/tools/apiTool.ts +1 -1
  6. package/agentResponseEvents.ts +197 -0
  7. package/apiBasedTools.ts +118 -284
  8. package/build.log +12 -2
  9. package/custom/ChatSurface.vue +31 -21
  10. package/custom/composables/agentAudio/agent-processing.mp3 +0 -0
  11. package/custom/composables/agentStore/constants.ts +8 -1
  12. package/custom/composables/agentStore/useAgentSessions.ts +85 -12
  13. package/custom/composables/useAgentAudio.ts +392 -0
  14. package/custom/composables/useAgentStore.ts +52 -5
  15. package/custom/conversation_area/ConversationArea.vue +1 -1
  16. package/custom/conversation_area/MessageRenderer.vue +12 -1
  17. package/custom/conversation_area/SystemMessageRenderer.vue +28 -0
  18. package/custom/conversation_area/TextRenderer.vue +4 -3
  19. package/custom/conversation_area/ToolRenderer.vue +1 -1
  20. package/custom/package.json +2 -1
  21. package/custom/pnpm-lock.yaml +29 -0
  22. package/custom/speech_recognition_frontend/AudioLines.vue +97 -0
  23. package/custom/speech_recognition_frontend/MicrophoneButon.vue +157 -0
  24. package/custom/speech_recognition_frontend/types/voice-activity-detection.d.ts +22 -0
  25. package/custom/speech_recognition_frontend/voiceActivityDetection.ts +151 -0
  26. package/custom/types.ts +52 -2
  27. package/dist/agent/languageDetect.js +0 -6
  28. package/dist/agent/simpleAgent.js +4 -3
  29. package/dist/agent/systemPrompt.js +24 -3
  30. package/dist/agent/toolCallEvents.js +24 -2
  31. package/dist/agent/tools/apiTool.js +1 -1
  32. package/dist/agentResponseEvents.js +141 -0
  33. package/dist/apiBasedTools.js +95 -211
  34. package/dist/custom/ChatSurface.vue +31 -21
  35. package/dist/custom/composables/agentAudio/agent-processing.mp3 +0 -0
  36. package/dist/custom/composables/agentStore/constants.ts +8 -1
  37. package/dist/custom/composables/agentStore/useAgentSessions.ts +85 -12
  38. package/dist/custom/composables/useAgentAudio.ts +392 -0
  39. package/dist/custom/composables/useAgentStore.ts +52 -5
  40. package/dist/custom/conversation_area/ConversationArea.vue +1 -1
  41. package/dist/custom/conversation_area/MessageRenderer.vue +12 -1
  42. package/dist/custom/conversation_area/SystemMessageRenderer.vue +28 -0
  43. package/dist/custom/conversation_area/TextRenderer.vue +4 -3
  44. package/dist/custom/conversation_area/ToolRenderer.vue +1 -1
  45. package/dist/custom/package.json +2 -1
  46. package/dist/custom/pnpm-lock.yaml +29 -0
  47. package/dist/custom/speech_recognition_frontend/AudioLines.vue +97 -0
  48. package/dist/custom/speech_recognition_frontend/MicrophoneButon.vue +157 -0
  49. package/dist/custom/speech_recognition_frontend/types/voice-activity-detection.d.ts +22 -0
  50. package/dist/custom/speech_recognition_frontend/voiceActivityDetection.ts +151 -0
  51. package/dist/custom/types.ts +52 -2
  52. package/dist/index.js +290 -400
  53. package/index.ts +318 -492
  54. package/package.json +3 -2
  55. package/types.ts +1 -1
@@ -11,11 +11,13 @@ import { AdminForthDataTypes, logger, } from 'adminforth';
11
11
  import dayjs from 'dayjs';
12
12
  import timezone from 'dayjs/plugin/timezone.js';
13
13
  import utc from 'dayjs/plugin/utc.js';
14
- import { inspect } from 'util';
15
14
  import YAML from 'yaml';
16
15
  dayjs.extend(utc);
17
16
  dayjs.extend(timezone);
18
17
  const DEFAULT_USER_TIME_ZONE = 'UTC';
18
+ function hasRegisteredApiToolHandler(schema) {
19
+ return typeof schema.handler === 'function';
20
+ }
19
21
  function getInputString(inputs, key) {
20
22
  const value = inputs === null || inputs === void 0 ? void 0 : inputs[key];
21
23
  return typeof value === 'string' && value ? value : undefined;
@@ -138,38 +140,6 @@ function sanitizeForYaml(value) {
138
140
  }
139
141
  return JSON.parse(serialized);
140
142
  }
141
- export function serializeUnknownError(error) {
142
- var _a, _b;
143
- if (error instanceof Error) {
144
- const errorWithCause = error;
145
- const errorRecord = error;
146
- const serialized = {
147
- name: error.name,
148
- message: error.message,
149
- stack: error.stack,
150
- };
151
- if (errorWithCause.cause !== undefined) {
152
- serialized.cause = serializeUnknownError(errorWithCause.cause);
153
- }
154
- for (const key of Object.getOwnPropertyNames(error)) {
155
- if (key in serialized) {
156
- continue;
157
- }
158
- serialized[key] = errorRecord[key];
159
- }
160
- return serialized;
161
- }
162
- if (typeof error === 'object' && error !== null) {
163
- return {
164
- type: (_b = (_a = error.constructor) === null || _a === void 0 ? void 0 : _a.name) !== null && _b !== void 0 ? _b : 'Object',
165
- inspected: inspect(error, { depth: 6, breakLength: 120 }),
166
- };
167
- }
168
- return {
169
- type: typeof error,
170
- value: error,
171
- };
172
- }
173
143
  function wipePath(target, pathParts) {
174
144
  if (!target || typeof target !== 'object' || pathParts.length === 0) {
175
145
  return;
@@ -243,7 +213,7 @@ function formatDateTimeColumns(rows, dateTimeColumnNames, userTimeZone) {
243
213
  function applyToolOverride(params) {
244
214
  return __awaiter(this, void 0, void 0, function* () {
245
215
  var _a;
246
- const { adminforth, adminUser, httpExtra, inputs, invokeTool, output, toolName, userTimeZone, } = params;
216
+ const { adminforth, adminUser, inputs, output, toolName, userTimeZone } = params;
247
217
  const sanitizedOutput = sanitizeForYaml(output);
248
218
  const override = TOOL_OVERRIDES[toolName];
249
219
  if (!override) {
@@ -252,39 +222,15 @@ function applyToolOverride(params) {
252
222
  for (const path of (_a = override.wipe_frontend_specific_data) !== null && _a !== void 0 ? _a : []) {
253
223
  wipePath(sanitizedOutput, path.split('.'));
254
224
  }
255
- if (!override.post_process_response) {
256
- return sanitizedOutput;
257
- }
258
- const postProcessedOutput = yield override.post_process_response({
259
- adminforth,
260
- output: sanitizedOutput,
261
- adminUser,
262
- httpExtra,
263
- inputs,
264
- userTimeZone,
265
- invokeTool: (nestedToolName_1, ...args_1) => __awaiter(this, [nestedToolName_1, ...args_1], void 0, function* (nestedToolName, nestedParams = {}) {
266
- var _a, _b, _c;
267
- const nestedInputs = (_a = nestedParams.inputs) !== null && _a !== void 0 ? _a : inputs;
268
- const nestedHttpExtra = (_b = nestedParams.httpExtra) !== null && _b !== void 0 ? _b : httpExtra;
269
- const nestedUserTimeZone = (_c = nestedParams.userTimeZone) !== null && _c !== void 0 ? _c : userTimeZone;
270
- const nestedOutput = yield invokeTool(nestedToolName, {
271
- inputs: nestedInputs,
272
- httpExtra: nestedHttpExtra,
273
- userTimeZone: nestedUserTimeZone,
274
- });
275
- return applyToolOverride({
276
- adminforth,
277
- adminUser,
278
- httpExtra: nestedHttpExtra,
279
- inputs: nestedInputs,
280
- invokeTool,
281
- output: nestedOutput,
282
- toolName: nestedToolName,
283
- userTimeZone: nestedUserTimeZone,
284
- });
285
- }),
286
- });
287
- return sanitizeForYaml(postProcessedOutput);
225
+ return override.post_process_response
226
+ ? sanitizeForYaml(yield override.post_process_response({
227
+ adminforth,
228
+ adminUser,
229
+ output: sanitizedOutput,
230
+ inputs,
231
+ userTimeZone,
232
+ }))
233
+ : sanitizedOutput;
288
234
  });
289
235
  }
290
236
  function endpointPathToToolName(path) {
@@ -313,30 +259,16 @@ function formatLogNameList(names) {
313
259
  }
314
260
  export function formatApiBasedToolCall(params) {
315
261
  return __awaiter(this, void 0, void 0, function* () {
316
- var _a;
317
- const formatTool = (_a = TOOL_OVERRIDES[params.toolName]) === null || _a === void 0 ? void 0 : _a.format_tool;
318
- return yield (formatTool === null || formatTool === void 0 ? void 0 : formatTool({
262
+ var _a, _b;
263
+ return yield ((_b = (_a = TOOL_OVERRIDES[params.toolName]) === null || _a === void 0 ? void 0 : _a.format_tool) === null || _b === void 0 ? void 0 : _b.call(_a, {
319
264
  adminforth: params.adminforth,
320
265
  adminUser: params.adminUser,
321
- httpExtra: params.httpExtra,
322
266
  inputs: params.inputs,
323
267
  resourceLabel: resourceLabel(params.adminforth, params.inputs),
324
268
  userTimeZone: params.userTimeZone,
325
- invokeTool: () => __awaiter(this, void 0, void 0, function* () {
326
- throw new Error('Tool info formatting cannot invoke tools');
327
- }),
328
269
  }));
329
270
  });
330
271
  }
331
- function normalizeCookies(cookies) {
332
- if (!cookies) {
333
- return [];
334
- }
335
- if (Array.isArray(cookies)) {
336
- return cookies;
337
- }
338
- return Object.entries(cookies).map(([key, value]) => ({ key, value }));
339
- }
340
272
  function normalizeDateTimeInputsToUtc(body, adminforth, userTimeZone) {
341
273
  if (!userTimeZone || typeof body.resourceId !== 'string') {
342
274
  return body;
@@ -356,10 +288,8 @@ function normalizeDateTimeInputsToUtc(body, adminforth, userTimeZone) {
356
288
  if (columnType === AdminForthDataTypes.DATETIME) {
357
289
  return dayjs.tz(value, userTimeZone).utc().toISOString();
358
290
  }
359
- if (columnType === AdminForthDataTypes.TIME) {
360
- const userDate = dayjs().tz(userTimeZone).format('YYYY-MM-DD');
361
- return dayjs.tz(`${userDate}T${value}`, userTimeZone).utc().format('HH:mm:ss');
362
- }
291
+ const userDate = dayjs().tz(userTimeZone).format('YYYY-MM-DD');
292
+ return dayjs.tz(`${userDate}T${value}`, userTimeZone).utc().format('HH:mm:ss');
363
293
  };
364
294
  const normalizeValue = (value, key) => {
365
295
  const column = key ? columnsByName.get(key) : undefined;
@@ -386,124 +316,92 @@ function normalizeDateTimeInputsToUtc(body, adminforth, userTimeZone) {
386
316
  return normalizeValue(body);
387
317
  }
388
318
  const METHODS_WITHOUT_REQUEST_BODY = new Set(['GET', 'HEAD']);
389
- const HEADERS_NOT_FORWARDED_TO_API_TOOL = new Set([
390
- 'connection',
391
- 'content-length',
392
- 'host',
393
- 'keep-alive',
394
- 'proxy-authenticate',
395
- 'proxy-authorization',
396
- 'te',
397
- 'trailer',
398
- 'transfer-encoding',
399
- 'upgrade',
400
- ]);
401
- function isAbsoluteHttpUrl(value) {
402
- try {
403
- const url = new URL(value);
404
- return url.protocol === 'http:' || url.protocol === 'https:';
405
- }
406
- catch (_a) {
407
- return false;
408
- }
409
- }
410
- function resolveOpenApiRequestUrl(params) {
411
- var _a, _b;
412
- const internalApiOrigin = (_b = (_a = params.adminforth.express).getInternalApiOrigin) === null || _b === void 0 ? void 0 : _b.call(_a);
413
- if (internalApiOrigin) {
414
- const path = isAbsoluteHttpUrl(params.path)
415
- ? `${new URL(params.path).pathname}${new URL(params.path).search}`
416
- : params.path;
417
- return new URL(path, internalApiOrigin).toString();
418
- }
419
- throw new Error(`Tool "${params.toolName}" cannot call OpenAPI path "${params.path}" because internal API origin is unavailable.`);
420
- }
421
- function createToolRequestHeaders(httpExtra, userTimeZone) {
422
- var _a;
423
- const headers = {};
424
- for (const [name, value] of Object.entries((_a = httpExtra === null || httpExtra === void 0 ? void 0 : httpExtra.headers) !== null && _a !== void 0 ? _a : {})) {
425
- const headerName = name.toLowerCase();
426
- if (typeof value === 'string' && !HEADERS_NOT_FORWARDED_TO_API_TOOL.has(headerName)) {
427
- headers[headerName] = value;
428
- }
429
- }
430
- headers.accept = 'application/json';
431
- headers['content-type'] = 'application/json';
432
- if (userTimeZone) {
433
- headers['x-timezone'] = userTimeZone;
434
- }
435
- const cookieHeader = normalizeCookies(httpExtra === null || httpExtra === void 0 ? void 0 : httpExtra.cookies)
436
- .map(({ key, value }) => `${key}=${value}`)
437
- .join('; ');
438
- if (cookieHeader && !headers.cookie) {
439
- headers.cookie = cookieHeader;
440
- }
441
- return headers;
442
- }
443
- function appendInputsToQueryString(url, inputs) {
444
- const nextUrl = new URL(url);
445
- for (const [key, value] of Object.entries(inputs)) {
446
- if (value === undefined) {
447
- continue;
448
- }
449
- if (Array.isArray(value)) {
450
- for (const item of value) {
451
- nextUrl.searchParams.append(key, typeof item === 'object' && item !== null ? JSON.stringify(item) : String(item));
452
- }
453
- continue;
454
- }
455
- nextUrl.searchParams.set(key, typeof value === 'object' && value !== null ? JSON.stringify(value) : String(value));
456
- }
457
- return nextUrl.toString();
319
+ function createDirectToolResponse() {
320
+ const headers = [];
321
+ return {
322
+ headers,
323
+ status: 200,
324
+ setHeader(name, value) {
325
+ headers.push([name, value]);
326
+ },
327
+ setStatus(code, message) {
328
+ this.status = code;
329
+ this.message = message;
330
+ },
331
+ blobStream() {
332
+ throw new Error('blobStream is not available for API-based agent tools');
333
+ },
334
+ };
458
335
  }
459
- function parseOpenApiToolResponse(response) {
460
- return __awaiter(this, void 0, void 0, function* () {
461
- var _a;
462
- const responseText = yield response.text();
463
- const payload = responseText && ((_a = response.headers.get('content-type')) === null || _a === void 0 ? void 0 : _a.includes('application/json'))
464
- ? JSON.parse(responseText)
465
- : responseText;
466
- if (response.ok) {
467
- return responseText ? payload : { status: response.status };
468
- }
469
- return {
470
- error: 'HTTP_ERROR',
471
- status: response.status,
472
- statusText: response.statusText,
473
- response: payload,
474
- };
475
- });
336
+ function validationErrorResponse(error, details) {
337
+ return {
338
+ error,
339
+ details,
340
+ };
476
341
  }
477
342
  function callOpenApiSchema(params) {
478
343
  return __awaiter(this, void 0, void 0, function* () {
479
- var _a;
480
- const { adminforth, httpExtra, inputs, schema, toolName, userTimeZone } = params;
344
+ const { adminforth, adminUser, abortSignal, inputs, schema, toolName, userTimeZone } = params;
481
345
  const method = schema.method.toUpperCase();
482
- const body = normalizeDateTimeInputsToUtc(((_a = inputs !== null && inputs !== void 0 ? inputs : httpExtra === null || httpExtra === void 0 ? void 0 : httpExtra.body) !== null && _a !== void 0 ? _a : {}), adminforth, userTimeZone);
483
- const requestUrl = resolveOpenApiRequestUrl({
484
- adminforth,
485
- path: schema.path,
486
- toolName,
487
- });
346
+ const normalizedInputs = normalizeDateTimeInputsToUtc((inputs !== null && inputs !== void 0 ? inputs : {}), adminforth, userTimeZone);
488
347
  const hasRequestBody = !METHODS_WITHOUT_REQUEST_BODY.has(method);
489
- logger.info(`Calling OpenAPI tool "${toolName}" with method ${method} at URL ${requestUrl}`);
490
- const response = yield fetch(hasRequestBody ? requestUrl : appendInputsToQueryString(requestUrl, body), {
491
- method,
492
- headers: createToolRequestHeaders(httpExtra, userTimeZone),
493
- body: hasRequestBody ? JSON.stringify(body) : undefined,
348
+ const body = hasRequestBody ? normalizedInputs : {};
349
+ const query = hasRequestBody ? {} : normalizedInputs;
350
+ const requestValidation = adminforth.openApi.validateRequestSchema(schema, body);
351
+ if (!requestValidation.valid) {
352
+ return validationErrorResponse('REQUEST_VALIDATION_FAILED', requestValidation.errors);
353
+ }
354
+ const response = createDirectToolResponse();
355
+ logger.info(`Calling OpenAPI tool "${toolName}" with direct handler`);
356
+ const tr = (msg, category, trParams, pluralizationNumber) => adminforth.tr(msg, category, undefined, trParams, pluralizationNumber);
357
+ const output = yield schema.handler({
358
+ body,
359
+ query,
360
+ headers: {},
361
+ cookies: [],
362
+ adminUser,
363
+ response,
364
+ requestUrl: schema.path,
365
+ abortSignal: abortSignal !== null && abortSignal !== void 0 ? abortSignal : new AbortController().signal,
366
+ _raw_express_req: undefined,
367
+ _raw_express_res: undefined,
368
+ tr,
494
369
  });
495
- logger.info(`Received response with status ${response.status} from OpenAPI tool "${toolName}"`);
496
- return parseOpenApiToolResponse(response);
370
+ if (response.message) {
371
+ return response.status >= 400
372
+ ? {
373
+ error: 'HANDLER_ERROR',
374
+ status: response.status,
375
+ response: response.message,
376
+ }
377
+ : response.message;
378
+ }
379
+ if (output === null) {
380
+ return { status: response.status };
381
+ }
382
+ const responseValidation = adminforth.openApi.validateResponseSchema(schema, output);
383
+ if (!responseValidation.valid) {
384
+ return validationErrorResponse('RESPONSE_VALIDATION_FAILED', responseValidation.errors);
385
+ }
386
+ return response.status >= 400
387
+ ? {
388
+ error: 'HANDLER_ERROR',
389
+ status: response.status,
390
+ response: output,
391
+ }
392
+ : output;
497
393
  });
498
394
  }
499
395
  export function prepareApiBasedTools(adminforth, hiddenResourceIds = []) {
500
396
  const apiBasedTools = {};
501
- const openApiSchemas = adminforth.openApi.registeredSchemas.filter((schema) => schema.request_schema || schema.response_schema);
397
+ const openApiSchemas = adminforth.openApi.registeredSchemas;
502
398
  const openApiSchemasByToolName = new Map();
503
399
  const hiddenResourceIdSet = new Set(hiddenResourceIds);
504
400
  for (const schema of openApiSchemas) {
505
401
  const toolName = openApiSchemaPathToToolName(schema.path, adminforth);
506
- openApiSchemasByToolName.set(toolName, schema);
402
+ if (hasRegisteredApiToolHandler(schema)) {
403
+ openApiSchemasByToolName.set(toolName, schema);
404
+ }
507
405
  }
508
406
  logger.info(`AdminForth Agent OpenAPI APIs: ${formatLogNameList(adminforth.openApi.registeredSchemas.map((schema) => openApiSchemaPathToToolName(schema.path, adminforth)))}`);
509
407
  logger.info(`AdminForth Agent OpenAPI tools connected: ${formatLogNameList([...openApiSchemasByToolName.keys()])}`);
@@ -511,40 +409,27 @@ export function prepareApiBasedTools(adminforth, hiddenResourceIds = []) {
511
409
  apiBasedTools[toolName] = {
512
410
  description: schema.description,
513
411
  input_schema: schema.request_schema,
514
- input_schma: schema.request_schema,
515
412
  output_schema: schema.response_schema,
516
- call: (...args_1) => __awaiter(this, [...args_1], void 0, function* ({ adminUser, adminuser, inputs, httpExtra, userTimeZone } = {}) {
413
+ call: (...args_1) => __awaiter(this, [...args_1], void 0, function* ({ adminUser, adminuser, abortSignal, inputs, userTimeZone } = {}) {
517
414
  if (isHiddenResourceCall(hiddenResourceIdSet, inputs)) {
518
415
  return YAML.stringify({
519
416
  error: 'RESOURCE_NOT_AVAILABLE',
520
417
  message: 'This resource is not available to the agent.',
521
418
  });
522
419
  }
523
- const invokeTool = (nextToolName_1, ...args_2) => __awaiter(this, [nextToolName_1, ...args_2], void 0, function* (nextToolName, nextParams = {}) {
524
- const nextSchema = openApiSchemasByToolName.get(nextToolName);
525
- if (!nextSchema) {
526
- throw new Error(`Tool ${nextToolName} is not registered in OpenAPI`);
527
- }
528
- return callOpenApiSchema({
529
- adminforth,
530
- schema: nextSchema,
531
- toolName: nextToolName,
532
- inputs: nextParams.inputs,
533
- httpExtra: nextParams.httpExtra,
534
- userTimeZone: nextParams.userTimeZone,
535
- });
536
- });
537
- const output = yield invokeTool(toolName, {
420
+ const output = yield callOpenApiSchema({
421
+ adminforth,
422
+ adminUser: adminUser !== null && adminUser !== void 0 ? adminUser : adminuser,
423
+ abortSignal,
424
+ schema,
425
+ toolName,
538
426
  inputs,
539
- httpExtra,
540
427
  userTimeZone,
541
428
  });
542
429
  const processedOutput = yield applyToolOverride({
543
430
  adminforth,
544
431
  adminUser: adminUser !== null && adminUser !== void 0 ? adminUser : adminuser,
545
- httpExtra,
546
432
  inputs,
547
- invokeTool,
548
433
  output,
549
434
  toolName,
550
435
  userTimeZone,
@@ -562,7 +447,6 @@ export function serializeApiBasedTool(tool) {
562
447
  return {
563
448
  description: tool.description,
564
449
  input_schema: tool.input_schema,
565
- input_schma: tool.input_schma,
566
450
  output_schema: tool.output_schema,
567
451
  call: '[Function]',
568
452
  };
@@ -118,8 +118,12 @@
118
118
  transition: `transform ${agentTransitions.TRANSITION_DURATION}ms ease-in-out`
119
119
  }"
120
120
  >
121
- <div class="w-full border rounded-lg pb-8 dark:bg-gray-700">
121
+ <div
122
+ class="w-full border rounded-lg pb-8 dark:bg-gray-700"
123
+ :class="agentStore.isAudioChatMode ? 'border-none mt-8' : 'border'"
124
+ >
122
125
  <textarea
126
+ v-if="!agentStore.isAudioChatMode"
123
127
  v-model="agentStore.userMessageInput"
124
128
  ref="textInput"
125
129
  @input="autoResize"
@@ -169,26 +173,29 @@
169
173
  </button>
170
174
  </div>
171
175
  </div>
172
- <Button
173
- v-if="!agentStore.isResponseInProgress"
174
- class="absolute right-4 bottom-2 !p-0 h-9 w-9"
175
- @click="sendMessage"
176
- :disabled="!agentStore.trimmedUserMessage || agentStore.isResponseInProgress"
177
- >
178
- <IconArrowUpOutline
179
- class="w-8 h-8 p-1
180
- text-white"
181
- />
182
- </Button>
183
- <Button
184
- v-else
185
- class="absolute right-4 bottom-2 !p-0 h-9 w-9"
186
- @click="stopCurrentRequest"
187
- >
188
- <div
189
- class="w-3 h-3 bg-white rounded-sm"
190
- />
191
- </Button>
176
+ <MicrophoneButton v-if="props.meta.hasAudioAdapter" />
177
+ <template v-if="!agentStore.isAudioChatMode">
178
+ <Button
179
+ v-if="!agentStore.isResponseInProgress"
180
+ class="absolute right-4 bottom-2 !p-0 h-9 w-9 transition-opacity duration-200"
181
+ @click="sendMessage"
182
+ :disabled="!agentStore.trimmedUserMessage || agentStore.isResponseInProgress"
183
+ >
184
+ <IconArrowUpOutline
185
+ class="w-8 h-8 p-1
186
+ text-white"
187
+ />
188
+ </Button>
189
+ <Button
190
+ v-else
191
+ class="absolute right-4 bottom-2 !p-0 h-9 w-9"
192
+ @click="stopCurrentRequest"
193
+ >
194
+ <div
195
+ class="w-3 h-3 bg-white rounded-sm"
196
+ />
197
+ </Button>
198
+ </template>
192
199
  </div>
193
200
  </div>
194
201
  </div>
@@ -209,6 +216,7 @@ import { useAgentTransitions } from './composables/useAgentTransitions';
209
216
  import { Button } from '@/afcl';
210
217
  import { useCoreStore } from '@/stores/core';
211
218
  import { remToPx } from './utils';
219
+ import MicrophoneButton from './speech_recognition_frontend/MicrophoneButon.vue';
212
220
 
213
221
  const props = defineProps<{
214
222
  meta: {
@@ -218,6 +226,7 @@ const props = defineProps<{
218
226
  }>;
219
227
  defaultModeName: string | null;
220
228
  stickByDefault: boolean;
229
+ hasAudioAdapter: boolean;
221
230
  }
222
231
  adminUser: any
223
232
  }>();
@@ -321,6 +330,7 @@ function selectMode(modeName: string) {
321
330
  }
322
331
 
323
332
  async function sendMessage() {
333
+ if (agentStore.isAudioChatMode) return;
324
334
  isModeMenuOpen.value = false;
325
335
  await agentStore.sendMessage();
326
336
  autoResize();
@@ -9,4 +9,11 @@ export const MIN_WIDTH = 25;
9
9
  export const DEFAULT_TEXTAREA_PLACEHOLDER = 'Type a message...';
10
10
  export const PLACEHOLDER_TYPING_DELAY_MS = 60;
11
11
  export const PLACEHOLDER_DELETING_DELAY_MS = 35;
12
- export const PLACEHOLDER_HOLD_DELAY_MS = 3000;
12
+ export const PLACEHOLDER_HOLD_DELAY_MS = 3000;
13
+ export const PRE_SESSION_ID = 'pre-session';
14
+
15
+ export enum RESERVED_SYSTEM_MESSAGE_CONTENT {
16
+ START_AUDIO_CHAT = 'START_AUDIO_CHAT',
17
+ END_AUDIO_CHAT = 'END_AUDIO_CHAT',
18
+ AGENT_RESPONSE_ABORTED = 'AGENT_RESPONSE_ABORTED'
19
+ }
@@ -2,6 +2,8 @@ import type { ComputedRef, Ref, ShallowRef } from 'vue';
2
2
  import { callAdminForthApi } from '@/utils';
3
3
  import type { Chat } from '../../chat';
4
4
  import type { IAgentSession, ISessionsListItem, IPart } from '../../types';
5
+ import { PRE_SESSION_ID } from './constants';
6
+ import { ChatStatus } from 'ai';
5
7
  import { useI18n } from 'vue-i18n';
6
8
 
7
9
  type AdminforthLike = {
@@ -91,8 +93,8 @@ export function createAgentSessionManager({
91
93
  }
92
94
 
93
95
  async function deletePreSession() {
94
- sessionList.value = sessionList.value.filter((s: ISessionsListItem) => s.sessionId !== 'pre-session');
95
- if (activeSessionId.value === 'pre-session') {
96
+ sessionList.value = sessionList.value.filter((s: ISessionsListItem) => s.sessionId !== PRE_SESSION_ID);
97
+ if (activeSessionId.value === PRE_SESSION_ID) {
96
98
  activeSessionId.value = null;
97
99
  currentSession.value = null;
98
100
  }
@@ -129,7 +131,7 @@ export function createAgentSessionManager({
129
131
  if (!message || isResponseInProgress.value) {
130
132
  return;
131
133
  }
132
- if (!currentSession.value || currentSession.value.sessionId === 'pre-session') {
134
+ if (!currentSession.value || currentSession.value.sessionId === PRE_SESSION_ID) {
133
135
  await createNewSession(message);
134
136
  }
135
137
  currentSession.value!.timestamp = new Date().toISOString();
@@ -146,27 +148,27 @@ export function createAgentSessionManager({
146
148
 
147
149
  async function createPreSession() {
148
150
  saveCurrentSessionInCache();
149
- if (!sessionList.value.some((s: ISessionsListItem) => s.sessionId === 'pre-session')) {
151
+ if (!sessionList.value.some((s: ISessionsListItem) => s.sessionId === PRE_SESSION_ID)) {
150
152
  sessionList.value.unshift({
151
- sessionId: 'pre-session',
153
+ sessionId: PRE_SESSION_ID,
152
154
  title: 'New Session',
153
155
  timestamp: new Date().toISOString(),
154
156
  });
155
157
  }
156
158
 
157
- activeSessionId.value = 'pre-session';
159
+ activeSessionId.value = PRE_SESSION_ID;
158
160
  currentSession.value = {
159
- sessionId: 'pre-session',
161
+ sessionId: PRE_SESSION_ID,
160
162
  title: 'New Session',
161
163
  timestamp: new Date().toISOString(),
162
164
  messages: [],
163
165
  };
164
- sessions.value['pre-session'] = currentSession.value;
165
- setCurrentChat('pre-session');
166
+ sessions.value[PRE_SESSION_ID] = currentSession.value;
167
+ setCurrentChat(PRE_SESSION_ID);
166
168
  }
167
169
 
168
170
  async function deleteSession(sessionId: string) {
169
- if (sessionId === 'pre-session') {
171
+ if (sessionId === PRE_SESSION_ID) {
170
172
  deletePreSession();
171
173
  return;
172
174
  }
@@ -236,7 +238,10 @@ export function createAgentSessionManager({
236
238
  currentChat.value?.messages.push(debugMessage);
237
239
  }
238
240
 
239
- function addSystemMessage(message: string) {
241
+ async function addSystemMessage(message: string) {
242
+ if (!currentSession.value || currentSession.value.sessionId === PRE_SESSION_ID) {
243
+ await createNewSession('Audio chat');
244
+ }
240
245
  const systemMessage = {
241
246
  role: 'system',
242
247
  parts: [{
@@ -247,7 +252,7 @@ export function createAgentSessionManager({
247
252
  };
248
253
  currentChat.value?.messages.push(systemMessage);
249
254
  try {
250
- const res = callAdminForthApi({
255
+ const res = await callAdminForthApi({
251
256
  method: 'POST',
252
257
  path: '/agent/add-system-message-to-turns',
253
258
  body: {
@@ -260,6 +265,69 @@ export function createAgentSessionManager({
260
265
  }
261
266
  }
262
267
 
268
+ function addAgentMessage(message: string) {
269
+ const agentMessage = {
270
+ role: 'assistant',
271
+ parts: [{
272
+ type: 'text',
273
+ text: message,
274
+ state: 'done',
275
+ }]
276
+ };
277
+ currentChat.value?.messages.push(agentMessage);
278
+ }
279
+
280
+ function updateLastAgentMessage(message: string) {
281
+ const lastMsg = currentChat.value?.lastMessage;
282
+ if (lastMsg && lastMsg.role === 'assistant') {
283
+ lastMsg.parts = [{
284
+ type: 'text',
285
+ text: message,
286
+ state: 'done',
287
+ }];
288
+ currentChat.value?.messages.splice(currentChat.value.messages.length - 1, 1, lastMsg);
289
+ } else {
290
+ addAgentMessage(message);
291
+ }
292
+ }
293
+
294
+ function addUserMessage(message: string) {
295
+ const userMessage = {
296
+ role: 'user',
297
+ parts: [{
298
+ type: 'text',
299
+ text: message,
300
+ state: 'done',
301
+ }]
302
+ };
303
+ currentChat.value?.messages.push(userMessage);
304
+ }
305
+
306
+
307
+ function addDataToolCallMessage(data: any) {
308
+ const lastMessage = currentChat.value?.lastMessage;
309
+ if (lastMessage.role === 'assistant') {
310
+ lastMessage.parts.push({
311
+ type: 'data-tool-call',
312
+ data,
313
+ });
314
+ currentChat.value?.messages.splice(currentChat.value.messages.length - 1, 1, lastMessage);
315
+ } else {
316
+ const toolCallMessage = {
317
+ role: 'assistant',
318
+ parts: [{
319
+ type: 'data-tool-call',
320
+ data,
321
+ }]
322
+ };
323
+ currentChat.value?.messages.push(toolCallMessage);
324
+ }
325
+ }
326
+
327
+ function setCurrentChatStatus(status: ChatStatus) {
328
+ (currentChat.value as any)?.setStatus({status});
329
+ }
330
+
263
331
  return {
264
332
  sendMessage,
265
333
  createPreSession,
@@ -268,5 +336,10 @@ export function createAgentSessionManager({
268
336
  deleteSession,
269
337
  addDebugMessage,
270
338
  addSystemMessage,
339
+ addAgentMessage,
340
+ addUserMessage,
341
+ addDataToolCallMessage,
342
+ setCurrentChatStatus,
343
+ updateLastAgentMessage
271
344
  };
272
345
  }