@lobehub/lobehub 2.0.0-next.360 → 2.0.0-next.362

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/CHANGELOG.md +50 -0
  2. package/Dockerfile +2 -1
  3. package/changelog/v1.json +14 -0
  4. package/locales/en-US/chat.json +3 -1
  5. package/locales/zh-CN/chat.json +2 -0
  6. package/package.json +1 -1
  7. package/packages/const/src/userMemory.ts +1 -0
  8. package/packages/context-engine/src/base/BaseEveryUserContentProvider.ts +204 -0
  9. package/packages/context-engine/src/base/BaseLastUserContentProvider.ts +1 -8
  10. package/packages/context-engine/src/base/__tests__/BaseEveryUserContentProvider.test.ts +354 -0
  11. package/packages/context-engine/src/base/constants.ts +20 -0
  12. package/packages/context-engine/src/engine/messages/MessagesEngine.ts +27 -23
  13. package/packages/context-engine/src/engine/messages/__tests__/MessagesEngine.test.ts +364 -0
  14. package/packages/context-engine/src/providers/PageEditorContextInjector.ts +17 -13
  15. package/packages/context-engine/src/providers/PageSelectionsInjector.ts +65 -0
  16. package/packages/context-engine/src/providers/__tests__/PageSelectionsInjector.test.ts +333 -0
  17. package/packages/context-engine/src/providers/index.ts +3 -1
  18. package/packages/database/src/models/userMemory/model.ts +178 -3
  19. package/packages/database/src/models/userMemory/sources/benchmarkLoCoMo.ts +1 -1
  20. package/packages/memory-user-memory/package.json +2 -1
  21. package/packages/memory-user-memory/promptfoo/evals/activity/basic/buildMessages.ts +40 -0
  22. package/packages/memory-user-memory/promptfoo/evals/activity/basic/eval.yaml +13 -0
  23. package/packages/memory-user-memory/promptfoo/evals/activity/basic/prompt.ts +5 -0
  24. package/packages/memory-user-memory/promptfoo/evals/activity/basic/tests/cases.ts +106 -0
  25. package/packages/memory-user-memory/promptfoo/evals/activity/locomo/buildMessages.ts +104 -0
  26. package/packages/memory-user-memory/promptfoo/evals/activity/locomo/eval.yaml +13 -0
  27. package/packages/memory-user-memory/promptfoo/evals/activity/locomo/prompt.ts +5 -0
  28. package/packages/memory-user-memory/promptfoo/evals/activity/locomo/tests/benchmark-locomo-payload-conv-26.json +149 -0
  29. package/packages/memory-user-memory/promptfoo/evals/activity/locomo/tests/cases.ts +72 -0
  30. package/packages/memory-user-memory/promptfoo/response-formats/activity.json +370 -0
  31. package/packages/memory-user-memory/promptfoo/response-formats/experience.json +14 -0
  32. package/packages/memory-user-memory/promptfoo/response-formats/identity.json +281 -255
  33. package/packages/memory-user-memory/promptfooconfig.yaml +1 -0
  34. package/packages/memory-user-memory/scripts/generate-response-formats.ts +26 -2
  35. package/packages/memory-user-memory/src/extractors/activity.ts +44 -0
  36. package/packages/memory-user-memory/src/extractors/gatekeeper.test.ts +2 -1
  37. package/packages/memory-user-memory/src/extractors/gatekeeper.ts +2 -1
  38. package/packages/memory-user-memory/src/extractors/index.ts +1 -0
  39. package/packages/memory-user-memory/src/prompts/gatekeeper.ts +3 -3
  40. package/packages/memory-user-memory/src/prompts/index.ts +7 -1
  41. package/packages/memory-user-memory/src/prompts/layers/activity.ts +90 -0
  42. package/packages/memory-user-memory/src/prompts/layers/index.ts +1 -0
  43. package/packages/memory-user-memory/src/providers/existingUserMemory.test.ts +25 -1
  44. package/packages/memory-user-memory/src/providers/existingUserMemory.ts +113 -0
  45. package/packages/memory-user-memory/src/schemas/activity.ts +315 -0
  46. package/packages/memory-user-memory/src/schemas/experience.ts +5 -5
  47. package/packages/memory-user-memory/src/schemas/gatekeeper.ts +1 -0
  48. package/packages/memory-user-memory/src/schemas/index.ts +1 -0
  49. package/packages/memory-user-memory/src/services/extractExecutor.ts +29 -0
  50. package/packages/memory-user-memory/src/types.ts +7 -0
  51. package/packages/prompts/src/agents/index.ts +1 -0
  52. package/packages/prompts/src/agents/pageSelectionContext.ts +28 -0
  53. package/packages/types/src/aiChat.ts +4 -0
  54. package/packages/types/src/message/common/index.ts +1 -0
  55. package/packages/types/src/message/common/metadata.ts +8 -0
  56. package/packages/types/src/message/common/pageSelection.ts +36 -0
  57. package/packages/types/src/message/ui/params.ts +16 -0
  58. package/packages/types/src/serverConfig.ts +1 -1
  59. package/packages/types/src/userMemory/layers.ts +52 -0
  60. package/packages/types/src/userMemory/list.ts +20 -2
  61. package/packages/types/src/userMemory/shared.ts +22 -1
  62. package/packages/types/src/userMemory/trace.ts +1 -0
  63. package/packages/types/src/util.ts +9 -1
  64. package/scripts/prebuild.mts +1 -0
  65. package/src/features/ChatInput/Desktop/ContextContainer/ContextList.tsx +1 -1
  66. package/src/features/Conversation/ChatInput/index.tsx +9 -1
  67. package/src/features/Conversation/Messages/User/components/MessageContent.tsx +7 -1
  68. package/src/features/Conversation/Messages/User/components/PageSelections.tsx +62 -0
  69. package/src/features/PageEditor/EditorCanvas/useAskCopilotItem.tsx +5 -1
  70. package/src/libs/next/proxy/define-config.ts +1 -0
  71. package/src/locales/default/chat.ts +3 -2
  72. package/src/server/globalConfig/parseMemoryExtractionConfig.ts +7 -1
  73. package/src/server/routers/lambda/aiChat.ts +7 -0
  74. package/src/server/services/memory/userMemory/__tests__/extract.runtime.test.ts +2 -0
  75. package/src/server/services/memory/userMemory/extract.ts +108 -7
  76. package/src/store/chat/slices/aiChat/actions/conversationLifecycle.ts +5 -19
@@ -22,6 +22,26 @@ export enum UserMemoryContextSubjectType {
22
22
  }
23
23
  export const CONTEXT_SUBJECT_TYPES = Object.values(UserMemoryContextSubjectType);
24
24
 
25
+ export interface UserMemoryAssociatedLocation {
26
+ address?: string | null;
27
+ extra?: Record<string, unknown> | null;
28
+ name?: string | null;
29
+ tags?: string[] | null;
30
+ type?: string | null;
31
+ }
32
+
33
+ export interface UserMemoryAssociatedObject {
34
+ extra?: Record<string, unknown> | null;
35
+ name?: string;
36
+ type?: string | null;
37
+ }
38
+
39
+ export interface UserMemoryAssociatedSubject {
40
+ extra?: Record<string, unknown> | null;
41
+ name?: string;
42
+ type?: string | null;
43
+ }
44
+
25
45
  export interface UserMemoryContext extends UserMemoryTimestamps {
26
46
  associatedObjects:
27
47
  | {
@@ -112,3 +132,35 @@ export type UserMemoryPreferenceWithoutVectors = Omit<
112
132
  >;
113
133
 
114
134
  export type UserMemoryPreferencesListItem = Omit<UserMemoryPreferenceWithoutVectors, 'suggestions'>;
135
+
136
+ export interface UserMemoryActivity extends UserMemoryTimestamps {
137
+ associatedLocations: UserMemoryAssociatedLocation[] | null;
138
+ associatedObjects: UserMemoryAssociatedObject[] | null;
139
+ associatedSubjects: UserMemoryAssociatedSubject[] | null;
140
+ endsAt: Date | null;
141
+ feedback: string | null;
142
+ feedbackVector: number[] | null;
143
+ id: string;
144
+ metadata: Record<string, unknown> | null;
145
+ narrative: string | null;
146
+ narrativeVector: number[] | null;
147
+ notes: string | null;
148
+ startsAt: Date | null;
149
+ status: string | null;
150
+ tags: string[] | null;
151
+ timezone: string | null;
152
+ type: string | null;
153
+ userId: string | null;
154
+ userMemoryId: string | null;
155
+ }
156
+
157
+ // eslint-disable-next-line @typescript-eslint/no-empty-interface
158
+ export type UserMemoryActivityWithoutVectors = Omit<
159
+ UserMemoryActivity,
160
+ 'narrativeVector' | 'feedbackVector'
161
+ >;
162
+
163
+ export type UserMemoryActivitiesListItem = Omit<
164
+ UserMemoryActivityWithoutVectors,
165
+ 'feedback' | 'narrative' | 'notes'
166
+ >;
@@ -5,6 +5,8 @@ import {
5
5
  UserMemoryContextsListItem,
6
6
  UserMemoryExperiencesListItem,
7
7
  UserMemoryPreferencesListItem,
8
+ UserMemoryActivitiesListItem,
9
+ UserMemoryActivityWithoutVectors,
8
10
  } from './layers'
9
11
  import {
10
12
  UserMemoryIdentityWithoutVectors,
@@ -87,14 +89,30 @@ export interface IdentityMemoryDetail {
87
89
  sourceType?: MemorySourceType;
88
90
  }
89
91
 
92
+ export interface ActivityMemorySimple {
93
+ activity: UserMemoryActivitiesListItem;
94
+ layer: LayersEnum.Activity;
95
+ memory: UserMemoryListItem;
96
+ }
97
+
98
+ export interface ActivityMemoryDetail {
99
+ activity: UserMemoryActivityWithoutVectors;
100
+ layer: LayersEnum.Activity;
101
+ memory: UserMemoryWithoutVectors;
102
+ source?: MemorySource;
103
+ sourceType?: MemorySourceType;
104
+ }
105
+
90
106
  export type UserMemoryItemSimple =
91
107
  | ContextMemorySimple
92
108
  | ExperienceMemorySimple
93
109
  | IdentityMemorySimple
94
- | PreferenceMemorySimple;
110
+ | PreferenceMemorySimple
111
+ | ActivityMemorySimple;
95
112
 
96
113
  export type UserMemoryDetail =
97
114
  | ContextMemoryDetail
98
115
  | ExperienceMemoryDetail
99
116
  | IdentityMemoryDetail
100
- | PreferenceMemoryDetail;
117
+ | PreferenceMemoryDetail
118
+ | ActivityMemoryDetail;
@@ -46,6 +46,7 @@ export enum IdentityTypeEnum {
46
46
  export const IDENTITY_TYPES = Object.values(IdentityTypeEnum);
47
47
 
48
48
  export enum LayersEnum {
49
+ Activity = 'activity',
49
50
  Context = 'context',
50
51
  Experience = 'experience',
51
52
  Identity = 'identity',
@@ -80,7 +81,6 @@ export const CONTEXT_STATUS = Object.values(ContextStatusEnum);
80
81
  /**
81
82
  * Shared types for memory list queries
82
83
  */
83
-
84
84
  export interface BaseListParams {
85
85
  order?: 'asc' | 'desc';
86
86
  page?: number;
@@ -106,3 +106,24 @@ export interface BaseListItem {
106
106
  type: string | null;
107
107
  updatedAt: Date;
108
108
  }
109
+
110
+ export enum ActivityTypeEnum {
111
+ Appointment = 'appointment',
112
+ Call = 'call',
113
+ Celebration = 'celebration',
114
+ Class = 'class',
115
+ Conference = 'conference',
116
+ Errand = 'errand',
117
+ Event = 'event',
118
+ Exercise = 'exercise',
119
+ Meal = 'meal',
120
+ Meeting = 'meeting',
121
+ Other = 'other',
122
+ ProjectSession = 'project-session',
123
+ Social = 'social',
124
+ Task = 'task',
125
+ Trip = 'trip',
126
+ Workshop = 'workshop',
127
+ }
128
+
129
+ export const ACTIVITY_TYPES = Object.values(ActivityTypeEnum);
@@ -3,6 +3,7 @@ import type { MemorySourceType } from './list';
3
3
 
4
4
  export type MemoryExtractionAgent =
5
5
  | 'gatekeeper'
6
+ | 'layer-activity'
6
7
  | 'layer-context'
7
8
  | 'layer-experience'
8
9
  | 'layer-identity'
@@ -1,2 +1,10 @@
1
1
  export type Primitive = number | string | boolean | undefined | null | any[];
2
- export type Optional<T, K extends keyof T> = { [P in K]?: T[P] } & { [P in Exclude<keyof T, K>]: T[P] extends Primitive ? T[P] : Optional<T[P], keyof T[P] & K> };
2
+ type OptionalKeys<T> = { [P in keyof T]-?: undefined extends T[P] ? P : never }[keyof T];
3
+ type OptionalField<T, P extends keyof T, K extends keyof T> = T[P] extends Primitive
4
+ ? T[P]
5
+ : Optional<T[P], keyof T[P] & K>;
6
+
7
+ export type Optional<T, K extends keyof T> = T extends null | undefined
8
+ ? T
9
+ : Partial<Pick<T, K | OptionalKeys<T>>> &
10
+ { [P in Exclude<keyof T, K | OptionalKeys<T>>]: OptionalField<T, P, K> };
@@ -51,6 +51,7 @@ const checkRequiredEnvVars = () => {
51
51
  console.error(` 📖 Documentation: ${docUrl}\n`);
52
52
  }
53
53
  console.error('Please configure these environment variables and redeploy.');
54
+ console.error('\n💡 TIP: If you previously used NEXT_AUTH_SECRET, simply rename it to AUTH_SECRET.');
54
55
  console.error('═'.repeat(70) + '\n');
55
56
  process.exit(1);
56
57
  }
@@ -30,7 +30,7 @@ const ContextList = memo(() => {
30
30
  const rawSelectionList = useFileStore(fileChatSelectors.chatContextSelections);
31
31
  const showSelectionList = useFileStore(fileChatSelectors.chatContextSelectionHasItem);
32
32
  const clearChatContextSelections = useFileStore((s) => s.clearChatContextSelections);
33
-
33
+ console.log(rawSelectionList);
34
34
  // Clear selections only when agentId changes (not on initial mount)
35
35
  useEffect(() => {
36
36
  if (prevAgentIdRef.current !== undefined && prevAgentIdRef.current !== agentId) {
@@ -110,8 +110,16 @@ const ChatInput = memo<ChatInputProps>(
110
110
  fileStore.clearChatUploadFileList();
111
111
  fileStore.clearChatContextSelections();
112
112
 
113
+ // Convert ChatContextContent to PageSelection for persistence
114
+ const pageSelections = currentContextList.map((ctx) => ({
115
+ content: ctx.preview || '',
116
+ id: ctx.id,
117
+ pageId: ctx.pageId || '',
118
+ xml: ctx.content,
119
+ }));
120
+
113
121
  // Fire and forget - send with captured message
114
- await sendMessage({ contexts: currentContextList, files: currentFileList, message });
122
+ await sendMessage({ files: currentFileList, message, pageSelections });
115
123
  },
116
124
  [isAIGenerating, sendMessage],
117
125
  );
@@ -7,13 +7,19 @@ import { type UIChatMessage } from '@/types/index';
7
7
  import { useMarkdown } from '../useMarkdown';
8
8
  import FileListViewer from './FileListViewer';
9
9
  import ImageFileListViewer from './ImageFileListViewer';
10
+ import PageSelections from './PageSelections';
10
11
  import VideoFileListViewer from './VideoFileListViewer';
11
12
 
12
13
  const UserMessageContent = memo<UIChatMessage>(
13
- ({ id, content, imageList, videoList, fileList }) => {
14
+ ({ id, content, imageList, videoList, fileList, metadata }) => {
14
15
  const markdownProps = useMarkdown(id);
16
+ const pageSelections = metadata?.pageSelections;
17
+
15
18
  return (
16
19
  <Flexbox gap={8} id={id}>
20
+ {pageSelections && pageSelections.length > 0 && (
21
+ <PageSelections selections={pageSelections} />
22
+ )}
17
23
  {content && <MarkdownMessage {...markdownProps}>{content}</MarkdownMessage>}
18
24
  {imageList && imageList?.length > 0 && <ImageFileListViewer items={imageList} />}
19
25
  {videoList && videoList?.length > 0 && <VideoFileListViewer items={videoList} />}
@@ -0,0 +1,62 @@
1
+ import { Flexbox } from '@lobehub/ui';
2
+ import { createStaticStyles } from 'antd-style';
3
+ import { memo } from 'react';
4
+
5
+ import type { PageSelection } from '@/types/index';
6
+
7
+ const styles = createStaticStyles(({ css, cssVar }) => ({
8
+ container: css`
9
+ cursor: pointer;
10
+ position: relative;
11
+ border-radius: 8px;
12
+
13
+ :hover {
14
+ background: ${cssVar.colorFillQuaternary};
15
+ }
16
+ `,
17
+ content: css`
18
+ overflow: hidden;
19
+ display: -webkit-box;
20
+ -webkit-box-orient: vertical;
21
+ -webkit-line-clamp: 3;
22
+
23
+ font-size: 12px;
24
+ line-height: 1.5;
25
+ color: ${cssVar.colorTextSecondary};
26
+ white-space: pre-wrap;
27
+ `,
28
+ quote: css`
29
+ inset-block-start: 2px;
30
+ inset-inline-start: 0;
31
+
32
+ font-family: Georgia, serif;
33
+ font-size: 28px;
34
+ line-height: 1;
35
+ color: ${cssVar.colorTextQuaternary};
36
+ `,
37
+ wrapper: css``,
38
+ }));
39
+
40
+ interface PageSelectionsProps {
41
+ selections: PageSelection[];
42
+ }
43
+
44
+ const PageSelections = memo<PageSelectionsProps>(({ selections }) => {
45
+ if (!selections || selections.length === 0) return null;
46
+
47
+ return (
48
+ <Flexbox gap={8}>
49
+ {selections.map((selection) => (
50
+ <Flexbox className={styles.container} key={selection.id}>
51
+ <Flexbox className={styles.wrapper} gap={4} horizontal padding={4}>
52
+ {/* eslint-disable-next-line react/no-unescaped-entities */}
53
+ <span className={styles.quote}>"</span>
54
+ <div className={styles.content}>{selection.content}</div>
55
+ </Flexbox>
56
+ </Flexbox>
57
+ ))}
58
+ </Flexbox>
59
+ );
60
+ });
61
+
62
+ export default PageSelections;
@@ -12,6 +12,8 @@ import { useTranslation } from 'react-i18next';
12
12
  import { useFileStore } from '@/store/file';
13
13
  import { useGlobalStore } from '@/store/global';
14
14
 
15
+ import { usePageEditorStore } from '../store';
16
+
15
17
  const styles = createStaticStyles(({ css }) => ({
16
18
  askCopilot: css`
17
19
  border-radius: 6px;
@@ -26,6 +28,7 @@ const styles = createStaticStyles(({ css }) => ({
26
28
  export const useAskCopilotItem = (editor: IEditor | undefined): ChatInputActionsProps['items'] => {
27
29
  const { t } = useTranslation('common');
28
30
  const addSelectionContext = useFileStore((s) => s.addChatContextSelection);
31
+ const pageId = usePageEditorStore((s) => s.documentId);
29
32
 
30
33
  return useMemo(() => {
31
34
  if (!editor) return [];
@@ -60,6 +63,7 @@ export const useAskCopilotItem = (editor: IEditor | undefined): ChatInputActions
60
63
  content,
61
64
  format,
62
65
  id: `selection-${nanoid(6)}`,
66
+ pageId,
63
67
  preview,
64
68
  title: 'Selection',
65
69
  type: 'text',
@@ -95,5 +99,5 @@ export const useAskCopilotItem = (editor: IEditor | undefined): ChatInputActions
95
99
  onClick: () => {},
96
100
  },
97
101
  ];
98
- }, [addSelectionContext, editor, t]);
102
+ }, [addSelectionContext, editor, pageId, t]);
99
103
  };
@@ -162,6 +162,7 @@ export function defineConfig() {
162
162
  '/api/webhooks(.*)',
163
163
  '/api/workflows(.*)',
164
164
  '/api/agent(.*)',
165
+ '/api/dev(.*)',
165
166
  '/webapi(.*)',
166
167
  '/trpc(.*)',
167
168
  // better auth
@@ -234,8 +234,9 @@ export default {
234
234
  'operation.sendMessage': 'Sending message',
235
235
  'owner': 'Group owner',
236
236
  'pageCopilot.title': 'Page Agent',
237
- 'pageCopilot.welcome':
238
- '**Clearer, sharper writing**\n\nDraft, rewrite, or polish—tell me your intent and I’ll refine the rest.',
237
+ 'pageCopilot.welcome': `**Clearer, sharper writing**\n\nDraft, rewrite, or polish—tell me your intent and I'll refine the rest.`,
238
+ 'pageSelection.lines': 'Lines {{start}}-{{end}}',
239
+ 'pageSelection.reference': 'Selected Text',
239
240
  'pin': 'Pin',
240
241
  'pinOff': 'Unpin',
241
242
  'prompts.summaryExpert':
@@ -8,7 +8,13 @@ import {
8
8
  type MemoryLayerExtractorPublicConfig,
9
9
  } from '@/types/serverConfig';
10
10
 
11
- const MEMORY_LAYERS: GlobalMemoryLayer[] = ['context', 'experience', 'identity', 'preference'];
11
+ const MEMORY_LAYERS: GlobalMemoryLayer[] = [
12
+ 'activity',
13
+ 'context',
14
+ 'experience',
15
+ 'identity',
16
+ 'preference',
17
+ ];
12
18
 
13
19
  const parseTokenLimitEnv = (value?: string) => {
14
20
  if (value === undefined) return undefined;
@@ -123,11 +123,18 @@ export const aiChatRouter = router({
123
123
 
124
124
  // create user message
125
125
  log('creating user message with content length: %d', input.newUserMessage.content.length);
126
+
127
+ // Build user message metadata with pageSelections if present
128
+ const userMessageMetadata = input.newUserMessage.pageSelections?.length
129
+ ? { pageSelections: input.newUserMessage.pageSelections }
130
+ : undefined;
131
+
126
132
  const userMessageItem = await ctx.messageModel.create({
127
133
  agentId: input.agentId,
128
134
  content: input.newUserMessage.content,
129
135
  files: input.newUserMessage.files,
130
136
  groupId: input.groupId,
137
+ metadata: userMessageMetadata,
131
138
  parentId: input.newUserMessage.parentId,
132
139
  role: 'user',
133
140
  sessionId,
@@ -26,6 +26,7 @@ const createExecutor = (privateOverrides?: Partial<MemoryExtractionPrivateConfig
26
26
  agentLayerExtractor: {
27
27
  contextLimit: 2048,
28
28
  layers: {
29
+ activity: 'layer-act',
29
30
  context: 'layer-ctx',
30
31
  experience: 'layer-exp',
31
32
  identity: 'layer-id',
@@ -38,6 +39,7 @@ const createExecutor = (privateOverrides?: Partial<MemoryExtractionPrivateConfig
38
39
  embedding: { model: 'embed-1', provider: 'provider-e' },
39
40
  featureFlags: { enableBenchmarkLoCoMo: false },
40
41
  observabilityS3: { enabled: false },
42
+ webhookHeaders: {},
41
43
  };
42
44
 
43
45
  const serverConfig = {
@@ -90,6 +90,7 @@ const LAYER_ALIAS = new Set<LayersEnum>([
90
90
  ]);
91
91
 
92
92
  const LAYER_LABEL_MAP: Record<LayersEnum, string> = {
93
+ [LayersEnum.Activity]: 'activities',
93
94
  [LayersEnum.Context]: 'contexts',
94
95
  [LayersEnum.Experience]: 'experiences',
95
96
  [LayersEnum.Identity]: 'identities',
@@ -267,6 +268,7 @@ const resolveLayerModels = (
267
268
  layers: Partial<Record<GlobalMemoryLayer, string>> | undefined,
268
269
  fallback: Record<GlobalMemoryLayer, string>,
269
270
  ): Record<LayersEnum, string> => ({
271
+ activity: layers?.activity ?? fallback.activity,
270
272
  context: layers?.context ?? fallback.context,
271
273
  experience: layers?.experience ?? fallback.experience,
272
274
  identity: layers?.identity ?? fallback.identity,
@@ -569,6 +571,79 @@ export class MemoryExtractionExecutor {
569
571
  });
570
572
  }
571
573
 
574
+ async persistActivityMemories(
575
+ job: MemoryExtractionJob,
576
+ messageIds: string[],
577
+ result: NonNullable<MemoryExtractionResult['outputs']['activity']>['data'],
578
+ runtime: ModelRuntime,
579
+ model: string,
580
+ tokenLimit: number | undefined,
581
+ db: Awaited<ReturnType<typeof getServerDB>>,
582
+ ) {
583
+ const insertedIds: string[] = [];
584
+ const userMemoryModel = new UserMemoryModel(db, job.userId);
585
+
586
+ for (const item of result?.memories ?? []) {
587
+ const activityTags = item.withActivity?.tags ?? item.tags;
588
+ const associatedObjects = UserMemoryModel.parseAssociatedObjects(
589
+ item.withActivity?.associatedObjects,
590
+ );
591
+ const associatedSubjects = UserMemoryModel.parseAssociatedSubjects(
592
+ item.withActivity?.associatedSubjects,
593
+ );
594
+ const associatedLocations = UserMemoryModel.parseAssociatedLocations(
595
+ item.withActivity?.associatedLocations,
596
+ );
597
+ const [summaryVector, detailsVector, narrativeVector, feedbackVector] =
598
+ await this.generateEmbeddings(
599
+ runtime,
600
+ model,
601
+ [item.summary, item.details, item.withActivity?.narrative, item.withActivity?.feedback],
602
+ tokenLimit,
603
+ );
604
+ const baseMetadata = this.buildBaseMetadata(
605
+ job,
606
+ messageIds,
607
+ LayersEnum.Activity,
608
+ activityTags,
609
+ );
610
+
611
+ const { memory } = await userMemoryModel.createActivityMemory({
612
+ activity: {
613
+ associatedLocations: associatedLocations.length > 0 ? associatedLocations : [],
614
+ associatedObjects: associatedObjects.length > 0 ? associatedObjects : [],
615
+ associatedSubjects: associatedSubjects.length > 0 ? associatedSubjects : [],
616
+ capturedAt: job.sourceUpdatedAt,
617
+ endsAt: UserMemoryModel.parseDateFromString(item.withActivity?.endsAt),
618
+ feedback: item.withActivity?.feedback ?? null,
619
+ feedbackVector: feedbackVector ?? null,
620
+ metadata: baseMetadata,
621
+ narrative: item.withActivity?.narrative ?? null,
622
+ narrativeVector: narrativeVector ?? null,
623
+ notes: item.withActivity?.notes ?? null,
624
+ startsAt: UserMemoryModel.parseDateFromString(item.withActivity?.startsAt),
625
+ status: item.withActivity?.status ?? 'pending',
626
+ tags: activityTags ?? null,
627
+ timezone: item.withActivity?.timezone ?? null,
628
+ type: item.withActivity?.type ?? 'other',
629
+ },
630
+ capturedAt: job.sourceUpdatedAt,
631
+ details: item.details ?? '',
632
+ detailsEmbedding: detailsVector ?? undefined,
633
+ memoryCategory: item.memoryCategory ?? null,
634
+ memoryLayer: (item.memoryLayer as LayersEnum) ?? LayersEnum.Activity,
635
+ memoryType: (item.memoryType as TypesEnum) ?? TypesEnum.Activity,
636
+ summary: item.summary ?? '',
637
+ summaryEmbedding: summaryVector ?? undefined,
638
+ title: item.title ?? '',
639
+ });
640
+
641
+ insertedIds.push(memory.id);
642
+ }
643
+
644
+ return insertedIds;
645
+ }
646
+
572
647
  async persistContextMemories(
573
648
  job: MemoryExtractionJob,
574
649
  messageIds: string[],
@@ -601,7 +676,7 @@ export class MemoryExtractionExecutor {
601
676
  associatedObjects: UserMemoryModel.parseAssociatedObjects(
602
677
  item.withContext?.associatedObjects,
603
678
  ),
604
- associatedSubjects: UserMemoryModel.parseAssociatedObjects(
679
+ associatedSubjects: UserMemoryModel.parseAssociatedSubjects(
605
680
  item.withContext?.associatedSubjects,
606
681
  ),
607
682
  capturedAt: job.sourceUpdatedAt,
@@ -923,13 +998,14 @@ export class MemoryExtractionExecutor {
923
998
  if (vector) {
924
999
  const retrieved = await userMemoryModel.searchWithEmbedding({
925
1000
  embedding: vector,
926
- limits: { contexts: topK, experiences: topK, preferences: topK },
1001
+ limits: { activities: topK, contexts: topK, experiences: topK, preferences: topK },
927
1002
  });
928
1003
 
929
1004
  return retrieved;
930
1005
  }
931
1006
 
932
1007
  return {
1008
+ activities: [],
933
1009
  contexts: [],
934
1010
  experiences: [],
935
1011
  preferences: [],
@@ -1563,6 +1639,24 @@ export class MemoryExtractionExecutor {
1563
1639
  );
1564
1640
  };
1565
1641
 
1642
+ const activityOutput = extraction.outputs.activity;
1643
+ if (activityOutput?.error) {
1644
+ appendError(LayersEnum.Activity, 'extract', activityOutput.error);
1645
+ }
1646
+ if (activityOutput?.data) {
1647
+ await persistWithSpan(LayersEnum.Activity, () =>
1648
+ this.persistActivityMemories(
1649
+ job,
1650
+ messageIds,
1651
+ activityOutput.data,
1652
+ runtimes.embeddings,
1653
+ this.modelConfig.embeddingsModel,
1654
+ this.embeddingContextLimit,
1655
+ db,
1656
+ ),
1657
+ );
1658
+ }
1659
+
1566
1660
  const contextOutput = extraction.outputs.context;
1567
1661
  if (contextOutput?.error) {
1568
1662
  appendError(LayersEnum.Context, 'extract', contextOutput.error);
@@ -1636,7 +1730,7 @@ export class MemoryExtractionExecutor {
1636
1730
  }
1637
1731
 
1638
1732
  if (errors.length) {
1639
- const detail = errors.map((error) => error.message).join('; ');
1733
+ const detail = errors.map((error) => `${error.message}${error.cause ? `: ${error.cause}` : ''}`).join('; ');
1640
1734
  throw new AggregateError(errors, `Memory extraction encountered layer errors: ${detail}`);
1641
1735
  }
1642
1736
 
@@ -1825,14 +1919,21 @@ export class MemoryExtractionExecutor {
1825
1919
  userId: params.userId,
1826
1920
  });
1827
1921
 
1922
+ const latestCreatedAt = params.parts.reduce<Date | undefined>((latest, part) => {
1923
+ if (!part.createdAt) return latest;
1924
+
1925
+ const date = new Date(part.createdAt);
1926
+ if (Number.isNaN(date.getTime())) return latest;
1927
+
1928
+ return !latest || date > latest ? date : latest;
1929
+ }, undefined);
1930
+
1828
1931
  extractionJob = {
1829
1932
  force: params.forceAll ?? true,
1830
1933
  layers: params.layers,
1831
1934
  source: params.source,
1832
1935
  sourceId: params.sourceId,
1833
- sourceUpdatedAt: params.parts.at(-1)?.createdAt
1834
- ? new Date(params.parts.at(-1)!.createdAt as Date)
1835
- : new Date(),
1936
+ sourceUpdatedAt: latestCreatedAt ?? new Date(),
1836
1937
  userId: params.userId,
1837
1938
  };
1838
1939
 
@@ -1898,7 +1999,7 @@ export class MemoryExtractionExecutor {
1898
1999
  language,
1899
2000
  retrievedContexts: [trimmedContext],
1900
2001
  retrievedIdentitiesContext: undefined,
1901
- sessionDate: new Date().toISOString(),
2002
+ sessionDate: (latestCreatedAt ?? new Date()).toISOString(),
1902
2003
  topK: 10,
1903
2004
  username:
1904
2005
  userState.fullName || `${userState.firstName} ${userState.lastName}`.trim() || 'User',
@@ -77,11 +77,11 @@ export const conversationLifecycle: StateCreator<
77
77
  sendMessage: async ({
78
78
  message,
79
79
  files,
80
- contexts,
81
80
  onlyAddUserMessage,
82
81
  context,
83
82
  messages: inputMessages,
84
83
  parentId: inputParentId,
84
+ pageSelections,
85
85
  }) => {
86
86
  const { internal_execAgentRuntime, mainInputEditor } = get();
87
87
 
@@ -186,6 +186,8 @@ export const conversationLifecycle: StateCreator<
186
186
  threadId: operationContext.threadId ?? undefined,
187
187
  imageList: tempImages.length > 0 ? tempImages : undefined,
188
188
  videoList: tempVideos.length > 0 ? tempVideos : undefined,
189
+ // Pass pageSelections metadata for immediate display
190
+ metadata: pageSelections?.length ? { pageSelections } : undefined,
189
191
  },
190
192
  { operationId, tempMessageId: tempId },
191
193
  );
@@ -222,7 +224,7 @@ export const conversationLifecycle: StateCreator<
222
224
  const topicId = operationContext.topicId;
223
225
  data = await aiChatService.sendMessageInServer(
224
226
  {
225
- newUserMessage: { content: message, files: fileIdList, parentId },
227
+ newUserMessage: { content: message, files: fileIdList, pageSelections, parentId },
226
228
  // if there is topicId,then add topicId to message
227
229
  topicId: topicId ?? undefined,
228
230
  threadId: operationContext.threadId ?? undefined,
@@ -372,26 +374,10 @@ export const conversationLifecycle: StateCreator<
372
374
  messageMapKey(execContext),
373
375
  )(get());
374
376
 
375
- const contextMessages =
376
- contexts?.map((item, index) => {
377
- const now = Date.now();
378
- const title = item.title ? `${item.title}\n` : '';
379
- return {
380
- content: `Context ${index + 1}:\n${title}${item.content}`,
381
- createdAt: now,
382
- id: `ctx_${tempId}_${index}`,
383
- role: 'system' as const,
384
- updatedAt: now,
385
- };
386
- }) ?? [];
387
-
388
- const runtimeMessages =
389
- contextMessages.length > 0 ? [...displayMessages, ...contextMessages] : displayMessages;
390
-
391
377
  try {
392
378
  await internal_execAgentRuntime({
393
379
  context: execContext,
394
- messages: runtimeMessages,
380
+ messages: displayMessages,
395
381
  parentMessageId: data.assistantMessageId,
396
382
  parentMessageType: 'assistant',
397
383
  parentOperationId: operationId, // Pass as parent operation