@lobehub/lobehub 2.0.0-next.360 → 2.0.0-next.362
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +50 -0
- package/Dockerfile +2 -1
- package/changelog/v1.json +14 -0
- package/locales/en-US/chat.json +3 -1
- package/locales/zh-CN/chat.json +2 -0
- package/package.json +1 -1
- package/packages/const/src/userMemory.ts +1 -0
- package/packages/context-engine/src/base/BaseEveryUserContentProvider.ts +204 -0
- package/packages/context-engine/src/base/BaseLastUserContentProvider.ts +1 -8
- package/packages/context-engine/src/base/__tests__/BaseEveryUserContentProvider.test.ts +354 -0
- package/packages/context-engine/src/base/constants.ts +20 -0
- package/packages/context-engine/src/engine/messages/MessagesEngine.ts +27 -23
- package/packages/context-engine/src/engine/messages/__tests__/MessagesEngine.test.ts +364 -0
- package/packages/context-engine/src/providers/PageEditorContextInjector.ts +17 -13
- package/packages/context-engine/src/providers/PageSelectionsInjector.ts +65 -0
- package/packages/context-engine/src/providers/__tests__/PageSelectionsInjector.test.ts +333 -0
- package/packages/context-engine/src/providers/index.ts +3 -1
- package/packages/database/src/models/userMemory/model.ts +178 -3
- package/packages/database/src/models/userMemory/sources/benchmarkLoCoMo.ts +1 -1
- package/packages/memory-user-memory/package.json +2 -1
- package/packages/memory-user-memory/promptfoo/evals/activity/basic/buildMessages.ts +40 -0
- package/packages/memory-user-memory/promptfoo/evals/activity/basic/eval.yaml +13 -0
- package/packages/memory-user-memory/promptfoo/evals/activity/basic/prompt.ts +5 -0
- package/packages/memory-user-memory/promptfoo/evals/activity/basic/tests/cases.ts +106 -0
- package/packages/memory-user-memory/promptfoo/evals/activity/locomo/buildMessages.ts +104 -0
- package/packages/memory-user-memory/promptfoo/evals/activity/locomo/eval.yaml +13 -0
- package/packages/memory-user-memory/promptfoo/evals/activity/locomo/prompt.ts +5 -0
- package/packages/memory-user-memory/promptfoo/evals/activity/locomo/tests/benchmark-locomo-payload-conv-26.json +149 -0
- package/packages/memory-user-memory/promptfoo/evals/activity/locomo/tests/cases.ts +72 -0
- package/packages/memory-user-memory/promptfoo/response-formats/activity.json +370 -0
- package/packages/memory-user-memory/promptfoo/response-formats/experience.json +14 -0
- package/packages/memory-user-memory/promptfoo/response-formats/identity.json +281 -255
- package/packages/memory-user-memory/promptfooconfig.yaml +1 -0
- package/packages/memory-user-memory/scripts/generate-response-formats.ts +26 -2
- package/packages/memory-user-memory/src/extractors/activity.ts +44 -0
- package/packages/memory-user-memory/src/extractors/gatekeeper.test.ts +2 -1
- package/packages/memory-user-memory/src/extractors/gatekeeper.ts +2 -1
- package/packages/memory-user-memory/src/extractors/index.ts +1 -0
- package/packages/memory-user-memory/src/prompts/gatekeeper.ts +3 -3
- package/packages/memory-user-memory/src/prompts/index.ts +7 -1
- package/packages/memory-user-memory/src/prompts/layers/activity.ts +90 -0
- package/packages/memory-user-memory/src/prompts/layers/index.ts +1 -0
- package/packages/memory-user-memory/src/providers/existingUserMemory.test.ts +25 -1
- package/packages/memory-user-memory/src/providers/existingUserMemory.ts +113 -0
- package/packages/memory-user-memory/src/schemas/activity.ts +315 -0
- package/packages/memory-user-memory/src/schemas/experience.ts +5 -5
- package/packages/memory-user-memory/src/schemas/gatekeeper.ts +1 -0
- package/packages/memory-user-memory/src/schemas/index.ts +1 -0
- package/packages/memory-user-memory/src/services/extractExecutor.ts +29 -0
- package/packages/memory-user-memory/src/types.ts +7 -0
- package/packages/prompts/src/agents/index.ts +1 -0
- package/packages/prompts/src/agents/pageSelectionContext.ts +28 -0
- package/packages/types/src/aiChat.ts +4 -0
- package/packages/types/src/message/common/index.ts +1 -0
- package/packages/types/src/message/common/metadata.ts +8 -0
- package/packages/types/src/message/common/pageSelection.ts +36 -0
- package/packages/types/src/message/ui/params.ts +16 -0
- package/packages/types/src/serverConfig.ts +1 -1
- package/packages/types/src/userMemory/layers.ts +52 -0
- package/packages/types/src/userMemory/list.ts +20 -2
- package/packages/types/src/userMemory/shared.ts +22 -1
- package/packages/types/src/userMemory/trace.ts +1 -0
- package/packages/types/src/util.ts +9 -1
- package/scripts/prebuild.mts +1 -0
- package/src/features/ChatInput/Desktop/ContextContainer/ContextList.tsx +1 -1
- package/src/features/Conversation/ChatInput/index.tsx +9 -1
- package/src/features/Conversation/Messages/User/components/MessageContent.tsx +7 -1
- package/src/features/Conversation/Messages/User/components/PageSelections.tsx +62 -0
- package/src/features/PageEditor/EditorCanvas/useAskCopilotItem.tsx +5 -1
- package/src/libs/next/proxy/define-config.ts +1 -0
- package/src/locales/default/chat.ts +3 -2
- package/src/server/globalConfig/parseMemoryExtractionConfig.ts +7 -1
- package/src/server/routers/lambda/aiChat.ts +7 -0
- package/src/server/services/memory/userMemory/__tests__/extract.runtime.test.ts +2 -0
- package/src/server/services/memory/userMemory/extract.ts +108 -7
- package/src/store/chat/slices/aiChat/actions/conversationLifecycle.ts +5 -19
|
@@ -22,6 +22,26 @@ export enum UserMemoryContextSubjectType {
|
|
|
22
22
|
}
|
|
23
23
|
export const CONTEXT_SUBJECT_TYPES = Object.values(UserMemoryContextSubjectType);
|
|
24
24
|
|
|
25
|
+
export interface UserMemoryAssociatedLocation {
|
|
26
|
+
address?: string | null;
|
|
27
|
+
extra?: Record<string, unknown> | null;
|
|
28
|
+
name?: string | null;
|
|
29
|
+
tags?: string[] | null;
|
|
30
|
+
type?: string | null;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export interface UserMemoryAssociatedObject {
|
|
34
|
+
extra?: Record<string, unknown> | null;
|
|
35
|
+
name?: string;
|
|
36
|
+
type?: string | null;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export interface UserMemoryAssociatedSubject {
|
|
40
|
+
extra?: Record<string, unknown> | null;
|
|
41
|
+
name?: string;
|
|
42
|
+
type?: string | null;
|
|
43
|
+
}
|
|
44
|
+
|
|
25
45
|
export interface UserMemoryContext extends UserMemoryTimestamps {
|
|
26
46
|
associatedObjects:
|
|
27
47
|
| {
|
|
@@ -112,3 +132,35 @@ export type UserMemoryPreferenceWithoutVectors = Omit<
|
|
|
112
132
|
>;
|
|
113
133
|
|
|
114
134
|
export type UserMemoryPreferencesListItem = Omit<UserMemoryPreferenceWithoutVectors, 'suggestions'>;
|
|
135
|
+
|
|
136
|
+
export interface UserMemoryActivity extends UserMemoryTimestamps {
|
|
137
|
+
associatedLocations: UserMemoryAssociatedLocation[] | null;
|
|
138
|
+
associatedObjects: UserMemoryAssociatedObject[] | null;
|
|
139
|
+
associatedSubjects: UserMemoryAssociatedSubject[] | null;
|
|
140
|
+
endsAt: Date | null;
|
|
141
|
+
feedback: string | null;
|
|
142
|
+
feedbackVector: number[] | null;
|
|
143
|
+
id: string;
|
|
144
|
+
metadata: Record<string, unknown> | null;
|
|
145
|
+
narrative: string | null;
|
|
146
|
+
narrativeVector: number[] | null;
|
|
147
|
+
notes: string | null;
|
|
148
|
+
startsAt: Date | null;
|
|
149
|
+
status: string | null;
|
|
150
|
+
tags: string[] | null;
|
|
151
|
+
timezone: string | null;
|
|
152
|
+
type: string | null;
|
|
153
|
+
userId: string | null;
|
|
154
|
+
userMemoryId: string | null;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// eslint-disable-next-line @typescript-eslint/no-empty-interface
|
|
158
|
+
export type UserMemoryActivityWithoutVectors = Omit<
|
|
159
|
+
UserMemoryActivity,
|
|
160
|
+
'narrativeVector' | 'feedbackVector'
|
|
161
|
+
>;
|
|
162
|
+
|
|
163
|
+
export type UserMemoryActivitiesListItem = Omit<
|
|
164
|
+
UserMemoryActivityWithoutVectors,
|
|
165
|
+
'feedback' | 'narrative' | 'notes'
|
|
166
|
+
>;
|
|
@@ -5,6 +5,8 @@ import {
|
|
|
5
5
|
UserMemoryContextsListItem,
|
|
6
6
|
UserMemoryExperiencesListItem,
|
|
7
7
|
UserMemoryPreferencesListItem,
|
|
8
|
+
UserMemoryActivitiesListItem,
|
|
9
|
+
UserMemoryActivityWithoutVectors,
|
|
8
10
|
} from './layers'
|
|
9
11
|
import {
|
|
10
12
|
UserMemoryIdentityWithoutVectors,
|
|
@@ -87,14 +89,30 @@ export interface IdentityMemoryDetail {
|
|
|
87
89
|
sourceType?: MemorySourceType;
|
|
88
90
|
}
|
|
89
91
|
|
|
92
|
+
export interface ActivityMemorySimple {
|
|
93
|
+
activity: UserMemoryActivitiesListItem;
|
|
94
|
+
layer: LayersEnum.Activity;
|
|
95
|
+
memory: UserMemoryListItem;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export interface ActivityMemoryDetail {
|
|
99
|
+
activity: UserMemoryActivityWithoutVectors;
|
|
100
|
+
layer: LayersEnum.Activity;
|
|
101
|
+
memory: UserMemoryWithoutVectors;
|
|
102
|
+
source?: MemorySource;
|
|
103
|
+
sourceType?: MemorySourceType;
|
|
104
|
+
}
|
|
105
|
+
|
|
90
106
|
export type UserMemoryItemSimple =
|
|
91
107
|
| ContextMemorySimple
|
|
92
108
|
| ExperienceMemorySimple
|
|
93
109
|
| IdentityMemorySimple
|
|
94
|
-
| PreferenceMemorySimple
|
|
110
|
+
| PreferenceMemorySimple
|
|
111
|
+
| ActivityMemorySimple;
|
|
95
112
|
|
|
96
113
|
export type UserMemoryDetail =
|
|
97
114
|
| ContextMemoryDetail
|
|
98
115
|
| ExperienceMemoryDetail
|
|
99
116
|
| IdentityMemoryDetail
|
|
100
|
-
| PreferenceMemoryDetail
|
|
117
|
+
| PreferenceMemoryDetail
|
|
118
|
+
| ActivityMemoryDetail;
|
|
@@ -46,6 +46,7 @@ export enum IdentityTypeEnum {
|
|
|
46
46
|
export const IDENTITY_TYPES = Object.values(IdentityTypeEnum);
|
|
47
47
|
|
|
48
48
|
export enum LayersEnum {
|
|
49
|
+
Activity = 'activity',
|
|
49
50
|
Context = 'context',
|
|
50
51
|
Experience = 'experience',
|
|
51
52
|
Identity = 'identity',
|
|
@@ -80,7 +81,6 @@ export const CONTEXT_STATUS = Object.values(ContextStatusEnum);
|
|
|
80
81
|
/**
|
|
81
82
|
* Shared types for memory list queries
|
|
82
83
|
*/
|
|
83
|
-
|
|
84
84
|
export interface BaseListParams {
|
|
85
85
|
order?: 'asc' | 'desc';
|
|
86
86
|
page?: number;
|
|
@@ -106,3 +106,24 @@ export interface BaseListItem {
|
|
|
106
106
|
type: string | null;
|
|
107
107
|
updatedAt: Date;
|
|
108
108
|
}
|
|
109
|
+
|
|
110
|
+
export enum ActivityTypeEnum {
|
|
111
|
+
Appointment = 'appointment',
|
|
112
|
+
Call = 'call',
|
|
113
|
+
Celebration = 'celebration',
|
|
114
|
+
Class = 'class',
|
|
115
|
+
Conference = 'conference',
|
|
116
|
+
Errand = 'errand',
|
|
117
|
+
Event = 'event',
|
|
118
|
+
Exercise = 'exercise',
|
|
119
|
+
Meal = 'meal',
|
|
120
|
+
Meeting = 'meeting',
|
|
121
|
+
Other = 'other',
|
|
122
|
+
ProjectSession = 'project-session',
|
|
123
|
+
Social = 'social',
|
|
124
|
+
Task = 'task',
|
|
125
|
+
Trip = 'trip',
|
|
126
|
+
Workshop = 'workshop',
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
export const ACTIVITY_TYPES = Object.values(ActivityTypeEnum);
|
|
@@ -1,2 +1,10 @@
|
|
|
1
1
|
export type Primitive = number | string | boolean | undefined | null | any[];
|
|
2
|
-
|
|
2
|
+
type OptionalKeys<T> = { [P in keyof T]-?: undefined extends T[P] ? P : never }[keyof T];
|
|
3
|
+
type OptionalField<T, P extends keyof T, K extends keyof T> = T[P] extends Primitive
|
|
4
|
+
? T[P]
|
|
5
|
+
: Optional<T[P], keyof T[P] & K>;
|
|
6
|
+
|
|
7
|
+
export type Optional<T, K extends keyof T> = T extends null | undefined
|
|
8
|
+
? T
|
|
9
|
+
: Partial<Pick<T, K | OptionalKeys<T>>> &
|
|
10
|
+
{ [P in Exclude<keyof T, K | OptionalKeys<T>>]: OptionalField<T, P, K> };
|
package/scripts/prebuild.mts
CHANGED
|
@@ -51,6 +51,7 @@ const checkRequiredEnvVars = () => {
|
|
|
51
51
|
console.error(` 📖 Documentation: ${docUrl}\n`);
|
|
52
52
|
}
|
|
53
53
|
console.error('Please configure these environment variables and redeploy.');
|
|
54
|
+
console.error('\n💡 TIP: If you previously used NEXT_AUTH_SECRET, simply rename it to AUTH_SECRET.');
|
|
54
55
|
console.error('═'.repeat(70) + '\n');
|
|
55
56
|
process.exit(1);
|
|
56
57
|
}
|
|
@@ -30,7 +30,7 @@ const ContextList = memo(() => {
|
|
|
30
30
|
const rawSelectionList = useFileStore(fileChatSelectors.chatContextSelections);
|
|
31
31
|
const showSelectionList = useFileStore(fileChatSelectors.chatContextSelectionHasItem);
|
|
32
32
|
const clearChatContextSelections = useFileStore((s) => s.clearChatContextSelections);
|
|
33
|
-
|
|
33
|
+
console.log(rawSelectionList);
|
|
34
34
|
// Clear selections only when agentId changes (not on initial mount)
|
|
35
35
|
useEffect(() => {
|
|
36
36
|
if (prevAgentIdRef.current !== undefined && prevAgentIdRef.current !== agentId) {
|
|
@@ -110,8 +110,16 @@ const ChatInput = memo<ChatInputProps>(
|
|
|
110
110
|
fileStore.clearChatUploadFileList();
|
|
111
111
|
fileStore.clearChatContextSelections();
|
|
112
112
|
|
|
113
|
+
// Convert ChatContextContent to PageSelection for persistence
|
|
114
|
+
const pageSelections = currentContextList.map((ctx) => ({
|
|
115
|
+
content: ctx.preview || '',
|
|
116
|
+
id: ctx.id,
|
|
117
|
+
pageId: ctx.pageId || '',
|
|
118
|
+
xml: ctx.content,
|
|
119
|
+
}));
|
|
120
|
+
|
|
113
121
|
// Fire and forget - send with captured message
|
|
114
|
-
await sendMessage({
|
|
122
|
+
await sendMessage({ files: currentFileList, message, pageSelections });
|
|
115
123
|
},
|
|
116
124
|
[isAIGenerating, sendMessage],
|
|
117
125
|
);
|
|
@@ -7,13 +7,19 @@ import { type UIChatMessage } from '@/types/index';
|
|
|
7
7
|
import { useMarkdown } from '../useMarkdown';
|
|
8
8
|
import FileListViewer from './FileListViewer';
|
|
9
9
|
import ImageFileListViewer from './ImageFileListViewer';
|
|
10
|
+
import PageSelections from './PageSelections';
|
|
10
11
|
import VideoFileListViewer from './VideoFileListViewer';
|
|
11
12
|
|
|
12
13
|
const UserMessageContent = memo<UIChatMessage>(
|
|
13
|
-
({ id, content, imageList, videoList, fileList }) => {
|
|
14
|
+
({ id, content, imageList, videoList, fileList, metadata }) => {
|
|
14
15
|
const markdownProps = useMarkdown(id);
|
|
16
|
+
const pageSelections = metadata?.pageSelections;
|
|
17
|
+
|
|
15
18
|
return (
|
|
16
19
|
<Flexbox gap={8} id={id}>
|
|
20
|
+
{pageSelections && pageSelections.length > 0 && (
|
|
21
|
+
<PageSelections selections={pageSelections} />
|
|
22
|
+
)}
|
|
17
23
|
{content && <MarkdownMessage {...markdownProps}>{content}</MarkdownMessage>}
|
|
18
24
|
{imageList && imageList?.length > 0 && <ImageFileListViewer items={imageList} />}
|
|
19
25
|
{videoList && videoList?.length > 0 && <VideoFileListViewer items={videoList} />}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { Flexbox } from '@lobehub/ui';
|
|
2
|
+
import { createStaticStyles } from 'antd-style';
|
|
3
|
+
import { memo } from 'react';
|
|
4
|
+
|
|
5
|
+
import type { PageSelection } from '@/types/index';
|
|
6
|
+
|
|
7
|
+
const styles = createStaticStyles(({ css, cssVar }) => ({
|
|
8
|
+
container: css`
|
|
9
|
+
cursor: pointer;
|
|
10
|
+
position: relative;
|
|
11
|
+
border-radius: 8px;
|
|
12
|
+
|
|
13
|
+
:hover {
|
|
14
|
+
background: ${cssVar.colorFillQuaternary};
|
|
15
|
+
}
|
|
16
|
+
`,
|
|
17
|
+
content: css`
|
|
18
|
+
overflow: hidden;
|
|
19
|
+
display: -webkit-box;
|
|
20
|
+
-webkit-box-orient: vertical;
|
|
21
|
+
-webkit-line-clamp: 3;
|
|
22
|
+
|
|
23
|
+
font-size: 12px;
|
|
24
|
+
line-height: 1.5;
|
|
25
|
+
color: ${cssVar.colorTextSecondary};
|
|
26
|
+
white-space: pre-wrap;
|
|
27
|
+
`,
|
|
28
|
+
quote: css`
|
|
29
|
+
inset-block-start: 2px;
|
|
30
|
+
inset-inline-start: 0;
|
|
31
|
+
|
|
32
|
+
font-family: Georgia, serif;
|
|
33
|
+
font-size: 28px;
|
|
34
|
+
line-height: 1;
|
|
35
|
+
color: ${cssVar.colorTextQuaternary};
|
|
36
|
+
`,
|
|
37
|
+
wrapper: css``,
|
|
38
|
+
}));
|
|
39
|
+
|
|
40
|
+
interface PageSelectionsProps {
|
|
41
|
+
selections: PageSelection[];
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const PageSelections = memo<PageSelectionsProps>(({ selections }) => {
|
|
45
|
+
if (!selections || selections.length === 0) return null;
|
|
46
|
+
|
|
47
|
+
return (
|
|
48
|
+
<Flexbox gap={8}>
|
|
49
|
+
{selections.map((selection) => (
|
|
50
|
+
<Flexbox className={styles.container} key={selection.id}>
|
|
51
|
+
<Flexbox className={styles.wrapper} gap={4} horizontal padding={4}>
|
|
52
|
+
{/* eslint-disable-next-line react/no-unescaped-entities */}
|
|
53
|
+
<span className={styles.quote}>"</span>
|
|
54
|
+
<div className={styles.content}>{selection.content}</div>
|
|
55
|
+
</Flexbox>
|
|
56
|
+
</Flexbox>
|
|
57
|
+
))}
|
|
58
|
+
</Flexbox>
|
|
59
|
+
);
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
export default PageSelections;
|
|
@@ -12,6 +12,8 @@ import { useTranslation } from 'react-i18next';
|
|
|
12
12
|
import { useFileStore } from '@/store/file';
|
|
13
13
|
import { useGlobalStore } from '@/store/global';
|
|
14
14
|
|
|
15
|
+
import { usePageEditorStore } from '../store';
|
|
16
|
+
|
|
15
17
|
const styles = createStaticStyles(({ css }) => ({
|
|
16
18
|
askCopilot: css`
|
|
17
19
|
border-radius: 6px;
|
|
@@ -26,6 +28,7 @@ const styles = createStaticStyles(({ css }) => ({
|
|
|
26
28
|
export const useAskCopilotItem = (editor: IEditor | undefined): ChatInputActionsProps['items'] => {
|
|
27
29
|
const { t } = useTranslation('common');
|
|
28
30
|
const addSelectionContext = useFileStore((s) => s.addChatContextSelection);
|
|
31
|
+
const pageId = usePageEditorStore((s) => s.documentId);
|
|
29
32
|
|
|
30
33
|
return useMemo(() => {
|
|
31
34
|
if (!editor) return [];
|
|
@@ -60,6 +63,7 @@ export const useAskCopilotItem = (editor: IEditor | undefined): ChatInputActions
|
|
|
60
63
|
content,
|
|
61
64
|
format,
|
|
62
65
|
id: `selection-${nanoid(6)}`,
|
|
66
|
+
pageId,
|
|
63
67
|
preview,
|
|
64
68
|
title: 'Selection',
|
|
65
69
|
type: 'text',
|
|
@@ -95,5 +99,5 @@ export const useAskCopilotItem = (editor: IEditor | undefined): ChatInputActions
|
|
|
95
99
|
onClick: () => {},
|
|
96
100
|
},
|
|
97
101
|
];
|
|
98
|
-
}, [addSelectionContext, editor, t]);
|
|
102
|
+
}, [addSelectionContext, editor, pageId, t]);
|
|
99
103
|
};
|
|
@@ -234,8 +234,9 @@ export default {
|
|
|
234
234
|
'operation.sendMessage': 'Sending message',
|
|
235
235
|
'owner': 'Group owner',
|
|
236
236
|
'pageCopilot.title': 'Page Agent',
|
|
237
|
-
'pageCopilot.welcome':
|
|
238
|
-
|
|
237
|
+
'pageCopilot.welcome': `**Clearer, sharper writing**\n\nDraft, rewrite, or polish—tell me your intent and I'll refine the rest.`,
|
|
238
|
+
'pageSelection.lines': 'Lines {{start}}-{{end}}',
|
|
239
|
+
'pageSelection.reference': 'Selected Text',
|
|
239
240
|
'pin': 'Pin',
|
|
240
241
|
'pinOff': 'Unpin',
|
|
241
242
|
'prompts.summaryExpert':
|
|
@@ -8,7 +8,13 @@ import {
|
|
|
8
8
|
type MemoryLayerExtractorPublicConfig,
|
|
9
9
|
} from '@/types/serverConfig';
|
|
10
10
|
|
|
11
|
-
const MEMORY_LAYERS: GlobalMemoryLayer[] = [
|
|
11
|
+
const MEMORY_LAYERS: GlobalMemoryLayer[] = [
|
|
12
|
+
'activity',
|
|
13
|
+
'context',
|
|
14
|
+
'experience',
|
|
15
|
+
'identity',
|
|
16
|
+
'preference',
|
|
17
|
+
];
|
|
12
18
|
|
|
13
19
|
const parseTokenLimitEnv = (value?: string) => {
|
|
14
20
|
if (value === undefined) return undefined;
|
|
@@ -123,11 +123,18 @@ export const aiChatRouter = router({
|
|
|
123
123
|
|
|
124
124
|
// create user message
|
|
125
125
|
log('creating user message with content length: %d', input.newUserMessage.content.length);
|
|
126
|
+
|
|
127
|
+
// Build user message metadata with pageSelections if present
|
|
128
|
+
const userMessageMetadata = input.newUserMessage.pageSelections?.length
|
|
129
|
+
? { pageSelections: input.newUserMessage.pageSelections }
|
|
130
|
+
: undefined;
|
|
131
|
+
|
|
126
132
|
const userMessageItem = await ctx.messageModel.create({
|
|
127
133
|
agentId: input.agentId,
|
|
128
134
|
content: input.newUserMessage.content,
|
|
129
135
|
files: input.newUserMessage.files,
|
|
130
136
|
groupId: input.groupId,
|
|
137
|
+
metadata: userMessageMetadata,
|
|
131
138
|
parentId: input.newUserMessage.parentId,
|
|
132
139
|
role: 'user',
|
|
133
140
|
sessionId,
|
|
@@ -26,6 +26,7 @@ const createExecutor = (privateOverrides?: Partial<MemoryExtractionPrivateConfig
|
|
|
26
26
|
agentLayerExtractor: {
|
|
27
27
|
contextLimit: 2048,
|
|
28
28
|
layers: {
|
|
29
|
+
activity: 'layer-act',
|
|
29
30
|
context: 'layer-ctx',
|
|
30
31
|
experience: 'layer-exp',
|
|
31
32
|
identity: 'layer-id',
|
|
@@ -38,6 +39,7 @@ const createExecutor = (privateOverrides?: Partial<MemoryExtractionPrivateConfig
|
|
|
38
39
|
embedding: { model: 'embed-1', provider: 'provider-e' },
|
|
39
40
|
featureFlags: { enableBenchmarkLoCoMo: false },
|
|
40
41
|
observabilityS3: { enabled: false },
|
|
42
|
+
webhookHeaders: {},
|
|
41
43
|
};
|
|
42
44
|
|
|
43
45
|
const serverConfig = {
|
|
@@ -90,6 +90,7 @@ const LAYER_ALIAS = new Set<LayersEnum>([
|
|
|
90
90
|
]);
|
|
91
91
|
|
|
92
92
|
const LAYER_LABEL_MAP: Record<LayersEnum, string> = {
|
|
93
|
+
[LayersEnum.Activity]: 'activities',
|
|
93
94
|
[LayersEnum.Context]: 'contexts',
|
|
94
95
|
[LayersEnum.Experience]: 'experiences',
|
|
95
96
|
[LayersEnum.Identity]: 'identities',
|
|
@@ -267,6 +268,7 @@ const resolveLayerModels = (
|
|
|
267
268
|
layers: Partial<Record<GlobalMemoryLayer, string>> | undefined,
|
|
268
269
|
fallback: Record<GlobalMemoryLayer, string>,
|
|
269
270
|
): Record<LayersEnum, string> => ({
|
|
271
|
+
activity: layers?.activity ?? fallback.activity,
|
|
270
272
|
context: layers?.context ?? fallback.context,
|
|
271
273
|
experience: layers?.experience ?? fallback.experience,
|
|
272
274
|
identity: layers?.identity ?? fallback.identity,
|
|
@@ -569,6 +571,79 @@ export class MemoryExtractionExecutor {
|
|
|
569
571
|
});
|
|
570
572
|
}
|
|
571
573
|
|
|
574
|
+
async persistActivityMemories(
|
|
575
|
+
job: MemoryExtractionJob,
|
|
576
|
+
messageIds: string[],
|
|
577
|
+
result: NonNullable<MemoryExtractionResult['outputs']['activity']>['data'],
|
|
578
|
+
runtime: ModelRuntime,
|
|
579
|
+
model: string,
|
|
580
|
+
tokenLimit: number | undefined,
|
|
581
|
+
db: Awaited<ReturnType<typeof getServerDB>>,
|
|
582
|
+
) {
|
|
583
|
+
const insertedIds: string[] = [];
|
|
584
|
+
const userMemoryModel = new UserMemoryModel(db, job.userId);
|
|
585
|
+
|
|
586
|
+
for (const item of result?.memories ?? []) {
|
|
587
|
+
const activityTags = item.withActivity?.tags ?? item.tags;
|
|
588
|
+
const associatedObjects = UserMemoryModel.parseAssociatedObjects(
|
|
589
|
+
item.withActivity?.associatedObjects,
|
|
590
|
+
);
|
|
591
|
+
const associatedSubjects = UserMemoryModel.parseAssociatedSubjects(
|
|
592
|
+
item.withActivity?.associatedSubjects,
|
|
593
|
+
);
|
|
594
|
+
const associatedLocations = UserMemoryModel.parseAssociatedLocations(
|
|
595
|
+
item.withActivity?.associatedLocations,
|
|
596
|
+
);
|
|
597
|
+
const [summaryVector, detailsVector, narrativeVector, feedbackVector] =
|
|
598
|
+
await this.generateEmbeddings(
|
|
599
|
+
runtime,
|
|
600
|
+
model,
|
|
601
|
+
[item.summary, item.details, item.withActivity?.narrative, item.withActivity?.feedback],
|
|
602
|
+
tokenLimit,
|
|
603
|
+
);
|
|
604
|
+
const baseMetadata = this.buildBaseMetadata(
|
|
605
|
+
job,
|
|
606
|
+
messageIds,
|
|
607
|
+
LayersEnum.Activity,
|
|
608
|
+
activityTags,
|
|
609
|
+
);
|
|
610
|
+
|
|
611
|
+
const { memory } = await userMemoryModel.createActivityMemory({
|
|
612
|
+
activity: {
|
|
613
|
+
associatedLocations: associatedLocations.length > 0 ? associatedLocations : [],
|
|
614
|
+
associatedObjects: associatedObjects.length > 0 ? associatedObjects : [],
|
|
615
|
+
associatedSubjects: associatedSubjects.length > 0 ? associatedSubjects : [],
|
|
616
|
+
capturedAt: job.sourceUpdatedAt,
|
|
617
|
+
endsAt: UserMemoryModel.parseDateFromString(item.withActivity?.endsAt),
|
|
618
|
+
feedback: item.withActivity?.feedback ?? null,
|
|
619
|
+
feedbackVector: feedbackVector ?? null,
|
|
620
|
+
metadata: baseMetadata,
|
|
621
|
+
narrative: item.withActivity?.narrative ?? null,
|
|
622
|
+
narrativeVector: narrativeVector ?? null,
|
|
623
|
+
notes: item.withActivity?.notes ?? null,
|
|
624
|
+
startsAt: UserMemoryModel.parseDateFromString(item.withActivity?.startsAt),
|
|
625
|
+
status: item.withActivity?.status ?? 'pending',
|
|
626
|
+
tags: activityTags ?? null,
|
|
627
|
+
timezone: item.withActivity?.timezone ?? null,
|
|
628
|
+
type: item.withActivity?.type ?? 'other',
|
|
629
|
+
},
|
|
630
|
+
capturedAt: job.sourceUpdatedAt,
|
|
631
|
+
details: item.details ?? '',
|
|
632
|
+
detailsEmbedding: detailsVector ?? undefined,
|
|
633
|
+
memoryCategory: item.memoryCategory ?? null,
|
|
634
|
+
memoryLayer: (item.memoryLayer as LayersEnum) ?? LayersEnum.Activity,
|
|
635
|
+
memoryType: (item.memoryType as TypesEnum) ?? TypesEnum.Activity,
|
|
636
|
+
summary: item.summary ?? '',
|
|
637
|
+
summaryEmbedding: summaryVector ?? undefined,
|
|
638
|
+
title: item.title ?? '',
|
|
639
|
+
});
|
|
640
|
+
|
|
641
|
+
insertedIds.push(memory.id);
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
return insertedIds;
|
|
645
|
+
}
|
|
646
|
+
|
|
572
647
|
async persistContextMemories(
|
|
573
648
|
job: MemoryExtractionJob,
|
|
574
649
|
messageIds: string[],
|
|
@@ -601,7 +676,7 @@ export class MemoryExtractionExecutor {
|
|
|
601
676
|
associatedObjects: UserMemoryModel.parseAssociatedObjects(
|
|
602
677
|
item.withContext?.associatedObjects,
|
|
603
678
|
),
|
|
604
|
-
associatedSubjects: UserMemoryModel.
|
|
679
|
+
associatedSubjects: UserMemoryModel.parseAssociatedSubjects(
|
|
605
680
|
item.withContext?.associatedSubjects,
|
|
606
681
|
),
|
|
607
682
|
capturedAt: job.sourceUpdatedAt,
|
|
@@ -923,13 +998,14 @@ export class MemoryExtractionExecutor {
|
|
|
923
998
|
if (vector) {
|
|
924
999
|
const retrieved = await userMemoryModel.searchWithEmbedding({
|
|
925
1000
|
embedding: vector,
|
|
926
|
-
limits: { contexts: topK, experiences: topK, preferences: topK },
|
|
1001
|
+
limits: { activities: topK, contexts: topK, experiences: topK, preferences: topK },
|
|
927
1002
|
});
|
|
928
1003
|
|
|
929
1004
|
return retrieved;
|
|
930
1005
|
}
|
|
931
1006
|
|
|
932
1007
|
return {
|
|
1008
|
+
activities: [],
|
|
933
1009
|
contexts: [],
|
|
934
1010
|
experiences: [],
|
|
935
1011
|
preferences: [],
|
|
@@ -1563,6 +1639,24 @@ export class MemoryExtractionExecutor {
|
|
|
1563
1639
|
);
|
|
1564
1640
|
};
|
|
1565
1641
|
|
|
1642
|
+
const activityOutput = extraction.outputs.activity;
|
|
1643
|
+
if (activityOutput?.error) {
|
|
1644
|
+
appendError(LayersEnum.Activity, 'extract', activityOutput.error);
|
|
1645
|
+
}
|
|
1646
|
+
if (activityOutput?.data) {
|
|
1647
|
+
await persistWithSpan(LayersEnum.Activity, () =>
|
|
1648
|
+
this.persistActivityMemories(
|
|
1649
|
+
job,
|
|
1650
|
+
messageIds,
|
|
1651
|
+
activityOutput.data,
|
|
1652
|
+
runtimes.embeddings,
|
|
1653
|
+
this.modelConfig.embeddingsModel,
|
|
1654
|
+
this.embeddingContextLimit,
|
|
1655
|
+
db,
|
|
1656
|
+
),
|
|
1657
|
+
);
|
|
1658
|
+
}
|
|
1659
|
+
|
|
1566
1660
|
const contextOutput = extraction.outputs.context;
|
|
1567
1661
|
if (contextOutput?.error) {
|
|
1568
1662
|
appendError(LayersEnum.Context, 'extract', contextOutput.error);
|
|
@@ -1636,7 +1730,7 @@ export class MemoryExtractionExecutor {
|
|
|
1636
1730
|
}
|
|
1637
1731
|
|
|
1638
1732
|
if (errors.length) {
|
|
1639
|
-
const detail = errors.map((error) => error.message).join('; ');
|
|
1733
|
+
const detail = errors.map((error) => `${error.message}${error.cause ? `: ${error.cause}` : ''}`).join('; ');
|
|
1640
1734
|
throw new AggregateError(errors, `Memory extraction encountered layer errors: ${detail}`);
|
|
1641
1735
|
}
|
|
1642
1736
|
|
|
@@ -1825,14 +1919,21 @@ export class MemoryExtractionExecutor {
|
|
|
1825
1919
|
userId: params.userId,
|
|
1826
1920
|
});
|
|
1827
1921
|
|
|
1922
|
+
const latestCreatedAt = params.parts.reduce<Date | undefined>((latest, part) => {
|
|
1923
|
+
if (!part.createdAt) return latest;
|
|
1924
|
+
|
|
1925
|
+
const date = new Date(part.createdAt);
|
|
1926
|
+
if (Number.isNaN(date.getTime())) return latest;
|
|
1927
|
+
|
|
1928
|
+
return !latest || date > latest ? date : latest;
|
|
1929
|
+
}, undefined);
|
|
1930
|
+
|
|
1828
1931
|
extractionJob = {
|
|
1829
1932
|
force: params.forceAll ?? true,
|
|
1830
1933
|
layers: params.layers,
|
|
1831
1934
|
source: params.source,
|
|
1832
1935
|
sourceId: params.sourceId,
|
|
1833
|
-
sourceUpdatedAt:
|
|
1834
|
-
? new Date(params.parts.at(-1)!.createdAt as Date)
|
|
1835
|
-
: new Date(),
|
|
1936
|
+
sourceUpdatedAt: latestCreatedAt ?? new Date(),
|
|
1836
1937
|
userId: params.userId,
|
|
1837
1938
|
};
|
|
1838
1939
|
|
|
@@ -1898,7 +1999,7 @@ export class MemoryExtractionExecutor {
|
|
|
1898
1999
|
language,
|
|
1899
2000
|
retrievedContexts: [trimmedContext],
|
|
1900
2001
|
retrievedIdentitiesContext: undefined,
|
|
1901
|
-
sessionDate: new Date().toISOString(),
|
|
2002
|
+
sessionDate: (latestCreatedAt ?? new Date()).toISOString(),
|
|
1902
2003
|
topK: 10,
|
|
1903
2004
|
username:
|
|
1904
2005
|
userState.fullName || `${userState.firstName} ${userState.lastName}`.trim() || 'User',
|
|
@@ -77,11 +77,11 @@ export const conversationLifecycle: StateCreator<
|
|
|
77
77
|
sendMessage: async ({
|
|
78
78
|
message,
|
|
79
79
|
files,
|
|
80
|
-
contexts,
|
|
81
80
|
onlyAddUserMessage,
|
|
82
81
|
context,
|
|
83
82
|
messages: inputMessages,
|
|
84
83
|
parentId: inputParentId,
|
|
84
|
+
pageSelections,
|
|
85
85
|
}) => {
|
|
86
86
|
const { internal_execAgentRuntime, mainInputEditor } = get();
|
|
87
87
|
|
|
@@ -186,6 +186,8 @@ export const conversationLifecycle: StateCreator<
|
|
|
186
186
|
threadId: operationContext.threadId ?? undefined,
|
|
187
187
|
imageList: tempImages.length > 0 ? tempImages : undefined,
|
|
188
188
|
videoList: tempVideos.length > 0 ? tempVideos : undefined,
|
|
189
|
+
// Pass pageSelections metadata for immediate display
|
|
190
|
+
metadata: pageSelections?.length ? { pageSelections } : undefined,
|
|
189
191
|
},
|
|
190
192
|
{ operationId, tempMessageId: tempId },
|
|
191
193
|
);
|
|
@@ -222,7 +224,7 @@ export const conversationLifecycle: StateCreator<
|
|
|
222
224
|
const topicId = operationContext.topicId;
|
|
223
225
|
data = await aiChatService.sendMessageInServer(
|
|
224
226
|
{
|
|
225
|
-
newUserMessage: { content: message, files: fileIdList, parentId },
|
|
227
|
+
newUserMessage: { content: message, files: fileIdList, pageSelections, parentId },
|
|
226
228
|
// if there is topicId,then add topicId to message
|
|
227
229
|
topicId: topicId ?? undefined,
|
|
228
230
|
threadId: operationContext.threadId ?? undefined,
|
|
@@ -372,26 +374,10 @@ export const conversationLifecycle: StateCreator<
|
|
|
372
374
|
messageMapKey(execContext),
|
|
373
375
|
)(get());
|
|
374
376
|
|
|
375
|
-
const contextMessages =
|
|
376
|
-
contexts?.map((item, index) => {
|
|
377
|
-
const now = Date.now();
|
|
378
|
-
const title = item.title ? `${item.title}\n` : '';
|
|
379
|
-
return {
|
|
380
|
-
content: `Context ${index + 1}:\n${title}${item.content}`,
|
|
381
|
-
createdAt: now,
|
|
382
|
-
id: `ctx_${tempId}_${index}`,
|
|
383
|
-
role: 'system' as const,
|
|
384
|
-
updatedAt: now,
|
|
385
|
-
};
|
|
386
|
-
}) ?? [];
|
|
387
|
-
|
|
388
|
-
const runtimeMessages =
|
|
389
|
-
contextMessages.length > 0 ? [...displayMessages, ...contextMessages] : displayMessages;
|
|
390
|
-
|
|
391
377
|
try {
|
|
392
378
|
await internal_execAgentRuntime({
|
|
393
379
|
context: execContext,
|
|
394
|
-
messages:
|
|
380
|
+
messages: displayMessages,
|
|
395
381
|
parentMessageId: data.assistantMessageId,
|
|
396
382
|
parentMessageType: 'assistant',
|
|
397
383
|
parentOperationId: operationId, // Pass as parent operation
|