@lobehub/lobehub 2.0.0-next.360 → 2.0.0-next.362

package/packages/context-engine/src/providers/__tests__/PageSelectionsInjector.test.ts

@@ -0,0 +1,333 @@
+import { describe, expect, it } from 'vitest';
+
+import type { PipelineContext } from '../../types';
+import { PageSelectionsInjector } from '../PageSelectionsInjector';
+
+describe('PageSelectionsInjector', () => {
+  const createContext = (messages: any[] = []): PipelineContext => ({
+    initialState: {
+      messages: [],
+      model: 'test-model',
+      provider: 'test-provider',
+    },
+    isAborted: false,
+    messages,
+    metadata: {
+      maxTokens: 4000,
+      model: 'test-model',
+    },
+  });
+
+  const createPageSelection = (id: string, xmlContent: string, pageId = 'page-1') => ({
+    content: xmlContent, // preview content
+    id,
+    pageId,
+    xml: xmlContent, // actual content used by formatPageSelections
+  });
+
+  describe('enabled/disabled', () => {
+    it('should skip injection when disabled', async () => {
+      const injector = new PageSelectionsInjector({ enabled: false });
+
+      const context = createContext([
+        {
+          content: 'Question',
+          metadata: {
+            pageSelections: [createPageSelection('sel-1', 'Selected text')],
+          },
+          role: 'user',
+        },
+      ]);
+
+      const result = await injector.process(context);
+
+      expect(result.messages[0].content).toBe('Question');
+      expect(result.metadata.PageSelectionsInjectorInjectedCount).toBeUndefined();
+    });
+
+    it('should inject when enabled', async () => {
+      const injector = new PageSelectionsInjector({ enabled: true });
+
+      const context = createContext([
+        {
+          content: 'Question',
+          metadata: {
+            pageSelections: [createPageSelection('sel-1', 'Selected text')],
+          },
+          role: 'user',
+        },
+      ]);
+
+      const result = await injector.process(context);
+
+      expect(result.messages[0].content).toContain('Question');
+      expect(result.messages[0].content).toContain('<user_page_selections>');
+      expect(result.messages[0].content).toContain('Selected text');
+    });
+  });
+
+  describe('injection to every user message', () => {
+    it('should inject selections to each user message that has them', async () => {
+      const injector = new PageSelectionsInjector({ enabled: true });
+
+      const context = createContext([
+        {
+          content: 'First question',
+          metadata: {
+            pageSelections: [createPageSelection('sel-1', 'First selection')],
+          },
+          role: 'user',
+        },
+        { content: 'First answer', role: 'assistant' },
+        {
+          content: 'Second question',
+          metadata: {
+            pageSelections: [createPageSelection('sel-2', 'Second selection')],
+          },
+          role: 'user',
+        },
+        { content: 'Second answer', role: 'assistant' },
+        {
+          content: 'Third question without selection',
+          role: 'user',
+        },
+      ]);
+
+      const result = await injector.process(context);
+
+      // First user message should have first selection
+      expect(result.messages[0].content).toContain('First question');
+      expect(result.messages[0].content).toContain('First selection');
+      expect(result.messages[0].content).toContain('<user_page_selections>');
+
+      // Second user message should have second selection
+      expect(result.messages[2].content).toContain('Second question');
+      expect(result.messages[2].content).toContain('Second selection');
+      expect(result.messages[2].content).toContain('<user_page_selections>');
+
+      // Third user message should NOT have injection (no selections)
+      expect(result.messages[4].content).toBe('Third question without selection');
+
+      // Assistant messages should be unchanged
+      expect(result.messages[1].content).toBe('First answer');
+      expect(result.messages[3].content).toBe('Second answer');
+
+      // Metadata should show 2 injections
+      expect(result.metadata.PageSelectionsInjectorInjectedCount).toBe(2);
+    });
+
+    it('should skip user messages without pageSelections', async () => {
+      const injector = new PageSelectionsInjector({ enabled: true });
+
+      const context = createContext([
+        { content: 'No selections here', role: 'user' },
+        { content: 'Answer', role: 'assistant' },
+        {
+          content: 'With selections',
+          metadata: {
+            pageSelections: [createPageSelection('sel-1', 'Some text')],
+          },
+          role: 'user',
+        },
+      ]);
+
+      const result = await injector.process(context);
+
+      expect(result.messages[0].content).toBe('No selections here');
+      expect(result.messages[2].content).toContain('With selections');
+      expect(result.messages[2].content).toContain('Some text');
+    });
+
+    it('should skip user messages with empty pageSelections array', async () => {
+      const injector = new PageSelectionsInjector({ enabled: true });
+
+      const context = createContext([
+        {
+          content: 'Empty selections',
+          metadata: { pageSelections: [] },
+          role: 'user',
+        },
+      ]);
+
+      const result = await injector.process(context);
+
+      expect(result.messages[0].content).toBe('Empty selections');
+    });
+  });
+
+  describe('SYSTEM CONTEXT wrapper', () => {
+    it('should wrap selection content with SYSTEM CONTEXT markers', async () => {
+      const injector = new PageSelectionsInjector({ enabled: true });
+
+      const context = createContext([
+        {
+          content: 'Question',
+          metadata: {
+            pageSelections: [createPageSelection('sel-1', 'Selected text')],
+          },
+          role: 'user',
+        },
+      ]);
+
+      const result = await injector.process(context);
+      const content = result.messages[0].content as string;
+
+      expect(content).toContain('<!-- SYSTEM CONTEXT (NOT PART OF USER QUERY) -->');
+      expect(content).toContain('<context.instruction>');
+      expect(content).toContain('<user_page_selections>');
+      expect(content).toContain('</user_page_selections>');
+      expect(content).toContain('<!-- END SYSTEM CONTEXT -->');
+    });
+
+    it('should have only one SYSTEM CONTEXT wrapper per message even with multiple selections', async () => {
+      const injector = new PageSelectionsInjector({ enabled: true });
+
+      const context = createContext([
+        {
+          content: 'Question',
+          metadata: {
+            pageSelections: [
+              createPageSelection('sel-1', 'First selection'),
+              createPageSelection('sel-2', 'Second selection'),
+            ],
+          },
+          role: 'user',
+        },
+      ]);
+
+      const result = await injector.process(context);
+      const content = result.messages[0].content as string;
+
+      const startCount = (content.match(/<!-- SYSTEM CONTEXT/g) || []).length;
+      const endCount = (content.match(/<!-- END SYSTEM CONTEXT/g) || []).length;
+
+      expect(startCount).toBe(1);
+      expect(endCount).toBe(1);
+    });
+
+    it('should create separate SYSTEM CONTEXT wrappers for each user message', async () => {
+      const injector = new PageSelectionsInjector({ enabled: true });
+
+      const context = createContext([
+        {
+          content: 'First question',
+          metadata: {
+            pageSelections: [createPageSelection('sel-1', 'First selection')],
+          },
+          role: 'user',
+        },
+        { content: 'Answer', role: 'assistant' },
+        {
+          content: 'Second question',
+          metadata: {
+            pageSelections: [createPageSelection('sel-2', 'Second selection')],
+          },
+          role: 'user',
+        },
+      ]);
+
+      const result = await injector.process(context);
+
+      // Each user message should have its own SYSTEM CONTEXT wrapper
+      const firstContent = result.messages[0].content as string;
+      const secondContent = result.messages[2].content as string;
+
+      expect(firstContent).toContain('<!-- SYSTEM CONTEXT');
+      expect(firstContent).toContain('First selection');
+
+      expect(secondContent).toContain('<!-- SYSTEM CONTEXT');
+      expect(secondContent).toContain('Second selection');
+    });
+  });
+
+  describe('multimodal messages', () => {
+    it('should handle array content with text parts', async () => {
+      const injector = new PageSelectionsInjector({ enabled: true });
+
+      const context = createContext([
+        {
+          content: [
+            { text: 'Question with image', type: 'text' },
+            { image_url: { url: 'http://example.com/img.png' }, type: 'image_url' },
+          ],
+          metadata: {
+            pageSelections: [createPageSelection('sel-1', 'Selected text')],
+          },
+          role: 'user',
+        },
+      ]);
+
+      const result = await injector.process(context);
+
+      expect(result.messages[0].content[0].text).toContain('Question with image');
+      expect(result.messages[0].content[0].text).toContain('Selected text');
+      expect(result.messages[0].content[0].text).toContain('<user_page_selections>');
+      expect(result.messages[0].content[1]).toEqual({
+        image_url: { url: 'http://example.com/img.png' },
+        type: 'image_url',
+      });
+    });
+  });
+
+  describe('integration with PageEditorContextInjector', () => {
+    it('should create wrapper that PageEditorContextInjector can reuse', async () => {
+      const injector = new PageSelectionsInjector({ enabled: true });
+
+      const context = createContext([
+        {
+          content: 'Question about the page',
+          metadata: {
+            pageSelections: [createPageSelection('sel-1', 'Selected paragraph')],
+          },
+          role: 'user',
+        },
+      ]);
+
+      const result = await injector.process(context);
+      const content = result.messages[0].content as string;
+
+      // Verify the wrapper structure is correct for reuse
+      expect(content).toContain('<!-- SYSTEM CONTEXT (NOT PART OF USER QUERY) -->');
+      expect(content).toContain('<context.instruction>');
+      expect(content).toContain('<!-- END SYSTEM CONTEXT -->');
+
+      // Verify the content is in the right position (between instruction and end marker)
+      const instructionIndex = content.indexOf('</context.instruction>');
+      const selectionsIndex = content.indexOf('<user_page_selections>');
+      const endIndex = content.indexOf('<!-- END SYSTEM CONTEXT -->');
+
+      expect(instructionIndex).toBeLessThan(selectionsIndex);
+      expect(selectionsIndex).toBeLessThan(endIndex);
+    });
+  });
+
+  describe('metadata', () => {
+    it('should set metadata when injections are made', async () => {
+      const injector = new PageSelectionsInjector({ enabled: true });
+
+      const context = createContext([
+        {
+          content: 'Question',
+          metadata: {
+            pageSelections: [createPageSelection('sel-1', 'Text')],
+          },
+          role: 'user',
+        },
+      ]);
+
+      const result = await injector.process(context);
+
+      expect(result.metadata.PageSelectionsInjectorInjectedCount).toBe(1);
+    });
+
+    it('should not set metadata when no injections are made', async () => {
+      const injector = new PageSelectionsInjector({ enabled: true });
+
+      const context = createContext([{ content: 'No selections', role: 'user' }]);
+
+      const result = await injector.process(context);
+
+      expect(result.metadata.PageSelectionsInjectorInjectedCount).toBeUndefined();
+    });
+  });
+});

package/packages/context-engine/src/providers/index.ts

@@ -7,6 +7,7 @@ export { GTDTodoInjector } from './GTDTodoInjector';
 export { HistorySummaryProvider } from './HistorySummary';
 export { KnowledgeInjector } from './KnowledgeInjector';
 export { PageEditorContextInjector } from './PageEditorContextInjector';
+export { PageSelectionsInjector } from './PageSelectionsInjector';
 export { SystemRoleInjector } from './SystemRoleInjector';
 export { ToolSystemRoleProvider } from './ToolSystemRole';
 export { UserMemoryInjector } from './UserMemoryInjector';
@@ -27,11 +28,12 @@ export type {
   GroupContextInjectorConfig,
   GroupMemberInfo as GroupContextMemberInfo,
 } from './GroupContextInjector';
-export type { HistorySummaryConfig } from './HistorySummary';
 export type { GTDPlan, GTDPlanInjectorConfig } from './GTDPlanInjector';
 export type { GTDTodoInjectorConfig, GTDTodoItem, GTDTodoList } from './GTDTodoInjector';
+export type { HistorySummaryConfig } from './HistorySummary';
 export type { KnowledgeInjectorConfig } from './KnowledgeInjector';
 export type { PageEditorContextInjectorConfig } from './PageEditorContextInjector';
+export type { PageSelectionsInjectorConfig } from './PageSelectionsInjector';
 export type { SystemRoleInjectorConfig } from './SystemRoleInjector';
 export type { ToolSystemRoleConfig } from './ToolSystemRole';
 export type { UserMemoryInjectorConfig } from './UserMemoryInjector';

package/packages/database/src/models/userMemory/model.ts

@@ -1,5 +1,6 @@
 import { AssociatedObjectSchema } from '@lobechat/memory-user-memory';
 import {
+  ActivityTypeEnum,
   IdentityTypeEnum,
   LayersEnum,
   MemorySourceType,
@@ -40,7 +41,10 @@ import {
   UserMemoryIdentity,
   UserMemoryItem,
   UserMemoryPreference,
+  UserMemoryActivitiesWithoutVectors,
+  UserMemoryActivity,
   userMemories,
+  userMemoriesActivities,
   userMemoriesContexts,
   userMemoriesExperiences,
   userMemoriesIdentities,
@@ -104,6 +108,16 @@ export interface CreateUserMemoryContextParams extends BaseCreateUserMemoryParam
   >;
 }
 
+export interface CreateUserMemoryActivityParams extends BaseCreateUserMemoryParams {
+  activity: Optional<
+    Omit<
+      UserMemoryActivity,
+      'id' | 'userId' | 'createdAt' | 'updatedAt' | 'accessedAt' | 'userMemoryId'
+    >,
+    'capturedAt'
+  >;
+}
+
 export interface CreateUserMemoryExperienceParams extends BaseCreateUserMemoryParams {
   experience: Optional<
     Omit<
@@ -135,6 +149,7 @@ export interface CreateUserMemoryPreferenceParams extends BaseCreateUserMemoryPa
 }
 
 export type CreateUserMemoryParams =
+  | CreateUserMemoryActivityParams
   | CreateUserMemoryContextParams
   | CreateUserMemoryExperienceParams
   | CreateUserMemoryIdentityParams
@@ -143,7 +158,7 @@ export type CreateUserMemoryParams =
 export interface SearchUserMemoryParams {
   embedding?: number[];
   limit?: number;
-  limits?: Partial<Record<'contexts' | 'experiences' | 'preferences', number>>;
+  limits?: Partial<Record<'activities' | 'contexts' | 'experiences' | 'preferences', number>>;
   memoryCategory?: string;
   memoryType?: string;
   query?: string;
@@ -151,12 +166,13 @@ export interface SearchUserMemoryParams {
 
 export interface SearchUserMemoryWithEmbeddingParams {
   embedding?: number[];
-  limits?: Partial<Record<'contexts' | 'experiences' | 'preferences', number>>;
+  limits?: Partial<Record<'activities' | 'contexts' | 'experiences' | 'preferences', number>>;
   memoryCategory?: string;
   memoryType?: string;
 }
 
 export interface UserMemorySearchAggregatedResult {
+  activities: UserMemoryActivitiesWithoutVectors[];
   contexts: UserMemoryContextWithoutVectors[];
   experiences: UserMemoryExperienceWithoutVectors[];
   preferences: UserMemoryPreferenceWithoutVectors[];
@@ -380,12 +396,67 @@ export class UserMemoryModel {
         const extra = JSON.parse(parsed.data.extra || '{}');
         parsed.data.extra = extra;
         associations.push(parsed.data);
+        return;
+      }
+
+      if (
+        item &&
+        typeof item === 'object' &&
+        'name' in item &&
+        typeof (item as any).name === 'string'
+      ) {
+        associations.push({ name: (item as any).name });
       }
     });
 
     return associations.length > 0 ? associations : [];
   }
 
+  static parseAssociatedLocations(
+    value?:
+      | {
+          address?: unknown;
+          name?: unknown;
+          tags?: unknown;
+          type?: unknown;
+        }[]
+      | Record<string, unknown>,
+  ) {
+    if (!value) return [];
+
+    const raw = Array.isArray(value) ? value : [value];
+    const locations: {
+      address?: string;
+      name?: string;
+      tags?: string[];
+      type?: string;
+    }[] = [];
+
+    raw.forEach((item) => {
+      if (!item || typeof item !== 'object') return;
+
+      const address = typeof (item as any).address === 'string' ? (item as any).address : undefined;
+      const name = typeof (item as any).name === 'string' ? (item as any).name : undefined;
+      const type = typeof (item as any).type === 'string' ? (item as any).type : undefined;
+      const tagsRaw = (item as any).tags;
+      const tags =
+        Array.isArray(tagsRaw) && tagsRaw.every((tag) => typeof tag === 'string')
+          ? (tagsRaw as string[])
+          : undefined;
+
+      if (address || name || type || tags) {
+        locations.push({
+          address,
+          name,
+          tags,
+          type,
+        });
+      }
+    });
+
+    return locations;
+  }
+
   static parseDateFromString(value?: string | Date | null): Date | null {
     if (!value) return null;
     if (value instanceof Date) return Number.isNaN(value.getTime()) ? null : value;
@@ -537,6 +608,46 @@ export class UserMemoryModel {
     });
   };
 
+  createActivityMemory = async (
+    params: CreateUserMemoryActivityParams,
+  ): Promise<{ activity: UserMemoryActivity; memory: UserMemoryItem }> => {
+    return this.db.transaction(async (tx) => {
+      const baseValues = this.buildBaseMemoryInsertValues(params, {
+        metadata: params.activity.metadata ?? null,
+        status: params.activity.status ?? 'pending',
+        tags: params.activity.tags ?? null,
+      });
+
+      const [memory] = await tx.insert(userMemories).values(baseValues).returning();
+      if (!memory) throw new Error('Failed to create user memory activity');
+
+      const activityValues = {
+        associatedLocations: params.activity.associatedLocations ?? null,
+        associatedObjects: params.activity.associatedObjects ?? [],
+        associatedSubjects: params.activity.associatedSubjects ?? [],
+        capturedAt: params.activity.capturedAt,
+        endsAt: coerceDate(params.activity.endsAt),
+        feedback: params.activity.feedback ?? null,
+        feedbackVector: params.activity.feedbackVector ?? null,
+        metadata: params.activity.metadata ?? null,
+        narrative: params.activity.narrative ?? null,
+        narrativeVector: params.activity.narrativeVector ?? null,
+        notes: params.activity.notes ?? null,
+        startsAt: coerceDate(params.activity.startsAt),
+        status: params.activity.status ?? null,
+        tags: params.activity.tags ?? [],
+        timezone: params.activity.timezone ?? null,
+        type: params.activity.type ?? ActivityTypeEnum.Other,
+        userId: this.userId,
+        userMemoryId: memory.id,
+      } satisfies typeof userMemoriesActivities.$inferInsert;
+
+      const [activity] = await tx.insert(userMemoriesActivities).values(activityValues).returning();
+
+      return { activity, memory };
+    });
+  };
+
   createPreferenceMemory = async (
     params: CreateUserMemoryPreferenceParams,
   ): Promise<{ memory: UserMemoryItem; preference: UserMemoryPreference }> => {
@@ -575,12 +686,13 @@ export class UserMemoryModel {
     const { embedding, limits } = params;
 
     const resolvedLimits = {
+      activities: limits?.activities,
       contexts: limits?.contexts,
       experiences: limits?.experiences,
       preferences: limits?.preferences,
     };
 
-    const [experiences, contexts, preferences] = await Promise.all([
+    const [experiences, contexts, preferences, activities] = await Promise.all([
       this.searchExperiences({
         embedding,
         limit: resolvedLimits.experiences,
@@ -593,6 +705,10 @@ export class UserMemoryModel {
         embedding,
         limit: resolvedLimits.preferences,
       }),
+      this.searchActivities({
+        embedding,
+        limit: resolvedLimits.activities,
+      }),
     ]);
 
     const accessedMemoryIds = new Set<string>();
@@ -602,6 +718,9 @@ export class UserMemoryModel {
     preferences.forEach((preference) => {
       if (preference.userMemoryId) accessedMemoryIds.add(preference.userMemoryId);
     });
+    activities.forEach((activity) => {
+      if (activity.userMemoryId) accessedMemoryIds.add(activity.userMemoryId);
+    });
     const contextLinkIds: string[] = [];
     contexts.forEach((context) => {
       const ids = Array.isArray(context.userMemoryIds) ? (context.userMemoryIds as string[]) : [];
@@ -617,6 +736,7 @@ export class UserMemoryModel {
     }
 
     return {
+      activities,
       contexts,
       experiences,
       preferences,
@@ -1795,6 +1915,61 @@ export class UserMemoryModel {
     await this.db.delete(userMemories).where(eq(userMemories.userId, this.userId));
   };
 
+  searchActivities = async (params: {
+    embedding?: number[];
+    limit?: number;
+    type?: string;
+  }): Promise<UserMemoryActivitiesWithoutVectors[]> => {
+    const { embedding, limit = 5, type } = params;
+    if (limit <= 0) {
+      return [];
+    }
+
+    let query = this.db
+      .select({
+        accessedAt: userMemoriesActivities.accessedAt,
+        associatedLocations: userMemoriesActivities.associatedLocations,
+        associatedObjects: userMemoriesActivities.associatedObjects,
+        associatedSubjects: userMemoriesActivities.associatedSubjects,
+        capturedAt: userMemoriesActivities.capturedAt,
+        createdAt: userMemoriesActivities.createdAt,
+        endsAt: userMemoriesActivities.endsAt,
+        feedback: userMemoriesActivities.feedback,
+        id: userMemoriesActivities.id,
+        metadata: userMemoriesActivities.metadata,
+        narrative: userMemoriesActivities.narrative,
+        notes: userMemoriesActivities.notes,
+        startsAt: userMemoriesActivities.startsAt,
+        status: userMemoriesActivities.status,
+        tags: userMemoriesActivities.tags,
+        timezone: userMemoriesActivities.timezone,
+        type: userMemoriesActivities.type,
+        updatedAt: userMemoriesActivities.updatedAt,
+        userId: userMemoriesActivities.userId,
+        userMemoryId: userMemoriesActivities.userMemoryId,
+        ...(embedding && {
+          similarity: sql<number>`1 - (${cosineDistance(userMemoriesActivities.narrativeVector, embedding)}) AS similarity`,
+        }),
+      })
+      .from(userMemoriesActivities)
+      .$dynamic();
+
+    const conditions = [eq(userMemoriesActivities.userId, this.userId)];
+    if (type) {
+      conditions.push(eq(userMemoriesActivities.type, type));
+    }
+
+    query = query.where(and(...conditions));
+
+    if (embedding) {
+      query = query.orderBy(desc(sql`similarity`));
+    } else {
+      query = query.orderBy(desc(userMemoriesActivities.createdAt));
+    }
+
+    return query.limit(limit) as Promise<UserMemoryActivitiesWithoutVectors[]>;
+  };
+
   searchContexts = async (params: {
     embedding?: number[];
     limit?: number;

package/packages/database/src/models/userMemory/sources/benchmarkLoCoMo.ts

@@ -63,7 +63,7 @@ export class UserMemorySourceBenchmarkLoCoMoModel {
     return { id };
   }
 
-  async replaceParts(sourceId: string, parts: BenchmarkLoCoMoPart[]) {
+  replaceParts(sourceId: string, parts: BenchmarkLoCoMoPart[]) {
     const store = this.getPartStore();
     store.delete(sourceId);
     if (!parts.length) return;

package/packages/memory-user-memory/package.json

@@ -20,8 +20,8 @@
     "@lobechat/context-engine": "workspace:*",
     "@lobechat/model-runtime": "workspace:*",
     "@lobechat/prompts": "workspace:*",
-    "dotenv": "^17.2.3",
     "dayjs": "^1.11.11",
+    "dotenv": "^17.2.3",
     "ora": "^9.0.0",
     "unist-builder": "^4.0.0",
     "xast-util-to-xml": "^4.0.0",
@@ -30,6 +30,7 @@
   },
   "devDependencies": {
     "@lobechat/types": "workspace:*",
+    "@types/json-schema": "^7.0.15",
     "@types/xast": "^2.0.4",
     "promptfoo": "^0.118.17",
     "tsx": "^4.20.6"