@lobehub/lobehub 2.0.0-next.360 → 2.0.0-next.362

package/packages/memory-user-memory/src/extractors/activity.ts

@@ -0,0 +1,44 @@
+import { renderPlaceholderTemplate } from '@lobechat/context-engine';
+
+import { activityPrompt } from '../prompts';
+import { ActivityMemory, ActivityMemorySchema } from '../schemas';
+import { ExtractorTemplateProps } from '../types';
+import { BaseMemoryExtractor } from './base';
+
+export class ActivityExtractor extends BaseMemoryExtractor<ActivityMemory> {
+  getPrompt(): string {
+    return activityPrompt;
+  }
+
+  protected getPromptName(): string {
+    return 'layer-activity';
+  }
+
+  getSchema() {
+    return ActivityMemorySchema;
+  }
+
+  // Activity schema uses JSON Schema directly; no zod validation is applied here.
+  protected getResultSchema() {
+    return undefined;
+  }
+
+  getTemplateProps(options: ExtractorTemplateProps) {
+    return {
+      availableCategories: options.availableCategories,
+      language: options.language,
+      retrievedContext: options.retrievedContexts?.join('\n\n') || 'No similar memories retrieved.',
+      sessionDate: options.sessionDate,
+      topK: options.topK,
+      username: options.username,
+    };
+  }
+
+  buildUserPrompt(options: ExtractorTemplateProps): string {
+    if (!this.promptTemplate) {
+      throw new Error('Prompt template not loaded');
+    }
+
+    return renderPlaceholderTemplate(this.promptTemplate!, this.getTemplateProps(options));
+  }
+}

package/packages/memory-user-memory/src/extractors/gatekeeper.test.ts

@@ -31,7 +31,7 @@ describe('UserMemoryGateKeeper', () => {
     expect(schema?.strict).toBe(true);
 
     const properties = (schema?.schema as any).properties;
-    const requiredLayers = ['context', 'experience', 'identity', 'preference'];
+    const requiredLayers = ['activity', 'context', 'experience', 'identity', 'preference'];
 
     requiredLayers.forEach((layer) => {
       const layerSchema = properties[layer];
@@ -78,6 +78,7 @@ describe('UserMemoryGateKeeper', () => {
     const extractor = new UserMemoryGateKeeper(extractorConfig);
 
     const llmResult = {
+      activity: { reasoning: 'reasoning', shouldExtract: true },
       context: { reasoning: 'reasoning', shouldExtract: true },
       experience: { reasoning: 'reasoning', shouldExtract: false },
       identity: { reasoning: 'reasoning', shouldExtract: true },

package/packages/memory-user-memory/src/extractors/gatekeeper.ts

@@ -27,12 +27,13 @@ export class UserMemoryGateKeeper extends BaseMemoryExtractor<GatekeeperResult,
       schema: {
         additionalProperties: false,
         properties: {
+          activity: layerDecision,
           context: layerDecision,
           experience: layerDecision,
           identity: layerDecision,
           preference: layerDecision,
         },
-        required: ['context', 'experience', 'identity', 'preference'],
+        required: ['activity', 'context', 'experience', 'identity', 'preference'],
         type: 'object' as const,
       },
       strict: true,

package/packages/memory-user-memory/src/extractors/index.ts

@@ -1,3 +1,4 @@
+export { ActivityExtractor } from './activity';
 export { ContextExtractor } from './context';
 export { ExperienceExtractor } from './experience';
 export { UserMemoryGateKeeper } from './gatekeeper';

package/packages/memory-user-memory/src/prompts/gatekeeper.ts

@@ -9,11 +9,11 @@ export const gatekeeperPrompt = [
   '',
   'Evaluate the conversation for these memory layers:',
   '',
-  '**Activity Layer** - Episodic events with clear timelines and participants:',
+'**Activity Layer** - Episodic events with clear timelines and participants:',
   '',
   '- Meetings, calls, errands, and appointments with start/end times',
-  '- Locations (physical or virtual) and timezones',
-  '- Status (planned, completed) and follow-up actions',
+  '- Locations (physical or virtual) and timezones when provided (omit if absent)',
+  '- Status (planned, completed, cancelled, ongoing, on_hold, pending) and follow-up actions',
   '- Narratives summarizing what happened and feedback/notes',
   '',
   '**Identity Layer** - Information about actors, relationships, and personal attributes:',

package/packages/memory-user-memory/src/prompts/index.ts

@@ -1,2 +1,8 @@
 export { gatekeeperPrompt } from './gatekeeper';
-export { contextPrompt, experiencePrompt, identityPrompt, preferencePrompt } from './layers';
+export {
+  activityPrompt,
+  contextPrompt,
+  experiencePrompt,
+  identityPrompt,
+  preferencePrompt,
+} from './layers';

package/packages/memory-user-memory/src/prompts/layers/activity.ts

@@ -0,0 +1,90 @@
+export const activityPrompt = [
+  'You are a focused memory extraction assistant specialized in the **activity** layer.',
+  'When extracting, ensure all the content is using {{ language }}.',
+  '',
+  '\\<user_context>',
+  'Current user: {{ username }}',
+  'Session date: {{ sessionDate }}',
+  'Available memory categories: {{ availableCategories }}',
+  'Target layer: activity',
+  '\\</user_context>',
+  '',
+  '## Retrieved Memory (Top {{ topK }})',
+  '',
+  'Use the list below to de-duplicate and decide whether you need to extract anything new.',
+  'Do not copy these verbatim; treat them as comparison references.',
+  '',
+  '{{ retrievedContext }}',
+  '',
+  '## Your Task',
+  '',
+  'Extract **ALL** activity layer information from the conversation. Activities are time-bound events',
+  'with people, places, status, and a factual narrative of what happened. Capture how it felt via',
+  'feedback. Temporal (startsAt, endsAt, timezone) and association fields are OPTIONAL—only include',
+  'them when explicitly provided. If missing, **omit the fields entirely** (do not set null/empty) instead of guessing.',
+  '',
+  '**CRITICAL**: Return an **array** of activity memory items. Extract each distinct activity as a separate item.',
+  'Before extracting, review the retrieved similar memories (top {{ topK }} items shown above). Extract items',
+  'that are NEW or MATERIALLY UPDATED compared to retrieved entries. Avoid duplicates or near-duplicates.',
+  '',
+  '## Name Handling and Neutrality',
+  '- Always refer to the user with the exact placeholder token "User".',
+  "- Do not infer, invent, or translate the user's real name.",
+  '- Avoid gendered terms or honorifics; keep references neutral.',
+  '',
+  '## Output Format',
+  '',
+  'Return structured JSON data according to the provided schema:',
+  '- Basic fields: title, summary, details, memoryLayer, memoryType, memoryCategory, tags',
+  '- Activity-specific fields in withActivity: type (enum), status (planned/completed/cancelled/ongoing/on_hold/pending),',
+  '  optional timezone, startsAt, endsAt (ISO 8601), associatedSubjects (self-aware entities: people, pets, groups, orgs), associatedObjects',
+  '  (non-living items like transportation, devices),',
+  '  associatedLocations (places; capture any mentioned venue/address), notes (prep/agenda), narrative (factual story),',
+  '  feedback (how it felt), metadata (JSON), tags (activity facets).',
+  '- Do not fabricate dates, timezones, or locations. Only include them when explicitly mentioned or derivable.',
+  '',
+  '## Memory Formatting Guidelines',
+  '',
+  '> ALL MEMORY ITEMS MUST BE SELF-CONTAINED',
+  '- Use concrete names/entities—avoid pronouns.',
+  '- Preserve the original language from user input—do not translate.',
+  '- Include relevant details, participants, places, timing if present, and outcomes.',
+  '- Keep narratives factual; avoid speculative embellishment.',
+  '',
+  '✓ **Good Examples:**',
+  '- "User met with Alice at ACME HQ on 2024-05-03 14:00-15:00 America/New_York to review the Q2 renewal,',
+  '  aligned on reduced scope, and agreed to send revised pricing; User felt the call was positive and collaborative."',
+  '- "User visited Dr. Kim for a migraine check-up, discussed triggers and sleep, and started a low-dose preventive;',
+  '  User felt reassured."',
+  '',
+  '✗ **Bad Examples:**',
+  '- "They had a meeting" → Missing who, when, where, and outcome.',
+  '- "Felt good" → Missing context of the activity.',
+  '- "Probably in New York" → Do not guess locations or timezones.',
+  '',
+  '## Layer-Specific Guidance',
+  '- Focus on discrete events (meetings, calls, trips, workouts, meals, appointments).',
+  '- Always capture mentioned locations in associatedLocations. If none are stated, omit the field.',
+  '- If the activity is a trip, record transportation modes or specific rides/tickets in associatedObjects.',
+  '- Route any self-aware entity without a known identity ID (people, pets, groups, institutions/orgs) to associatedSubjects.',
+  '- type choices: meeting, appointment, exercise, meal, social, trip, task, workshop, conference, call, errand, celebration, class, project-session, other.',
+  '- Status handling: use provided status markers (planned, completed, cancelled, ongoing, on_hold, pending). If no status is stated, omit.',
+  '- Do not add the user as an associatedSubject; only include other mentioned living entities.',
+  '- Feedback captures how the activity felt (effort, mood, satisfaction); narrative captures the factual sequence.',
+  '- Location and temporal fields are optional—only include when stated.',
+  '- Temporal anchoring: when the conversation uses relative time phrases (yesterday, tomorrow, last week), anchor calculations to the specific message timestamp (created_at) if present; otherwise fall back to sessionDate. Keep the anchor timezone/time-of-day when converting relative references and do not invent new times if none are given.',
+  '',
+  '## Few-shot Hints',
+  '- Meeting with known person, time stated → include type=meeting, startsAt/endsAt/timezone, associatedSubjects for participants, location name if given.',
+  '- Workout described without time/place → type=exercise, omit time/timezone/location, keep narrative/feedback.',
+  '- Dinner with friends mentioned without exact time → type=meal or social, include attendees if named, omit time/place if absent.',
+  '',
+  '## Security Considerations',
+  '**NEVER extract or store sensitive information:** passwords, API keys, financial account numbers, or private encryption keys.',
+  '',
+  '---',
+  'Final instructions:',
+  '1. Extract each distinct activity memory as a separate item.',
+  '2. Ensure memories are self-contained and in {{ language }}.',
+  '3. Return JSON matching the schema. Use [] when nothing should be extracted.',
+].join('\\n');

package/packages/memory-user-memory/src/prompts/layers/index.ts

@@ -1,3 +1,4 @@
+export { activityPrompt } from './activity';
 export { contextPrompt } from './context';
 export { experiencePrompt } from './experience';
 export { identityPrompt } from './identity';

package/packages/memory-user-memory/src/providers/existingUserMemory.test.ts

@@ -19,6 +19,30 @@ describe('RetrievalUserMemoryContextProvider', () => {
     const provider = new RetrievalUserMemoryContextProvider({
       fetchedAt,
       retrievedMemories: {
+        activities: [
+          {
+            accessedAt: new Date(),
+            type: 'meeting',
+            associatedLocations: [{ name: 'Zoom' }],
+            associatedSubjects: [{ name: 'Alice', type: 'person' }],
+            capturedAt: new Date(),
+            createdAt: new Date(),
+            endsAt: new Date('2024-02-01T02:00:00.000Z'),
+            feedback: 'Felt good',
+            id: 'act-1',
+            metadata: {},
+            narrative: 'Weekly sync about roadmap',
+            notes: 'Agenda: roadmap',
+            startsAt: new Date('2024-02-01T01:00:00.000Z'),
+            status: 'completed',
+            tags: ['meeting'],
+            timezone: 'UTC',
+            updatedAt: new Date(),
+            userId: 'user-1',
+            userMemoryId: 'mem-act-1',
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+          } as any,
+        ],
         contexts: [
           {
             accessedAt: new Date(),
@@ -91,7 +115,7 @@ describe('RetrievalUserMemoryContextProvider', () => {
     expect(result.userId).toBe('user-1');
     expect(result.metadata).toEqual({});
     expect(result.context).equal(
-      '<user_memories contexts="1" experiences="1" memory_fetched_at="2024-02-01T00:00:00.000Z" preferences="1"><user_memories_context id="ctx-1" type="project"><context_title>LobeHub</context_title><context_description>Weekly syncs for LobeHub</context_description><context_current_status>active</context_current_status><context_tags>project, team</context_tags></user_memories_context><user_memories_experience id="exp-1" type="product"><experience_situation>Release planning</experience_situation><experience_key_learning>Ship smaller increments</experience_key_learning><experience_action>Shipped feature</experience_action><experience_reasoning>Better agility</experience_reasoning><experience_possible_outcome>Faster releases</experience_possible_outcome><experience_tags>release</experience_tags></user_memories_experience><user_memories_preference id="pref-1" type="style"><preference_conclusion_directives>Always keep updates concise</preference_conclusion_directives><preference_suggestions>Use bullet points</preference_suggestions><preference_tags>communication</preference_tags></user_memories_preference></user_memories>',
+      '<user_memories activities="1" contexts="1" experiences="1" memory_fetched_at="2024-02-01T00:00:00.000Z" preferences="1"><user_memories_activity id="act-1" activity_type="meeting" status="completed" timezone="UTC" starts_at="2024-02-01T01:00:00.000Z" ends_at="2024-02-01T02:00:00.000Z"><activity_narrative>Weekly sync about roadmap</activity_narrative><activity_notes>Agenda: roadmap</activity_notes><activity_feedback>Felt good</activity_feedback><activity_associated_location>Zoom</activity_associated_location><activity_associated_subject>Alice | type: person</activity_associated_subject><activity_tags>meeting</activity_tags></user_memories_activity><user_memories_context id="ctx-1" type="project"><context_title>LobeHub</context_title><context_description>Weekly syncs for LobeHub</context_description><context_current_status>active</context_current_status><context_tags>project, team</context_tags></user_memories_context><user_memories_experience id="exp-1" type="product"><experience_situation>Release planning</experience_situation><experience_key_learning>Ship smaller increments</experience_key_learning><experience_action>Shipped feature</experience_action><experience_reasoning>Better agility</experience_reasoning><experience_possible_outcome>Faster releases</experience_possible_outcome><experience_tags>release</experience_tags></user_memories_experience><user_memories_preference id="pref-1" type="style"><preference_conclusion_directives>Always keep updates concise</preference_conclusion_directives><preference_suggestions>Use bullet points</preference_suggestions><preference_tags>communication</preference_tags></user_memories_preference></user_memories>',
     );
   });
 });

package/packages/memory-user-memory/src/providers/existingUserMemory.ts

@@ -1,5 +1,6 @@
 import type {
   IdentityMemoryDetail,
+  UserMemoryActivityWithoutVectors,
   UserMemoryContextWithoutVectors,
   UserMemoryExperienceWithoutVectors,
   UserMemoryPreferenceWithoutVectors,
@@ -12,6 +13,7 @@ import { x } from 'xastscript';
 import type { BuiltContext, MemoryContextProvider, MemoryExtractionJob } from '../types';
 
 interface RetrievedMemories {
+  activities: UserMemoryActivityWithoutVectors[];
   contexts: UserMemoryContextWithoutVectors[];
   experiences: UserMemoryExperienceWithoutVectors[];
   preferences: UserMemoryPreferenceWithoutVectors[];
@@ -31,13 +33,123 @@ export class RetrievalUserMemoryContextProvider implements MemoryContextProvider
     this.fetchedAt = options.fetchedAt;
   }
 
+  private formatTags(tags?: unknown) {
+    return Array.isArray(tags) && tags.length ? `tags: ${tags.join(', ')}` : '';
+  }
+
+  private formatLocation(location: any) {
+    if (!location || typeof location !== 'object') return '';
+    const parts: string[] = [];
+
+    if (typeof location.name === 'string' && location.name) parts.push(location.name);
+    if (typeof location.type === 'string' && location.type) parts.push(`type: ${location.type}`);
+    if (typeof location.address === 'string' && location.address) parts.push(location.address);
+
+    const tags = this.formatTags(location.tags);
+    if (tags) parts.push(tags);
+
+    return parts.join(' | ');
+  }
+
+  private formatAssociation(value: any) {
+    if (!value || typeof value !== 'object') return '';
+    const parts: string[] = [];
+
+    if (typeof value.name === 'string' && value.name) parts.push(value.name);
+    if (typeof value.type === 'string' && value.type) parts.push(`type: ${value.type}`);
+
+    if (value.extra && typeof value.extra === 'object') {
+      try {
+        parts.push(`extra: ${JSON.stringify(value.extra)}`);
+      } catch {
+        parts.push('extra: [unserializable]');
+      }
+    }
+
+    return parts.join(' | ');
+  }
+
   async buildContext(job: MemoryExtractionJob): Promise<BuiltContext> {
+    const activities = this.retrievedMemories.activities || [];
     const contexts = this.retrievedMemories.contexts || [];
     const experiences = this.retrievedMemories.experiences || [];
     const preferences = this.retrievedMemories.preferences || [];
 
+
+
     const userMemoriesChildren: Child[] = [];
 
+    activities.forEach((activity) => {
+      const attributes: Record<string, string> = { id: activity.id ?? '' };
+      const similarity = (activity as { similarity?: number }).similarity;
+
+      if (typeof similarity === 'number') {
+        attributes.similarity = similarity.toFixed(3);
+      }
+      if (activity.type) {
+        attributes.activity_type = activity.type;
+      }
+      if (activity.status) {
+        attributes.status = activity.status;
+      }
+      if (activity.timezone) {
+        attributes.timezone = activity.timezone;
+      }
+      if (activity.startsAt) {
+        attributes.starts_at = new Date(activity.startsAt).toISOString();
+      }
+      if (activity.endsAt) {
+        attributes.ends_at = new Date(activity.endsAt).toISOString();
+      }
+
+      const children: Child[] = [];
+      const legacyLocation = (activity as { location?: unknown }).location;
+      const associatedLocations = Array.isArray(activity.associatedLocations)
+        ? activity.associatedLocations
+        : legacyLocation
+          ? [legacyLocation]
+          : [];
+      const associatedObjects = Array.isArray(activity.associatedObjects)
+        ? activity.associatedObjects
+        : [];
+      const associatedSubjects = Array.isArray(activity.associatedSubjects)
+        ? activity.associatedSubjects
+        : [];
+
+      if (activity.narrative) {
+        children.push(x('activity_narrative', activity.narrative));
+      }
+      if (activity.notes) {
+        children.push(x('activity_notes', activity.notes));
+      }
+      if (activity.feedback) {
+        children.push(x('activity_feedback', activity.feedback));
+      }
+      associatedLocations.forEach((location) => {
+        const value = this.formatLocation(location);
+        if (value) {
+          children.push(x('activity_associated_location', value));
+        }
+      });
+      associatedObjects.forEach((object) => {
+        const value = this.formatAssociation(object);
+        if (value) {
+          children.push(x('activity_associated_object', value));
+        }
+      });
+      associatedSubjects.forEach((subject) => {
+        const value = this.formatAssociation(subject);
+        if (value) {
+          children.push(x('activity_associated_subject', value));
+        }
+      });
+      if (Array.isArray(activity.tags) && activity.tags.length > 0) {
+        children.push(x('activity_tags', activity.tags.join(', ')));
+      }
+
+      userMemoriesChildren.push(x('user_memories_activity', attributes, ...children));
+    });
+
     contexts.forEach((context) => {
       const attributes: Record<string, string> = { id: context.id ?? '' };
       const similarity = (context as { similarity?: number }).similarity;
@@ -126,6 +238,7 @@ export class RetrievalUserMemoryContextProvider implements MemoryContextProvider
         x(
           'user_memories',
           {
+            activities: activities.length.toString(),
             contexts: contexts.length.toString(),
             experiences: experiences.length.toString(),
             memory_fetched_at: new Date(this.fetchedAt ?? Date.now()).toISOString(),