@lobehub/lobehub 2.0.0-next.360 → 2.0.0-next.362

package/packages/memory-user-memory/src/schemas/activity.ts

@@ -0,0 +1,315 @@
+import type { GenerateObjectSchema } from '@lobechat/model-runtime';
+import { ActivityTypeEnum, LayersEnum, TypesEnum } from '@lobechat/types';
+import { type JSONSchema7 } from 'json-schema';
+
+export interface WithActivity {
+  associatedLocations?: {
+    address?: string;
+    extra?: string | null;
+    name?: string;
+    tags?: string[];
+    type?: string;
+  }[];
+  associatedObjects?: {
+    extra?: string | null;
+    name?: string;
+    type?: string;
+  }[];
+  associatedSubjects?: {
+    extra?: string | null;
+    name?: string;
+    type?: string;
+  }[];
+  endsAt?: string;
+  feedback?: string;
+  metadata?: Record<string, unknown>;
+  narrative: string;
+  notes?: string;
+  startsAt?: string;
+  status?: string;
+  tags?: string[];
+  timezone?: string;
+  type: ActivityTypeEnum | string;
+}
+
+export interface ActivityMemoryItem {
+  details: string;
+  memoryCategory: string;
+  memoryLayer: LayersEnum.Activity;
+  memoryType: TypesEnum.Activity;
+  summary: string;
+  tags: string[];
+  title: string;
+  withActivity: WithActivity;
+}
+
+export interface ActivityMemory {
+  memories: ActivityMemoryItem[];
+}
+
+export const ActivityMemorySchema: GenerateObjectSchema = {
+  description:
+    'Extract episodic activities with clear timelines, participants, objects, subjects, locations, and feelings. Temporal and associated fields are optional—omit when missing rather than guessing.',
+  name: 'activity_extraction',
+  schema: {
+    additionalProperties: false,
+    properties: {
+      memories: {
+        description:
+          'Array of extracted activity memories. Use an empty array when no activity should be captured.',
+        items: {
+          additionalProperties: false,
+          description:
+            'Self-contained activity memory describing what happened, when, where, with whom, and how it felt.',
+          examples: [
+            {
+              details:
+                'Talked through renewal scope, confirmed timeline flexibility, and captured follow-ups.',
+              memoryCategory: 'work',
+              memoryType: 'activity',
+              summary: 'Client Q2 renewal meeting with Alice (ACME)',
+              tags: ['meeting', 'client', 'renewal'],
+              title: 'ACME Q2 renewal meeting',
+              withActivity: {
+                associatedLocations: [
+                  {
+                    address: '123 Main St, New York, NY',
+                    name: 'ACME HQ',
+                  },
+                ],
+                associatedSubjects: [
+                  { name: 'Alice Smith', type: 'person' },
+                ],
+                endsAt: '2024-05-03T15:00:00-04:00',
+                feedback: 'Positive momentum; Alice felt heard and open to renewal.',
+                narrative:
+                  'Alice and User reviewed Q2 renewal scope, aligned on reduced deliverables, and agreed to share revised pricing next week.',
+                notes: 'Agenda: renewal scope, pricing, next steps.',
+                startsAt: '2024-05-03T14:00:00-04:00',
+                status: 'completed',
+                timezone: 'America/New_York',
+                type: 'meeting',
+              },
+            },
+            {
+              details: 'Routine check-up; discussed migraines and sleep habits.',
+              memoryCategory: 'health',
+              memoryType: 'activity',
+              summary: 'Doctor appointment with Dr. Kim about migraines',
+              tags: ['appointment', 'health'],
+              title: 'Neurology follow-up',
+              withActivity: {
+                associatedLocations: [
+                  {
+                    name: 'City Neurology Clinic',
+                  },
+                ],
+                associatedSubjects: [
+                  { name: 'Dr. Kim', type: 'person' },
+                ],
+                feedback: 'Felt reassured; plan seems manageable.',
+                narrative:
+                  'User saw Dr. Kim to review migraine frequency; decided to track sleep, hydration, and start a low-dose preventive.',
+                notes: 'Discussed triggers, hydration, and medication side effects.',
+                status: 'completed',
+                type: 'appointment',
+              },
+            },
+          ],
+          properties: {
+            details: {
+              description:
+                'Optional detailed information or longer notes supporting the summary and narrative.',
+              type: 'string',
+            },
+            memoryCategory: {
+              description:
+                'Memory category best matching the activity (e.g., work, health, travel, relationships).',
+              type: 'string',
+            },
+            memoryType: {
+              const: TypesEnum.Activity,
+              description: 'Memory type; always activity.',
+              type: 'string',
+            },
+            summary: {
+              description: 'Concise overview of this activity.',
+              type: 'string',
+            },
+            tags: {
+              description: 'Model-generated tags summarizing key facets of the activity.',
+              items: { type: 'string' },
+              type: 'array',
+            },
+            title: {
+              description:
+                'Brief descriptive title for the activity, e.g., "Dinner with friends at Marina".',
+              type: 'string',
+            },
+            withActivity: {
+              additionalProperties: false,
+              description:
+                'Structured activity fields. Temporal and association values are optional—include only when the user mentioned them.',
+              properties: {
+                associatedLocations: {
+                  description:
+                    'Places linked to this activity. Capture any mentioned venue, address, or setting.',
+                  items: {
+                    additionalProperties: false,
+                    properties: {
+                      address: {
+                        description: 'Free-form address or directions if provided.',
+                        type: ['string', 'null'],
+                      },
+                      extra: {
+                        description: 'Optional key-value metadata related to the location.',
+                        type: ['string', 'null'],
+                      },
+                      name: {
+                        description: 'Place name or venue label.',
+                        type: 'string',
+                      },
+                      tags: {
+                        description: 'Place-related tags (e.g., indoor, outdoor, virtual).',
+                        items: { type: 'string' },
+                        type: ['array', 'null'],
+                      },
+                      type: {
+                        description: 'Place type or category (office, clinic, restaurant, virtual).',
+                        type: 'string',
+                      },
+                    },
+                    required: ['type', 'name', 'address', 'tags', 'extra'],
+                    type: 'object',
+                  },
+                  type: 'array',
+                },
+                associatedObjects: {
+                  description:
+                    'Non-living entities or items tied to the activity (e.g., transportation for trips, devices, tools).',
+                  items: {
+                    additionalProperties: false,
+                    properties: {
+                      extra: {
+                        description: 'Optional key-value metadata related to the object.',
+                        type: ['string', 'null'],
+                      },
+                      name: {
+                        description: 'Name or label of the object (e.g., “MacBook”, “flight UA123”).',
+                        type: 'string',
+                      },
+                      type: {
+                        description: 'Object category (e.g., transportation, device, document).',
+                        enum: ['application', 'item', 'knowledge', 'other', 'person', 'place'],
+                        type: 'string',
+                      },
+                    },
+                    required: ['type', 'name', 'extra'],
+                    type: 'object',
+                  },
+                  type: 'array',
+                },
+                associatedSubjects: {
+                  description:
+                    'Living beings involved (people, pets, groups). Use when the subject lacks a known identity ID.',
+                  items: {
+                    additionalProperties: false,
+                    properties: {
+                      extra: {
+                        description: 'Optional key-value metadata related to the subject.',
+                        type: ['string', 'null'],
+                      },
+                      name: {
+                        description: 'Name or short label of the subject.',
+                        type: 'string',
+                      },
+                      type: {
+                        description: 'Subject category (e.g., person, pet, group).',
+                        enum: ['person', 'pet', 'group', 'other'],
+                        type: 'string',
+                      },
+                    },
+                    required: ['type', 'name', 'extra'],
+                    type: 'object',
+                  },
+                  type: 'array',
+                },
+                endsAt: {
+                  description:
+                    'ISO 8601 end time for the activity when specified. Omit if not explicitly provided.',
+                  format: 'date-time',
+                  type: ['string', 'null'],
+                },
+                feedback: {
+                  description:
+                    'Subjective feelings or evaluation of how the activity went (mood, satisfaction, effort).',
+                  type: ['string', 'null'],
+                },
+                metadata: {
+                  additionalProperties: false,
+                  description:
+                    'Additional structured metadata to keep raw hints (JSON object). Use sparingly.',
+                  type: ['object', 'null'],
+                },
+                narrative: {
+                  description:
+                    'Factual story of what happened (chronology, participants, outcomes). Required for recall.',
+                  type: 'string',
+                },
+                notes: {
+                  description:
+                    'Short annotations such as agenda, preparation, or quick bullets distinct from narrative.',
+                  type: ['string', 'null'],
+                },
+                startsAt: {
+                  description:
+                    'ISO 8601 start time for the activity when specified. Omit if not explicitly provided.',
+                  format: 'date-time',
+                  type: ['string', 'null'],
+                },
+                status: {
+                  description:
+                    'Lifecycle status when mentioned. Use planned/completed/cancelled/ongoing/on_hold/pending. Omit if unclear.',
+                  enum: ['planned', 'completed', 'cancelled', 'ongoing', 'on_hold', 'pending'],
+                  type: ['string', 'null'],
+                },
+                tags: {
+                  description: 'Optional activity-specific tags or facets.',
+                  items: { type: 'string' },
+                  type: ['array', 'null'],
+                },
+                timezone: {
+                  description:
+                    'IANA timezone string for the start/end times when provided (e.g., "America/New_York").',
+                  type: ['string', 'null'],
+                },
+                type: {
+                  description:
+                    'Activity type enum. Choose the closest match; fall back to "other" when unclear.',
+                  enum: Object.values(ActivityTypeEnum),
+                  type: 'string',
+                },
+              },
+              required: ['type', 'narrative', 'feedback', 'notes', 'associatedLocations', 'associatedSubjects', 'associatedObjects', 'startsAt', 'endsAt', 'status', 'tags', 'timezone', 'metadata'],
+              type: 'object',
+            },
+          },
+          required: [
+            'title',
+            'summary',
+            'details',
+            'memoryType',
+            'memoryCategory',
+            'tags',
+            'withActivity',
+          ],
+          type: 'object',
+        },
+        type: 'array',
+      },
+    } satisfies JSONSchema7['properties'],
+    required: ['memories'],
+    type: 'object',
+  },
+  strict: true,
+};

package/packages/memory-user-memory/src/schemas/experience.ts

@@ -8,6 +8,11 @@ import { MemoryTypeSchema } from './common';
 export const WithExperienceSchema = z.object({
   action: z.string().describe('Narrative describing actions taken or behaviors exhibited'),
   keyLearning: z.string().describe('Narrative describing key insights or lessons learned'),
+  knowledgeValueScore: z
+    .number()
+    .min(0)
+    .max(1)
+    .describe('Numeric score (0-1) describing how reusable and shareable this experience is'),
   labels: z.array(z.string()).describe('Model generated tags that summarize the experience facets'),
   possibleOutcome: z.string().describe('Narrative describing potential outcomes or learnings'),
   problemSolvingScore: z
@@ -15,11 +20,6 @@ export const WithExperienceSchema = z.object({
     .min(0)
     .max(1)
     .describe('Numeric score (0-1) describing how effectively the problem was solved'),
-  knowledgeValueScore: z
-    .number()
-    .min(0)
-    .max(1)
-    .describe('Numeric score (0-1) describing how reusable and shareable this experience is'),
   reasoning: z.string().describe('Narrative describing the thought process or motivations'),
   scoreConfidence: z
     .number()

package/packages/memory-user-memory/src/schemas/gatekeeper.ts

@@ -14,6 +14,7 @@ export type LayerDecision = z.infer<typeof LayerDecisionSchema>;
  * Gatekeeper result schema for memory layers
  */
 export const GatekeeperResultSchema = z.object({
+  activity: LayerDecisionSchema,
   context: LayerDecisionSchema,
   experience: LayerDecisionSchema,
   identity: LayerDecisionSchema,

package/packages/memory-user-memory/src/schemas/index.ts

@@ -1,4 +1,5 @@
 export * from './common';
+export * from './activity';
 export * from './context';
 export * from './experience';
 export * from './gatekeeper';

package/packages/memory-user-memory/src/services/extractExecutor.ts

@@ -12,6 +12,7 @@ import { attributesCommon } from '@lobechat/observability-otel/node';
 import { LayersEnum } from '@lobechat/types';
 
 import {
+  ActivityExtractor,
   ContextExtractor,
   ExperienceExtractor,
   IdentityExtractor,
@@ -32,6 +33,7 @@ import {
 } from '../types';
 
 const LAYER_ORDER: LayersEnum[] = [
+  'activity' as LayersEnum,
   'identity' as LayersEnum,
   'context' as LayersEnum,
   'preference' as LayersEnum,
@@ -39,6 +41,7 @@ const LAYER_ORDER: LayersEnum[] = [
 ];
 
 const LAYER_LABEL_MAP: Record<LayersEnum, string> = {
+  activity: 'activities',
   context: 'contexts',
   experience: 'experiences',
   identity: 'identities',
@@ -61,6 +64,7 @@ export interface MemoryExtractionServiceOptions {
 
 export interface MemoryExtractionLayerOutputTypes {
   [LayersEnum.Context]: Awaited<ReturnType<ContextExtractor['structuredCall']>>;
+  [LayersEnum.Activity]: Awaited<ReturnType<ActivityExtractor['structuredCall']>>;
   [LayersEnum.Experience]: Awaited<ReturnType<ExperienceExtractor['structuredCall']>>;
   [LayersEnum.Preference]: Awaited<ReturnType<PreferenceExtractor['structuredCall']>>;
   [LayersEnum.Identity]: Awaited<ReturnType<IdentityExtractor['structuredCall']>>;
@@ -79,6 +83,7 @@ export class MemoryExtractionService<RO> {
   private readonly contextExtractor: ContextExtractor;
   private readonly experienceExtractor: ExperienceExtractor;
   private readonly preferenceExtractor: PreferenceExtractor;
+  private readonly activityExtractor: ActivityExtractor;
 
   private readonly gatekeeperRuntime: ModelRuntime;
   private readonly layerRuntime: ModelRuntime;
@@ -115,6 +120,9 @@ export class MemoryExtractionService<RO> {
     this.identityExtractor = new IdentityExtractor(
       buildExtractorConfig(LayersEnum.Identity, 'layer-identity'),
     );
+    this.activityExtractor = new ActivityExtractor(
+      buildExtractorConfig(LayersEnum.Activity, 'layer-activity'),
+    );
     this.contextExtractor = new ContextExtractor(
       buildExtractorConfig(LayersEnum.Context, 'layer-context'),
     );
@@ -214,6 +222,7 @@ export class MemoryExtractionService<RO> {
       const outputs = await this.runLayers(job, layersToExtract, { ...options });
 
       const processedLayersCount = {
+        activity: outputs.activity?.data ? outputs.activity?.data?.memories?.length : 0,
         context: outputs.context?.data ? outputs.context?.data?.memories?.length : 0,
         experience: outputs.experience?.data ? outputs.experience?.data?.memories?.length : 0,
         identity: outputs.identity?.data
@@ -224,6 +233,7 @@ export class MemoryExtractionService<RO> {
         preference: outputs.preference?.data ? outputs.preference?.data?.memories?.length : 0,
       };
       const processedErrorsCount = {
+        activity: outputs.activity?.error ? 1 : 0,
         context: outputs.context?.error ? 1 : 0,
         experience: outputs.experience?.error ? 1 : 0,
         identity: outputs.identity?.error ? 1 : 0,
@@ -281,6 +291,15 @@ export class MemoryExtractionService<RO> {
     );
   }
 
+  private async runActivityLayer(job: MemoryExtractionJob, options: ExtractorOptions) {
+    return this.runLayerExtractor(job, LayersEnum.Activity, () =>
+      this.activityExtractor.structuredCall({
+        ...options,
+        language: options.language ?? 'English',
+      }),
+    );
+  }
+
   private async runExperienceLayer(job: MemoryExtractionJob, options: ExtractorOptions) {
     return this.runLayerExtractor(job, LayersEnum.Experience, () =>
       this.experienceExtractor.structuredCall({
@@ -347,6 +366,12 @@ export class MemoryExtractionService<RO> {
             | { error: unknown };
           break;
         }
+        case LayersEnum.Activity: {
+          outputs.activity = result as
+            | { data: MemoryExtractionLayerOutputTypes[typeof layer] }
+            | { error: unknown };
+          break;
+        }
         case LayersEnum.Experience: {
           outputs.experience = result as
             | { data: MemoryExtractionLayerOutputTypes[typeof layer] }
@@ -374,6 +399,7 @@ export class MemoryExtractionService<RO> {
     await Promise.all(
       (
         [
+          LayersEnum.Activity,
           LayersEnum.Context,
           LayersEnum.Experience,
           LayersEnum.Preference,
@@ -407,6 +433,9 @@ export class MemoryExtractionService<RO> {
       case LayersEnum.Context: {
         return await this.runContextLayer(job, options);
       }
+      case LayersEnum.Activity: {
+        return await this.runActivityLayer(job, options);
+      }
       case LayersEnum.Experience: {
         return await this.runExperienceLayer(job, options);
       }

package/packages/memory-user-memory/src/types.ts

@@ -6,6 +6,7 @@ import type {
 import type { LayersEnum, MemorySourceType } from '@lobechat/types';
 
 import type {
+  ActivityExtractor,
   ContextExtractor,
   ExperienceExtractor,
   IdentityExtractor,
@@ -14,6 +15,7 @@ import type {
 
 export type MemoryExtractionAgent =
   | 'gatekeeper'
+  | 'layer-activity'
   | 'layer-context'
   | 'layer-experience'
   | 'layer-identity'
@@ -124,6 +126,10 @@ export interface PersistedMemoryResult {
 }
 
 export type MemoryExtractionLayerOutputs = Partial<{
+  activity: {
+    data?: Awaited<ReturnType<ActivityExtractor['structuredCall']>>;
+    error?: unknown;
+  };
   context: {
     data?: Awaited<ReturnType<ContextExtractor['structuredCall']>>;
     error?: unknown;
@@ -143,6 +149,7 @@ export type MemoryExtractionLayerOutputs = Partial<{
 }>;
 
 export interface GatekeeperDecision {
+  activity: MemoryLayerDecision;
   context: MemoryLayerDecision;
   experience: MemoryLayerDecision;
   identity: MemoryLayerDecision;

package/packages/prompts/src/agents/index.ts

@@ -1 +1,2 @@
 export * from './pageContentContext';
+export * from './pageSelectionContext';

package/packages/prompts/src/agents/pageSelectionContext.ts

@@ -0,0 +1,28 @@
+import type { PageSelection } from '@lobechat/types';
+
+/**
+ * Format page selections into a system prompt context
+ * Each selection is wrapped in a <selection> tag with metadata
+ */
+export const formatPageSelections = (selections: PageSelection[]): string => {
+  if (!selections || selections.length === 0) {
+    return '';
+  }
+
+  const formattedSelections = selections
+    .map((sel) => {
+      const lineInfo =
+        sel.startLine !== undefined
+          ? ` lines="${sel.startLine}-${sel.endLine ?? sel.startLine}"`
+          : '';
+
+      return `<selection ${lineInfo}>
+${sel.xml}
+</selection>`;
+    })
+    .join('\n');
+
+  return `<user_selections count="${selections.length}">
+${formattedSelections}
+</user_selections>`;
+};

package/packages/types/src/aiChat.ts

@@ -1,6 +1,7 @@
 import { z } from 'zod';
 
 import { UIChatMessage } from './message';
+import { PageSelection, PageSelectionSchema } from './message/ui/params';
 import { OpenAIChatMessage } from './openai/chat';
 import { LobeUniformTool, LobeUniformToolSchema } from './tool';
 import { ChatTopic } from './topic';
@@ -10,6 +11,8 @@ export interface SendNewMessage {
   content: string;
   // if message has attached with files, then add files to message and the agent
   files?: string[];
+  /** Page selections attached to this message (for Ask AI functionality) */
+  pageSelections?: PageSelection[];
   parentId?: string;
 }
 
@@ -83,6 +86,7 @@ export const AiSendMessageServerSchema = z.object({
   newUserMessage: z.object({
     content: z.string(),
     files: z.array(z.string()).optional(),
+    pageSelections: z.array(PageSelectionSchema).optional(),
     parentId: z.string().optional(),
   }),
   sessionId: z.string().optional(),

package/packages/types/src/message/common/index.ts

@@ -2,5 +2,6 @@ export * from './base';
 export * from './image';
 export * from './messageGroup';
 export * from './metadata';
+export * from './pageSelection';
 export * from './tools';
 export * from './translate';

package/packages/types/src/message/common/metadata.ts

@@ -1,6 +1,8 @@
 /* eslint-disable sort-keys-fix/sort-keys-fix , typescript-sort-keys/interface */
 import { z } from 'zod';
 
+import { PageSelection, PageSelectionSchema } from './pageSelection';
+
 export interface ModelTokensUsage {
   // Input tokens breakdown
   /**
@@ -80,6 +82,7 @@ export const MessageMetadataSchema = ModelUsageSchema.merge(ModelPerformanceSche
   inspectExpanded: z.boolean().optional(),
   isMultimodal: z.boolean().optional(),
   isSupervisor: z.boolean().optional(),
+  pageSelections: z.array(PageSelectionSchema).optional(),
 });
 
 export interface ModelUsage extends ModelTokensUsage {
@@ -147,4 +150,9 @@ export interface MessageMetadata extends ModelUsage, ModelPerformance {
    */
   instruction?: string;
   taskTitle?: string;
+  /**
+   * Page selections attached to user message
+   * Used for Ask AI functionality to persist selection context
+   */
+  pageSelections?: PageSelection[];
 }

package/packages/types/src/message/common/pageSelection.ts

@@ -0,0 +1,36 @@
+/* eslint-disable sort-keys-fix/sort-keys-fix , typescript-sort-keys/interface */
+import { z } from 'zod';
+
+/**
+ * Page selection represents a user-selected text region in a page/document.
+ * Used for Ask AI functionality to persist selection context with user messages.
+ */
+export interface PageSelection {
+  /** Selection unique identifier */
+  id: string;
+  anchor?: {
+    startNodeId: string;
+    endNodeId: string;
+    startOffset: number;
+    endOffset: number;
+  };
+  /** Selected content (plain text or markdown) */
+  content: string;
+  /** XML structure of the selected content (for positioning edits) */
+  xml?: string;
+  /** Page ID the selection belongs to */
+  pageId: string;
+  /** Start line number */
+  startLine?: number;
+  /** End line number */
+  endLine?: number;
+}
+
+export const PageSelectionSchema = z.object({
+  id: z.string(),
+  content: z.string(),
+  xml: z.string().optional(),
+  pageId: z.string(),
+  startLine: z.number().optional(),
+  endLine: z.number().optional(),
+});

package/packages/types/src/message/ui/params.ts

@@ -5,6 +5,8 @@ import { ConversationContext } from '../../conversation';
 import { UploadFileItem } from '../../files';
 import { MessageSemanticSearchChunk } from '../../rag';
 import { ChatMessageError, ChatMessageErrorSchema } from '../common/base';
+// Import for local use
+import type { PageSelection } from '../common/pageSelection';
 import { ChatPluginPayload, ToolInterventionSchema } from '../common/tools';
 import { UIChatMessage } from './chat';
 import { SemanticSearchChunkSchema } from './rag';
@@ -78,6 +80,10 @@ export interface ChatContextContent {
    */
   format?: 'xml' | 'text' | 'markdown';
   id: string;
+  /**
+   * Page ID the selection belongs to (for page editor selections)
+   */
+  pageId?: string;
   /**
    * Optional short preview for displaying in UI.
    */
@@ -86,6 +92,10 @@ export interface ChatContextContent {
   type: 'text';
 }
 
+// Re-export PageSelection from common for backwards compatibility
+export type { PageSelection } from '../common/pageSelection';
+export { PageSelectionSchema } from '../common/pageSelection';
+
 export interface SendMessageParams {
   /**
    * create a thread
@@ -119,8 +129,14 @@ export interface SendMessageParams {
   parentId?: string;
   /**
    * Additional contextual snippets (e.g., text selections) attached to the request.
+   * @deprecated Use pageSelections instead for page editor selections
    */
   contexts?: ChatContextContent[];
+  /**
+   * Page selections attached to the message (for Ask AI functionality)
+   * These will be persisted to the database and injected via context-engine
+   */
+  pageSelections?: PageSelection[];
 }
 
 export interface SendGroupMessageParams {

package/packages/types/src/serverConfig.ts

@@ -10,7 +10,7 @@ import {
   UserSystemAgentConfig,
 } from './user/settings';
 
-export type GlobalMemoryLayer = 'context' | 'experience' | 'identity' | 'preference';
+export type GlobalMemoryLayer = 'activity' | 'context' | 'experience' | 'identity' | 'preference';
 
 export interface MemoryAgentPublicConfig {
   baseURL?: string;