@lobehub/lobehub 2.0.0-next.239 → 2.0.0-next.240

package/src/server/services/file/__tests__/index.test.ts

@@ -231,12 +231,12 @@ describe('FileService', () => {
     expect(result).toBe(expectedUrl);
   });
 
-  it('should delegate getKeyFromFullUrl to implementation', () => {
+  it('should delegate getKeyFromFullUrl to implementation', async () => {
     const testUrl = 'https://example.com/path/to/file.jpg';
     const expectedKey = 'path/to/file.jpg';
-    vi.mocked(service['impl'].getKeyFromFullUrl).mockReturnValue(expectedKey);
+    vi.mocked(service['impl'].getKeyFromFullUrl).mockResolvedValue(expectedKey);
 
-    const result = service.getKeyFromFullUrl(testUrl);
+    const result = await service.getKeyFromFullUrl(testUrl);
 
     expect(service['impl'].getKeyFromFullUrl).toHaveBeenCalledWith(testUrl);
     expect(result).toBe(expectedKey);

package/src/server/services/file/impls/index.ts

@@ -1,3 +1,5 @@
+import { LobeChatDatabase } from '@lobechat/database';
+
 import { S3StaticFileImpl } from './s3';
 import { type FileServiceImpl } from './type';
 
@@ -5,8 +7,6 @@ import { type FileServiceImpl } from './type';
  * Create file service module
  * Returns S3 file implementation for cloud storage
  */
-export const createFileServiceModule = (): FileServiceImpl => {
-  return new S3StaticFileImpl();
+export const createFileServiceModule = (db: LobeChatDatabase): FileServiceImpl => {
+  return new S3StaticFileImpl(db);
 };
-
-export type { FileServiceImpl } from './type';

package/src/server/services/file/impls/s3.test.ts

@@ -1,5 +1,7 @@
 import { beforeEach, describe, expect, it, vi } from 'vitest';
 
+import { FileModel } from '@/database/models/file';
+
 import { S3StaticFileImpl } from './s3';
 
 const config = {
@@ -33,11 +35,14 @@ vi.mock('@/server/modules/S3', () => ({
   })),
 }));
 
+// Mock db
+const mockDb = {} as any;
+
 describe('S3StaticFileImpl', () => {
   let fileService: S3StaticFileImpl;
 
   beforeEach(() => {
-    fileService = new S3StaticFileImpl();
+    fileService = new S3StaticFileImpl(mockDb);
   });
 
   describe('getFullFileUrl', () => {
@@ -74,7 +79,7 @@ describe('S3StaticFileImpl', () => {
         const fullUrl = 'https://s3.example.com/bucket/path/to/file.jpg?X-Amz-Signature=expired';
 
         // Mock getKeyFromFullUrl to return the extracted key
-        vi.spyOn(fileService, 'getKeyFromFullUrl').mockReturnValue('path/to/file.jpg');
+        vi.spyOn(fileService, 'getKeyFromFullUrl').mockResolvedValue('path/to/file.jpg');
 
         const result = await fileService.getFullFileUrl(fullUrl);
 
@@ -86,7 +91,7 @@ describe('S3StaticFileImpl', () => {
       it('should handle full URL input by extracting key (S3_SET_ACL=true)', async () => {
         const fullUrl = 'https://s3.example.com/bucket/path/to/file.jpg';
 
-        vi.spyOn(fileService, 'getKeyFromFullUrl').mockReturnValue('path/to/file.jpg');
+        vi.spyOn(fileService, 'getKeyFromFullUrl').mockResolvedValue('path/to/file.jpg');
 
         const result = await fileService.getFullFileUrl(fullUrl);
 
@@ -108,13 +113,23 @@ describe('S3StaticFileImpl', () => {
       it('should handle http:// URLs for legacy compatibility', async () => {
         const httpUrl = 'http://s3.example.com/bucket/path/to/file.jpg';
 
-        vi.spyOn(fileService, 'getKeyFromFullUrl').mockReturnValue('path/to/file.jpg');
+        vi.spyOn(fileService, 'getKeyFromFullUrl').mockResolvedValue('path/to/file.jpg');
 
         const result = await fileService.getFullFileUrl(httpUrl);
 
         expect(fileService.getKeyFromFullUrl).toHaveBeenCalledWith(httpUrl);
         expect(result).toBe('https://example.com/path/to/file.jpg');
       });
+
+      it('should throw error when key extraction returns null', async () => {
+        const fullUrl = 'https://s3.example.com/f/nonexistent';
+
+        vi.spyOn(fileService, 'getKeyFromFullUrl').mockResolvedValue(null);
+
+        await expect(fileService.getFullFileUrl(fullUrl)).rejects.toThrow(
+          'Key not found from url: ' + fullUrl,
+        );
+      });
     });
   });
 
@@ -179,63 +194,66 @@ describe('S3StaticFileImpl', () => {
   });
 
   describe('getKeyFromFullUrl', () => {
-    it('当S3_ENABLE_PATH_STYLE为false时应该正确提取key', () => {
-      config.S3_ENABLE_PATH_STYLE = false;
-      const url = 'https://example.com/path/to/file.jpg';
+    it('should extract fileId from proxy URL and return S3 key from database', async () => {
+      const proxyUrl = 'http://localhost:3010/f/abc123';
+      const expectedKey = 'ppp/491067/image.jpg';
 
-      const result = fileService.getKeyFromFullUrl(url);
+      vi.spyOn(FileModel, 'getFileById').mockResolvedValue({ url: expectedKey } as any);
 
-      expect(result).toBe('path/to/file.jpg');
-      config.S3_ENABLE_PATH_STYLE = false; // reset
+      const result = await fileService.getKeyFromFullUrl(proxyUrl);
+
+      expect(FileModel.getFileById).toHaveBeenCalledWith(mockDb, 'abc123');
+      expect(result).toBe(expectedKey);
     });
 
-    it('当S3_ENABLE_PATH_STYLE为true时应该正确提取key', () => {
-      config.S3_ENABLE_PATH_STYLE = true;
-      const url = 'https://example.com/my-bucket/path/to/file.jpg';
+    it('should return null when file is not found in database', async () => {
+      const proxyUrl = 'http://localhost:3010/f/nonexistent';
 
-      const result = fileService.getKeyFromFullUrl(url);
+      vi.spyOn(FileModel, 'getFileById').mockResolvedValue(undefined);
 
-      expect(result).toBe('path/to/file.jpg');
-      config.S3_ENABLE_PATH_STYLE = false; // reset
+      const result = await fileService.getKeyFromFullUrl(proxyUrl);
+
+      expect(FileModel.getFileById).toHaveBeenCalledWith(mockDb, 'nonexistent');
+      expect(result).toBeNull();
     });
 
-    it('当S3_ENABLE_PATH_STYLE为true但缺少bucket名称时应该返回pathname', () => {
-      config.S3_ENABLE_PATH_STYLE = true;
-      config.S3_BUCKET = '';
-      const url = 'https://example.com/path/to/file.jpg';
+    it('should handle URL with different domain', async () => {
+      const proxyUrl = 'https://example.com/f/file456';
+      const expectedKey = 'uploads/file.png';
 
-      const result = fileService.getKeyFromFullUrl(url);
+      vi.spyOn(FileModel, 'getFileById').mockResolvedValue({ url: expectedKey } as any);
 
-      expect(result).toBe('path/to/file.jpg');
-      config.S3_ENABLE_PATH_STYLE = false; // reset
-      config.S3_BUCKET = 'my-bucket'; // reset
+      const result = await fileService.getKeyFromFullUrl(proxyUrl);
+
+      expect(FileModel.getFileById).toHaveBeenCalledWith(mockDb, 'file456');
+      expect(result).toBe(expectedKey);
     });
 
-    it('当URL格式不正确时应该返回原始字符串', () => {
-      const invalidUrl = 'not-a-valid-url';
+    it('should extract key from legacy S3 URL (non /f/ path)', async () => {
+      const s3Url = 'https://example.com/path/to/file.jpg';
 
-      const result = fileService.getKeyFromFullUrl(invalidUrl);
+      const result = await fileService.getKeyFromFullUrl(s3Url);
 
-      expect(result).toBe('not-a-valid-url');
+      // Legacy S3 URL: extract key from pathname
+      expect(result).toBe('path/to/file.jpg');
     });
 
-    it('应该处理根路径文件', () => {
-      config.S3_ENABLE_PATH_STYLE = false;
-      const url = 'https://example.com/file.jpg';
+    it('should extract key with path-style S3 URL', async () => {
+      config.S3_ENABLE_PATH_STYLE = true;
+      const s3Url = 'https://example.com/my-bucket/path/to/file.jpg';
 
-      const result = fileService.getKeyFromFullUrl(url);
+      const result = await fileService.getKeyFromFullUrl(s3Url);
 
-      expect(result).toBe('file.jpg');
+      expect(result).toBe('path/to/file.jpg');
+      config.S3_ENABLE_PATH_STYLE = false;
     });
 
-    it('当path-style URL路径格式不符合预期时应该使用fallback', () => {
-      config.S3_ENABLE_PATH_STYLE = true;
-      const url = 'https://example.com/unexpected/path/file.jpg';
+    it('should return null for invalid URL', async () => {
+      const invalidUrl = 'not-a-valid-url';
 
-      const result = fileService.getKeyFromFullUrl(url);
+      const result = await fileService.getKeyFromFullUrl(invalidUrl);
 
-      expect(result).toBe('unexpected/path/file.jpg');
-      config.S3_ENABLE_PATH_STYLE = false; // reset
+      expect(result).toBeNull();
     });
   });
 

package/src/server/services/file/impls/s3.ts

@@ -1,18 +1,21 @@
+import { LobeChatDatabase } from '@lobechat/database';
 import urlJoin from 'url-join';
 
+import { FileModel } from '@/database/models/file';
 import { fileEnv } from '@/envs/file';
 import { FileS3 } from '@/server/modules/S3';
 
 import { type FileServiceImpl } from './type';
-import { extractKeyFromUrlOrReturnOriginal } from './utils';
 
 /**
  * S3-based file service implementation
  */
 export class S3StaticFileImpl implements FileServiceImpl {
   private readonly s3: FileS3;
+  private readonly db: LobeChatDatabase;
 
-  constructor() {
+  constructor(db: LobeChatDatabase) {
+    this.db = db;
     this.s3 = new FileS3();
   }
 
@@ -51,8 +54,16 @@ export class S3StaticFileImpl implements FileServiceImpl {
   async getFullFileUrl(url?: string | null, expiresIn?: number): Promise<string> {
     if (!url) return '';
 
-    // Handle legacy data compatibility using shared utility
-    const key = extractKeyFromUrlOrReturnOriginal(url, this.getKeyFromFullUrl.bind(this));
+    // Handle legacy data compatibility - extract key from full URL if needed
+    // Related issue: https://github.com/lobehub/lobe-chat/issues/8994
+    let key = url;
+    if (url.startsWith('http://') || url.startsWith('https://')) {
+      const extractedKey = await this.getKeyFromFullUrl(url);
+      if (!extractedKey) {
+        throw new Error('Key not found from url: ' + url);
+      }
+      key = extractedKey;
+    }
 
     // If bucket is not set public read, the preview address needs to be regenerated each time
     if (!fileEnv.S3_SET_ACL) {
@@ -66,38 +77,35 @@ export class S3StaticFileImpl implements FileServiceImpl {
     return urlJoin(fileEnv.S3_PUBLIC_DOMAIN!, key);
   }
 
-  getKeyFromFullUrl(url: string): string {
+  async getKeyFromFullUrl(url: string): Promise<string | null> {
     try {
       const urlObject = new URL(url);
       const { pathname } = urlObject;
 
-      let key: string;
+      // Case 1: File proxy URL pattern /f/{fileId} - query database for S3 key
+      if (pathname.startsWith('/f/')) {
+        const fileId = pathname.slice(3); // Remove '/f/' prefix
+        const file = await FileModel.getFileById(this.db, fileId);
+        return file?.url ?? null;
+      }
 
+      // Case 2: Legacy S3 URL - extract key from pathname
       if (fileEnv.S3_ENABLE_PATH_STYLE) {
         if (!fileEnv.S3_BUCKET) {
-          // In path-style, we need bucket name to extract key
-          // but if not provided, we can only guess the key is the pathname
           return pathname.startsWith('/') ? pathname.slice(1) : pathname;
         }
-        // For path-style URLs, the path is /<bucket>/<key>
-        // We need to remove the leading slash and the bucket name.
         const bucketPrefix = `/${fileEnv.S3_BUCKET}/`;
         if (pathname.startsWith(bucketPrefix)) {
-          key = pathname.slice(bucketPrefix.length);
-        } else {
-          // Fallback for unexpected path format
-          key = pathname.startsWith('/') ? pathname.slice(1) : pathname;
+          return pathname.slice(bucketPrefix.length);
         }
-      } else {
-        // For virtual-hosted-style URLs, the path is /<key>
-        // We just need to remove the leading slash.
-        key = pathname.slice(1);
+        return pathname.startsWith('/') ? pathname.slice(1) : pathname;
       }
 
-      return key;
+      // Virtual-hosted-style: path is /<key>
+      return pathname.slice(1);
     } catch {
-      // if url is not a valid URL, it may be a key itself
-      return url;
+      // If url is not a valid URL, return null
+      return null;
     }
   }
 

package/src/server/services/file/impls/type.ts

@@ -6,7 +6,6 @@ export interface FileServiceImpl {
    * Create pre-signed upload URL
    */
   createPreSignedUrl(key: string): Promise<string>;
-
   /**
    * Create pre-signed preview URL
    */
@@ -46,7 +45,7 @@ export interface FileServiceImpl {
   /**
    * Extract key from full URL
    */
-  getKeyFromFullUrl(url: string): string;
+  getKeyFromFullUrl(url: string): Promise<string | null>;
 
   /**
    * Upload content

package/src/server/services/file/index.ts

@@ -9,7 +9,8 @@ import { type FileItem } from '@/database/schemas';
 import { appEnv } from '@/envs/app';
 import { TempFileManager } from '@/server/utils/tempFileManager';
 
-import { type FileServiceImpl, createFileServiceModule } from './impls';
+import { createFileServiceModule } from './impls';
+import { type FileServiceImpl } from './impls/type';
 
 /**
  * File service class
@@ -19,11 +20,12 @@ export class FileService {
   private userId: string;
   private fileModel: FileModel;
 
-  private impl: FileServiceImpl = createFileServiceModule();
+  private impl: FileServiceImpl;
 
   constructor(db: LobeChatDatabase, userId: string) {
     this.userId = userId;
     this.fileModel = new FileModel(db, userId);
+    this.impl = createFileServiceModule(db);
   }
 
   /**
@@ -95,7 +97,7 @@ export class FileService {
   /**
    * Extract key from full URL
    */
-  public getKeyFromFullUrl(url: string): string {
+  public async getKeyFromFullUrl(url: string): Promise<string | null> {
     return this.impl.getKeyFromFullUrl(url);
   }
 

package/src/tools/inspectors.ts

@@ -15,6 +15,7 @@ import {
   LocalSystemInspectors,
   LocalSystemManifest,
 } from '@lobechat/builtin-tool-local-system/client';
+import { NotebookInspectors, NotebookManifest } from '@lobechat/builtin-tool-notebook/client';
 import { PageAgentInspectors, PageAgentManifest } from '@lobechat/builtin-tool-page-agent/client';
 import {
   WebBrowsingInspectors,
@@ -38,6 +39,7 @@ const BuiltinToolInspectors: Record<string, Record<string, BuiltinInspector>> =
   [GTDManifest.identifier]: GTDInspectors as Record<string, BuiltinInspector>,
   [KnowledgeBaseManifest.identifier]: KnowledgeBaseInspectors as Record<string, BuiltinInspector>,
   [LocalSystemManifest.identifier]: LocalSystemInspectors as Record<string, BuiltinInspector>,
+  [NotebookManifest.identifier]: NotebookInspectors as Record<string, BuiltinInspector>,
   [PageAgentManifest.identifier]: PageAgentInspectors as Record<string, BuiltinInspector>,
   [WebBrowsingManifest.identifier]: WebBrowsingInspectors as Record<string, BuiltinInspector>,
 };

package/src/tools/placeholders.ts

@@ -4,6 +4,10 @@ import {
   LocalSystemListFilesPlaceholder,
   LocalSystemSearchFilesPlaceholder,
 } from '@lobechat/builtin-tool-local-system/client';
+import {
+  NotebookIdentifier,
+  NotebookPlaceholders,
+} from '@lobechat/builtin-tool-notebook/client';
 import {
   WebBrowsingManifest,
   WebBrowsingPlaceholders,
@@ -19,6 +23,7 @@ export const BuiltinToolPlaceholders: Record<string, Record<string, any>> = {
     [LocalSystemApiName.searchLocalFiles]: LocalSystemSearchFilesPlaceholder,
     [LocalSystemApiName.listLocalFiles]: LocalSystemListFilesPlaceholder,
   },
+  [NotebookIdentifier]: NotebookPlaceholders as Record<string, any>,
   [WebBrowsingManifest.identifier]: WebBrowsingPlaceholders as Record<string, any>,
 };
 

package/src/tools/streamings.ts

@@ -11,6 +11,7 @@ import {
   LocalSystemManifest,
   LocalSystemStreamings,
 } from '@lobechat/builtin-tool-local-system/client';
+import { NotebookManifest, NotebookStreamings } from '@lobechat/builtin-tool-notebook/client';
 import { type BuiltinStreaming } from '@lobechat/types';
 
 /**
@@ -29,6 +30,7 @@ const BuiltinToolStreamings: Record<string, Record<string, BuiltinStreaming>> =
   >,
   [GTDManifest.identifier]: GTDStreamings as Record<string, BuiltinStreaming>,
   [LocalSystemManifest.identifier]: LocalSystemStreamings as Record<string, BuiltinStreaming>,
+  [NotebookManifest.identifier]: NotebookStreamings as Record<string, BuiltinStreaming>,
 };
 
 /**

package/packages/memory-user-memory/src/prompts/gatekeeper.md

@@ -1,125 +0,0 @@
-You are a "gate keeper" that analyzes conversations between user and assistant to determine which memory layers contain extractable information worth storing.
-
-Your role is to efficiently filter conversations by identifying which of the five memory layers are present and valuable enough to extract.
-
-Bias toward sensitivity: when in doubt, prefer setting `shouldExtract: true` so downstream extractors can refine. Minor, incremental or clarifying updates should still be allowed to pass.
-
-## Memory Layer Definitions
-
-Evaluate the conversation for these memory layers:
-
-**Activity Layer** - Episodic events with clear timelines and participants:
-
-- Meetings, calls, errands, and appointments with start/end times
-- Locations (physical or virtual) and timezones
-- Status (planned, completed) and follow-up actions
-- Narratives summarizing what happened and feedback/notes
-
-**Identity Layer** - Information about actors, relationships, and personal attributes:
-
-- Describing labels and demographics
-- Current focusing and life priorities
-- Relationships with other people
-- Background and experience
-- Roles in various contexts
-- Identity should remain compact. Route lists of tools, stacks, or
-  implementation techniques to the preference layer unless they materially
-  change the user's biography.
-
-**Context Layer** - Only capture brand-new situational frameworks and ongoing situations that have not been recorded before:
-
-- Distinct situations, topics, research threads, sessions, or rounds that are clearly first-time mentions
-- Ongoing projects and their status when the project itself is new, not just progress updates
-- Long-term goals and objectives that are newly introduced
-- Persistent relationships and dynamics that have not been logged previously
-- Environmental factors and recurring situations only when they represent a new context
-- Timelines and temporal context that establish a novel situation
-- Impact and urgency assessments tied to a new context
-- If the content overlaps with learnings or takeaways, treat it as Experience instead of Context
-
-**Preference Layer** - Durable user choices and behavioral directives that apply across multiple conversations:
-
-- Explicit long-term preferences with clear temporal markers (e.g., "always", "never", "from now on")
-- Likes and dislikes stated as persistent traits (e.g., "I prefer...", "I don't like...")
-- Workflow and approach preferences explicitly stated as ongoing (what to do, not how to implement)
-- Communication style preferences explicitly stated as ongoing
-- Response formatting preferences explicitly stated as ongoing
-- Priority levels for preference resolution
-- Capture recurring tool/stacks/technology usage as preferences rather than
-  duplicating them as identity facts.
-
-Note: Task-specific requirements, constraints for a single object/entity, or one-time instructions do NOT belong to this layer.
-
-**Experience Layer** - Learned insights and practical knowledge worth reusing and
-sharing publicly:
-
-- Lessons learned and insights gained, especially surprising aha/yurika moments
-- Practical tricks and techniques that solve tough or non-obvious problems
-- Transferable knowledge and wisdom that would make a strong blog/knowledge-base
-  snippet
-- Situation → Reasoning → Action → Outcome patterns
-- Key learnings from experiences with self-assessed confidence/impact
-- Skip routine or repetitive steps already well covered in retrieved memories
-
-## Gate Keeping Guidelines
-
-For each layer, consider:
-
-- **Relevance**: Does the conversation contain information for this layer?
-- **Value**: Is the information substantial enough to be worth extracting?
-- **Clarity**: Is the information clear and extractable (not vague or ambiguous)?
-
-Additionally, review the retrieved similar memories first (top {{ topK }} items below). If the conversation content is clearly and fully covered by existing memories with no meaningful nuance or update, set `shouldExtract: false`.
-For the Experience layer, prefer `shouldExtract: true` only when the conversation adds a new takeaway, aha moment, or harder challenge resolution beyond what is already recorded.
-For the Context layer specifically, favor `shouldExtract: false` unless the conversation introduces a genuinely new situation/topic/research/session that is not already represented in existing context memories or overlaps with Experience items.
-Otherwise, favor `shouldExtract: true` for:
-
-- Novel facts OR even small clarifications/precision improvements to existing details
-- Incremental changes to status/progress of an ongoing context
-- A new preference OR a refinement that affects future behavior
-- Distinct experiences/lessons, even if closely related to prior items
-
-Preference-specific filters (STRICT - when in doubt, set shouldExtract: false):
-
-CRITICAL: Distinguish between "user's behavioral preferences" vs. "task requirements for a specific deliverable":
-
-- If the instruction describes what the OUTPUT should be like (e.g., "name should have natural vibe", "don't use surname"), it's a task requirement, NOT a user preference.
-- If the instruction describes how the ASSISTANT should behave across conversations (e.g., "always explain before coding", "never use jargon"), it's a preference.
-
-Specific exclusion rules:
-
-- Do NOT treat the conversation's language as a preference unless the user explicitly states a persistent preference for language.
-- Exclude one-off task instructions and implementation steps; these are not preferences.
-- Exclude task-specific constraints, requirements, or clarifications (e.g., "don't use surname Wang", "make it summer-themed", "natural vibe") - these describe the current task deliverable, not user's persistent preferences.
-- Exclude requirements or attributes for a specific object/entity being discussed (e.g., naming a cat, designing a logo) - these are task parameters, not user preferences.
-- Exclude in-conversation clarifications or corrections (e.g., user first says X, then says "no, not X") - these refine the current task, not future behavior.
-- Only extract when the user uses explicit preference markers like "I always...", "I prefer...", "I never...", "from now on...", "please always..." that clearly indicate cross-session intent.
-- If the preference is about how to complete THIS specific task rather than how to behave in FUTURE tasks, exclude it.
-- Ask yourself: "Would this apply to a completely different conversation topic?" If no, it's not a preference.
-
-Examples of what NOT to extract:
-
-- User asks for a cat name with "natural vibe" and "born in summer" → NOT a preference (task constraint)
-- User says "don't use surname Wang" for naming → NOT a preference (task clarification)
-- User wants a "minimalist design" for this logo → NOT a preference (task requirement)
-
-Examples of what TO extract:
-
-- "I always prefer concise responses" → IS a preference (cross-session directive)
-- "Never use technical jargon when explaining to me" → IS a preference (persistent rule)
-- "From now on, please format code with comments" → IS a preference (ongoing instruction)
-
-If uncertain about novelty, but information is relevant and clear, set `shouldExtract: true` and include a short reasoning.
-
-## Output Guidelines
-
-For each memory layer (activity, identity, context, preference, experience), provide:
-
-- `shouldExtract`: Boolean indicating whether extraction is recommended
-- `reasoning`: Brief explanation of your decision (write in English)
-
-## Retrieved Memory (Top {{ topK }})
-
-Use the list below as context for deduplication and to determine whether extraction is necessary. Do not repeat these verbatim unless needed for comparison.
-
-{{ retrievedContext }}