@lobehub/lobehub 2.0.0-next.97 → 2.0.0-next.99

package/packages/prompts/src/prompts/knowledgeBaseQA/formatFileContents.test.ts

@@ -0,0 +1,118 @@
+import { describe, expect, it } from 'vitest';
+
+import { promptFileContents } from './formatFileContents';
+import type { FileContent } from './formatFileContents';
+
+describe('promptFileContents', () => {
+  it('should format single file content', () => {
+    const fileContents: FileContent[] = [
+      {
+        content: 'This is the file content with some important information.',
+        fileId: 'file-001',
+        filename: 'document.md',
+      },
+    ];
+
+    const result = promptFileContents(fileContents);
+    expect(result).toMatchSnapshot();
+  });
+
+  it('should format multiple file contents', () => {
+    const fileContents: FileContent[] = [
+      {
+        content: 'First file content about API authentication.',
+        fileId: 'file-001',
+        filename: 'auth-guide.md',
+      },
+      {
+        content: 'Second file content about database setup and configuration.',
+        fileId: 'file-002',
+        filename: 'db-setup.md',
+      },
+      {
+        content: 'Third file content with deployment instructions.',
+        fileId: 'file-003',
+        filename: 'deployment.md',
+      },
+    ];
+
+    const result = promptFileContents(fileContents);
+    expect(result).toMatchSnapshot();
+  });
+
+  it('should handle file with error', () => {
+    const fileContents: FileContent[] = [
+      {
+        content: '',
+        error: 'File not found',
+        fileId: 'file-404',
+        filename: 'missing.md',
+      },
+    ];
+
+    const result = promptFileContents(fileContents);
+    expect(result).toMatchSnapshot();
+  });
+
+  it('should handle mixed successful and error files', () => {
+    const fileContents: FileContent[] = [
+      {
+        content: 'Successfully loaded content from this file.',
+        fileId: 'file-001',
+        filename: 'success.md',
+      },
+      {
+        content: '',
+        error: 'Permission denied',
+        fileId: 'file-002',
+        filename: 'restricted.md',
+      },
+      {
+        content: 'Another successfully loaded file with more content.',
+        fileId: 'file-003',
+        filename: 'another-success.md',
+      },
+    ];
+
+    const result = promptFileContents(fileContents);
+    expect(result).toMatchSnapshot();
+  });
+
+  it('should handle file with special characters in name', () => {
+    const fileContents: FileContent[] = [
+      {
+        content: 'Content from a file with special characters in the name.',
+        fileId: 'file-special',
+        filename: 'FAQ: Q&A (2024).md',
+      },
+    ];
+
+    const result = promptFileContents(fileContents);
+    expect(result).toMatchSnapshot();
+  });
+
+  it('should handle file with long content', () => {
+    const fileContents: FileContent[] = [
+      {
+        content: `# Comprehensive Guide
+
+## Introduction
+This is a very long document with multiple sections and detailed information.
+
+## Section 1
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
+
+## Section 2
+Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
+
+## Conclusion
+Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.`,
+        fileId: 'file-long',
+        filename: 'comprehensive-guide.md',
+      },
+    ];
+
+    const result = promptFileContents(fileContents);
+    expect(result).toMatchSnapshot();
+  });
+});

package/packages/prompts/src/prompts/knowledgeBaseQA/formatFileContents.ts

@@ -0,0 +1,31 @@
+export interface FileContent {
+  content: string;
+  error?: string;
+  fileId: string;
+  filename: string;
+}
+
+/**
+ * Formats a single file content with XML tags
+ */
+const formatFileContent = (file: FileContent): string => {
+  if (file.error) {
+    return `<file id="${file.fileId}" name="${file.filename}" error="${file.error}" />`;
+  }
+
+  return `<file id="${file.fileId}" name="${file.filename}">
+${file.content}
+</file>`;
+};
+
+/**
+ * Format file contents prompt for AI consumption using XML structure
+ */
+export const promptFileContents = (fileContents: FileContent[]): string => {
+  const filesXml = fileContents.map((file) => formatFileContent(file)).join('\n');
+
+  return `<knowledge_base_files totalCount="${fileContents.length}">
+<instruction>Use the information from these files to answer the user's question. Always cite the source files.</instruction>
+${filesXml}
+</knowledge_base_files>`;
+};

package/packages/prompts/src/prompts/knowledgeBaseQA/formatNoSearchResults.test.ts

@@ -0,0 +1,25 @@
+import { describe, expect, it } from 'vitest';
+
+import { promptNoSearchResults } from './formatNoSearchResults';
+
+describe('promptNoSearchResults', () => {
+  it('should format no search results with simple query', () => {
+    const result = promptNoSearchResults('how to install');
+    expect(result).toMatchSnapshot();
+  });
+
+  it('should format no search results with complex query', () => {
+    const result = promptNoSearchResults('API authentication with OAuth 2.0');
+    expect(result).toMatchSnapshot();
+  });
+
+  it('should format no search results with special characters', () => {
+    const result = promptNoSearchResults('How to use fetchData<T> with async/await?');
+    expect(result).toMatchSnapshot();
+  });
+
+  it('should format no search results with non-English query', () => {
+    const result = promptNoSearchResults('如何配置数据库连接');
+    expect(result).toMatchSnapshot();
+  });
+});

package/packages/prompts/src/prompts/knowledgeBaseQA/formatNoSearchResults.ts

@@ -0,0 +1,13 @@
+/**
+ * Format prompt when no search results found using XML structure
+ */
+export const promptNoSearchResults = (query: string): string => {
+  return `<knowledge_base_search_results query="${query}" totalCount="0">
+<instruction>No relevant files found in the knowledge base for this query.</instruction>
+<suggestions>
+<suggestion>Try rephrasing your question with different keywords</suggestion>
+<suggestion>Check if the information exists in the uploaded documents</suggestion>
+<suggestion>Ask the user to provide more context or upload relevant documents</suggestion>
+</suggestions>
+</knowledge_base_search_results>`;
+};

package/packages/prompts/src/prompts/knowledgeBaseQA/formatSearchResults.test.ts

@@ -0,0 +1,191 @@
+import { describe, expect, it } from 'vitest';
+
+import { formatSearchResults } from './formatSearchResults';
+import type { FileSearchResult } from './formatSearchResults';
+
+describe('formatSearchResults', () => {
+  it('should format single file with multiple chunks', () => {
+    const fileResults: FileSearchResult[] = [
+      {
+        fileId: 'file-123',
+        fileName: 'Getting Started Guide.md',
+        relevanceScore: 0.92,
+        topChunks: [
+          {
+            similarity: 0.95,
+            text: 'To get started with the application, first install all dependencies using npm install or yarn install. Make sure you have Node.js version 16 or higher installed on your system.',
+          },
+          {
+            similarity: 0.88,
+            text: 'After installation, you can run the development server with npm run dev. The application will be available at http://localhost:3000 by default.',
+          },
+        ],
+      },
+    ];
+
+    const result = formatSearchResults(fileResults, 'how to install the application');
+    expect(result).toMatchSnapshot();
+  });
+
+  it('should format multiple files with varying relevance scores', () => {
+    const fileResults: FileSearchResult[] = [
+      {
+        fileId: 'doc-001',
+        fileName: 'API Reference.md',
+        relevanceScore: 0.95,
+        topChunks: [
+          {
+            similarity: 0.98,
+            text: 'The Authentication API provides endpoints for user login, logout, and token refresh. All endpoints require HTTPS and proper API keys.',
+          },
+          {
+            similarity: 0.92,
+            text: 'To authenticate a user, send a POST request to /api/auth/login with username and password in the request body.',
+          },
+        ],
+      },
+      {
+        fileId: 'doc-002',
+        fileName: 'Security Best Practices.md',
+        relevanceScore: 0.87,
+        topChunks: [
+          {
+            similarity: 0.89,
+            text: 'Always use environment variables to store sensitive information like API keys and database credentials. Never commit these to version control.',
+          },
+          {
+            similarity: 0.85,
+            text: 'Implement rate limiting on all public endpoints to prevent abuse and DDoS attacks.',
+          },
+        ],
+      },
+      {
+        fileId: 'doc-003',
+        fileName: 'Troubleshooting Guide.md',
+        relevanceScore: 0.73,
+        topChunks: [
+          {
+            similarity: 0.75,
+            text: 'If you encounter authentication errors, first verify that your API key is valid and has not expired.',
+          },
+        ],
+      },
+    ];
+
+    const result = formatSearchResults(fileResults, 'API authentication security');
+    expect(result).toMatchSnapshot();
+  });
+
+  it('should format long text chunks without truncation', () => {
+    const fileResults: FileSearchResult[] = [
+      {
+        fileId: 'long-doc',
+        fileName: 'Detailed Documentation.md',
+        relevanceScore: 0.91,
+        topChunks: [
+          {
+            similarity: 0.93,
+            text: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.',
+          },
+        ],
+      },
+    ];
+
+    const result = formatSearchResults(fileResults, 'documentation details');
+    expect(result).toMatchSnapshot();
+  });
+
+  it('should handle empty file results', () => {
+    const fileResults: FileSearchResult[] = [];
+    const result = formatSearchResults(fileResults, 'no results query');
+    expect(result).toMatchSnapshot();
+  });
+
+  it('should format file with special characters in name', () => {
+    const fileResults: FileSearchResult[] = [
+      {
+        fileId: 'special-123',
+        fileName: 'FAQ: Common Questions & Answers (2024).md',
+        relevanceScore: 0.88,
+        topChunks: [
+          {
+            similarity: 0.9,
+            text: 'Q: How do I reset my password? A: Click on the "Forgot Password" link on the login page.',
+          },
+        ],
+      },
+    ];
+
+    const result = formatSearchResults(fileResults, 'password reset');
+    expect(result).toMatchSnapshot();
+  });
+
+  it('should handle files with low relevance scores', () => {
+    const fileResults: FileSearchResult[] = [
+      {
+        fileId: 'low-relevance',
+        fileName: 'Tangentially Related.md',
+        relevanceScore: 0.32,
+        topChunks: [
+          {
+            similarity: 0.35,
+            text: 'This document contains some loosely related information that might be helpful.',
+          },
+        ],
+      },
+    ];
+
+    const result = formatSearchResults(fileResults, 'specific technical query');
+    expect(result).toMatchSnapshot();
+  });
+
+  it('should format query with special characters', () => {
+    const fileResults: FileSearchResult[] = [
+      {
+        fileId: 'code-sample',
+        fileName: 'Code Examples.md',
+        relevanceScore: 0.94,
+        topChunks: [
+          {
+            similarity: 0.96,
+            text: 'Use the following TypeScript code: const result = await fetchData<T>({ url, params });',
+          },
+        ],
+      },
+    ];
+
+    const result = formatSearchResults(fileResults, 'How to use fetchData<T> with async/await?');
+    expect(result).toMatchSnapshot();
+  });
+
+  it('should handle file with multiple high-similarity chunks', () => {
+    const fileResults: FileSearchResult[] = [
+      {
+        fileId: 'comprehensive-guide',
+        fileName: 'Complete Tutorial.md',
+        relevanceScore: 0.96,
+        topChunks: [
+          {
+            similarity: 0.98,
+            text: 'Step 1: Initialize the project with the configuration file.',
+          },
+          {
+            similarity: 0.97,
+            text: 'Step 2: Configure the environment variables for your deployment.',
+          },
+          {
+            similarity: 0.96,
+            text: 'Step 3: Run the migration scripts to set up the database schema.',
+          },
+          {
+            similarity: 0.95,
+            text: 'Step 4: Start the development server and verify everything is working.',
+          },
+        ],
+      },
+    ];
+
+    const result = formatSearchResults(fileResults, 'project setup steps');
+    expect(result).toMatchSnapshot();
+  });
+});

package/packages/prompts/src/prompts/knowledgeBaseQA/formatSearchResults.ts

@@ -0,0 +1,50 @@
+export interface FileSearchResultChunk {
+  similarity: number;
+  text: string;
+}
+
+export interface FileSearchResult {
+  fileId: string;
+  fileName: string;
+  relevanceScore: number;
+  topChunks: FileSearchResultChunk[];
+}
+
+/**
+ * Formats a single chunk with XML tags
+ */
+const formatChunk = (chunk: FileSearchResultChunk, fileId: string, fileName: string): string => {
+  return `<chunk fileId="${fileId}" fileName="${fileName}" similarity="${chunk.similarity}">${chunk.text}</chunk>`;
+};
+
+/**
+ * Formats a single file search result with XML tags
+ */
+const formatFile = (file: FileSearchResult): string => {
+  const chunks = file.topChunks.map((chunk) => formatChunk(chunk, file.fileId, file.fileName));
+
+  return `<file id="${file.fileId}" name="${file.fileName}" relevanceScore="${file.relevanceScore}">
+${chunks.join('\n')}
+</file>`;
+};
+
+/**
+ * Formats knowledge base search results into an XML structure
+ * @param fileResults - Array of file search results with relevance scores and chunks
+ * @param query - The original search query
+ * @returns Formatted XML string with search results
+ */
+export const formatSearchResults = (fileResults: FileSearchResult[], query: string): string => {
+  if (fileResults.length === 0) {
+    return `<knowledge_base_search_results query="${query}" totalCount="0">
+<instruction>No relevant files found in the knowledge base for this query.</instruction>
+</knowledge_base_search_results>`;
+  }
+
+  const filesXml = fileResults.map((file) => formatFile(file)).join('\n');
+
+  return `<knowledge_base_search_results query="${query}" totalCount="${fileResults.length}">
+<instruction>Here are the search results from the knowledge base. Use the readKnowledge tool with file IDs to get complete content.</instruction>
+${filesXml}
+</knowledge_base_search_results>`;
+};

package/packages/prompts/src/prompts/knowledgeBaseQA/index.ts

@@ -4,6 +4,12 @@ import { chunkPrompts } from './chunk';
 import { knowledgePrompts } from './knowledge';
 import { userQueryPrompt } from './userQuery';
 
+export type { FileContent } from './formatFileContents';
+export { promptFileContents } from './formatFileContents';
+export { promptNoSearchResults } from './formatNoSearchResults';
+export type { FileSearchResult, FileSearchResultChunk } from './formatSearchResults';
+export { formatSearchResults } from './formatSearchResults';
+
 export const knowledgeBaseQAPrompts = ({
   chunks,
   knowledge,

package/packages/types/src/rag.ts

@@ -5,12 +5,21 @@ import { ChatSemanticSearchChunk } from './chunk';
 export const SemanticSearchSchema = z.object({
   fileIds: z.array(z.string()).optional(),
   knowledgeIds: z.array(z.string()).optional(),
-  messageId: z.string(),
-  model: z.string().optional(),
-  rewriteQuery: z.string(),
-  userQuery: z.string(),
+  query: z.string(),
+  topK: z.number().optional(),
 });
 
 export type SemanticSearchSchemaType = z.infer<typeof SemanticSearchSchema>;
 
 export type MessageSemanticSearchChunk = Pick<ChatSemanticSearchChunk, 'id' | 'similarity'>;
+
+export interface FileSearchResult {
+  fileId: string;
+  fileName: string;
+  relevanceScore: number;
+  topChunks: Array<{
+    id: string;
+    similarity: number;
+    text: string;
+  }>;
+}

package/scripts/checkConsoleLog.mts

@@ -0,0 +1,148 @@
+#!/usr/bin/env tsx
+
+import { execSync } from 'node:child_process';
+import { readFileSync } from 'node:fs';
+
+interface WhitelistConfig {
+  files?: string[];
+  patterns?: string[];
+}
+
+const WHITELIST_PATH = '.console-log-whitelist.json';
+
+/**
+ * Load whitelist configuration
+ */
+const loadWhitelist = (): WhitelistConfig => {
+  try {
+    const content = readFileSync(WHITELIST_PATH, 'utf8');
+    return JSON.parse(content);
+  } catch {
+    return { files: [], patterns: [] };
+  }
+};
+
+/**
+ * Check if a file is whitelisted
+ */
+const isWhitelisted = (filePath: string, whitelist: WhitelistConfig): boolean => {
+  const normalizedPath = filePath.replaceAll('\\', '/');
+
+  // Check exact file matches
+  if (whitelist.files?.some((f) => normalizedPath.includes(f.replaceAll('\\', '/')))) {
+    return true;
+  }
+
+  // Check pattern matches (simple glob-like patterns)
+  if (whitelist.patterns) {
+    for (const pattern of whitelist.patterns) {
+      // Escape dots and replace glob patterns
+      // Use a placeholder for ** to avoid conflicts with single *
+      let regexPattern = pattern
+        .replaceAll('.', '\\.')
+        .replaceAll('**', '\u0000DOUBLESTAR\u0000')
+        .replaceAll('*', '[^/]*')
+        .replaceAll('\u0000DOUBLESTAR\u0000', '.*');
+
+      // If pattern ends with /**, match everything under that directory
+      // If pattern ends with **, just match everything from that point
+      const regex = new RegExp(`^${regexPattern}`);
+      if (regex.test(normalizedPath)) {
+        return true;
+      }
+    }
+  }
+
+  return false;
+};
+
+/**
+ * Main check function
+ */
+const checkConsoleLogs = () => {
+  const whitelist = loadWhitelist();
+
+  console.log('🔍 Checking for console.log statements...\n');
+
+  try {
+    // Search for console.log in TypeScript and JavaScript files
+    const output = execSync(
+      `git grep -n "console\\.log" -- "*.ts" "*.tsx" "*.js" "*.jsx" "*.mts" "*.cts" || true`,
+      { encoding: 'utf8' },
+    );
+
+    if (!output.trim()) {
+      console.log('✅ No console.log statements found!');
+      return;
+    }
+
+    const lines = output.trim().split('\n');
+    const violations: Array<{ content: string, file: string; line: string; }> = [];
+
+    for (const line of lines) {
+      // Parse git grep output: filename:lineNumber:content
+      const match = line.match(/^([^:]+):(\d+):(.+)$/);
+      if (!match) continue;
+
+      const [, filePath, lineNumber, content] = match;
+
+      // Skip if whitelisted
+      if (isWhitelisted(filePath, whitelist)) {
+        continue;
+      }
+
+      // Skip comments
+      const trimmedContent = content.trim();
+      if (trimmedContent.startsWith('//') || trimmedContent.startsWith('*')) {
+        continue;
+      }
+
+      violations.push({
+        content: content.trim(),
+        file: filePath,
+        line: lineNumber,
+      });
+    }
+
+    if (violations.length === 0) {
+      console.log('✅ No console.log violations found (all matches are whitelisted or in comments)!');
+      return;
+    }
+
+    // Report violations as warnings
+    console.log('⚠️  Found console.log statements in the following files:\n');
+
+    // Use GitHub Actions annotation format for better visibility in CI
+    const isCI = process.env.CI === 'true' || process.env.GITHUB_ACTIONS === 'true';
+
+    for (const violation of violations) {
+      if (isCI) {
+        // GitHub Actions warning annotation format
+        console.log(`::warning file=${violation.file},line=${violation.line}::console.log found: ${violation.content}`);
+      } else {
+        console.log(`  ${violation.file}:${violation.line}`);
+        console.log(`    ${violation.content}\n`);
+      }
+    }
+
+    console.log(`\n💡 Total violations: ${violations.length}`);
+    console.log(
+      `\n📝 To whitelist files, add them to ${WHITELIST_PATH} in the following format:`,
+    );
+    console.log(`{
+  "files": ["path/to/file.ts"],
+  "patterns": ["scripts/**/*.mts", "**/*.test.ts"]
+}\n`);
+
+    // Exit with 0 to not block CI, but warnings will still be visible
+    process.exit(0);
+  } catch (error: unknown) {
+    if (error instanceof Error && 'status' in error && error.status !== 0) {
+      console.error('❌ Error running git grep:', error.message);
+      process.exit(1);
+    }
+    throw error;
+  }
+};
+
+checkConsoleLogs();

package/scripts/prebuild.mts

@@ -10,11 +10,11 @@ dotenv.config();
 /* eslint-disable sort-keys-fix/sort-keys-fix */
 const partialBuildPages = [
   // no need for desktop
-  {
-    name: 'changelog',
-    disabled: isDesktop,
-    paths: ['src/app/[variants]/(main)/changelog'],
-  },
+  // {
+  //   name: 'changelog',
+  //   disabled: isDesktop,
+  //   paths: ['src/app/[variants]/(main)/changelog'],
+  // },
   {
     name: 'auth',
     disabled: isDesktop,

package/src/app/[variants]/(main)/changelog/index.tsx

@@ -18,7 +18,7 @@ const Page = (props: { isMobile: boolean }) => {
   const { isMobile } = props;
   const { hideDocs } = useServerConfigStore(featureFlagsSelectors);
 
-  const { data } = useSWR('changelog-index', async () => {
+  const { data = [] } = useSWR('changelog-index', async () => {
     const changelogService = new ChangelogService();
     return await changelogService.getChangelogIndex();
   });