@lobehub/lobehub 2.0.0-next.340 → 2.0.0-next.342

package/src/server/routers/lambda/userMemories/reembed.ts

@@ -1,440 +0,0 @@
-import { asc, eq, gte, lte } from 'drizzle-orm';
-import pMap from 'p-map';
-import { z } from 'zod';
-
-import {
-  userMemories,
-  userMemoriesContexts,
-  userMemoriesExperiences,
-  userMemoriesIdentities,
-  userMemoriesPreferences,
-} from '@/database/schemas';
-
-import {
-  EMBEDDING_VECTOR_DIMENSION,
-  combineConditions,
-  getEmbeddingRuntime,
-  memoryProcedure,
-  normalizeEmbeddable,
-  router,
-} from './shared';
-
-const REEMBED_TABLE_KEYS = [
-  'userMemories',
-  'contexts',
-  'preferences',
-  'identities',
-  'experiences',
-] as const;
-type ReEmbedTableKey = (typeof REEMBED_TABLE_KEYS)[number];
-
-const reEmbedInputSchema = z.object({
-  concurrency: z.coerce.number().int().min(1).max(50).optional(),
-  endDate: z.coerce.date().optional(),
-  limit: z.coerce.number().int().min(1).optional(),
-  only: z.array(z.enum(REEMBED_TABLE_KEYS)).optional(),
-  startDate: z.coerce.date().optional(),
-});
-
-interface ReEmbedStats {
-  failed: number;
-  skipped: number;
-  succeeded: number;
-  total: number;
-}
-
-export const reembedRouter = router({
-  reEmbedMemories: memoryProcedure
-    .input(reEmbedInputSchema.optional())
-    .mutation(async ({ ctx, input }) => {
-      try {
-        const options = input ?? {};
-        const { agentRuntime, embeddingModel } = await getEmbeddingRuntime(
-          ctx.serverDB,
-          ctx.userId,
-        );
-        const concurrency = options.concurrency ?? 10;
-        const shouldProcess = (key: ReEmbedTableKey) =>
-          !options.only || options.only.length === 0 || options.only.includes(key);
-
-        const embedTexts = async (texts: string[]): Promise<number[][]> => {
-          if (texts.length === 0) return [];
-
-          const response = await agentRuntime.embeddings({
-            dimensions: EMBEDDING_VECTOR_DIMENSION,
-            input: texts,
-            model: embeddingModel,
-          });
-
-          if (!response || response.length !== texts.length) {
-            throw new Error('Embedding response length mismatch');
-          }
-
-          return response;
-        };
-
-        const results: Partial<Record<ReEmbedTableKey, ReEmbedStats>> = {};
-
-        const run = async (key: ReEmbedTableKey, handler: () => Promise<ReEmbedStats>) => {
-          if (!shouldProcess(key)) return;
-          results[key] = await handler();
-        };
-
-        // Re-embed userMemories
-        await run('userMemories', async () => {
-          const where = combineConditions([
-            eq(userMemories.userId, ctx.userId),
-            options.startDate ? gte(userMemories.createdAt, options.startDate) : undefined,
-            options.endDate ? lte(userMemories.createdAt, options.endDate) : undefined,
-          ]);
-
-          const rows = await ctx.serverDB.query.userMemories.findMany({
-            columns: { details: true, id: true, summary: true },
-            limit: options.limit,
-            orderBy: [asc(userMemories.createdAt)],
-            where,
-          });
-
-          let succeeded = 0;
-          let failed = 0;
-          let skipped = 0;
-
-          await pMap(
-            rows,
-            async (row) => {
-              const summaryText = normalizeEmbeddable(row.summary);
-              const detailsText = normalizeEmbeddable(row.details);
-
-              try {
-                if (!summaryText && !detailsText) {
-                  await ctx.memoryModel.updateUserMemoryVectors(row.id, {
-                    detailsVector1024: null,
-                    summaryVector1024: null,
-                  });
-                  skipped += 1;
-                  return;
-                }
-
-                const inputs: string[] = [];
-                if (summaryText) inputs.push(summaryText);
-                if (detailsText) inputs.push(detailsText);
-
-                const embeddings = await embedTexts(inputs);
-                let embedIndex = 0;
-
-                const summaryVector = summaryText ? (embeddings[embedIndex++] ?? null) : null;
-                const detailsVector = detailsText ? (embeddings[embedIndex++] ?? null) : null;
-
-                await ctx.memoryModel.updateUserMemoryVectors(row.id, {
-                  detailsVector1024: detailsVector,
-                  summaryVector1024: summaryVector,
-                });
-
-                succeeded += 1;
-              } catch (err) {
-                failed += 1;
-                console.error(
-                  `[memoryRouter.reEmbed] Failed to re-embed user memory ${row.id}`,
-                  err,
-                );
-              }
-            },
-            { concurrency },
-          );
-
-          return {
-            failed,
-            skipped,
-            succeeded,
-            total: rows.length,
-          } satisfies ReEmbedStats;
-        });
-
-        // Re-embed contexts
-        await run('contexts', async () => {
-          const where = combineConditions([
-            eq(userMemoriesContexts.userId, ctx.userId),
-            options.startDate ? gte(userMemoriesContexts.createdAt, options.startDate) : undefined,
-            options.endDate ? lte(userMemoriesContexts.createdAt, options.endDate) : undefined,
-          ]);
-
-          const rows = await ctx.serverDB.query.userMemoriesContexts.findMany({
-            columns: { description: true, id: true },
-            limit: options.limit,
-            orderBy: [asc(userMemoriesContexts.createdAt)],
-            where,
-          });
-
-          let succeeded = 0;
-          let failed = 0;
-          let skipped = 0;
-
-          await pMap(
-            rows,
-            async (row) => {
-              const description = normalizeEmbeddable(row.description);
-
-              try {
-                if (!description) {
-                  await ctx.memoryModel.updateContextVectors(row.id, {
-                    descriptionVector: null,
-                  });
-                  skipped += 1;
-                  return;
-                }
-
-                const [embedding] = await embedTexts([description]);
-
-                await ctx.memoryModel.updateContextVectors(row.id, {
-                  descriptionVector: embedding ?? null,
-                });
-                succeeded += 1;
-              } catch (err) {
-                failed += 1;
-                console.error(`[memoryRouter.reEmbed] Failed to re-embed context ${row.id}`, err);
-              }
-            },
-            { concurrency },
-          );
-
-          return {
-            failed,
-            skipped,
-            succeeded,
-            total: rows.length,
-          } satisfies ReEmbedStats;
-        });
-
-        // Re-embed preferences
-        await run('preferences', async () => {
-          const where = combineConditions([
-            eq(userMemoriesPreferences.userId, ctx.userId),
-            options.startDate
-              ? gte(userMemoriesPreferences.createdAt, options.startDate)
-              : undefined,
-            options.endDate ? lte(userMemoriesPreferences.createdAt, options.endDate) : undefined,
-          ]);
-
-          const rows = await ctx.serverDB.query.userMemoriesPreferences.findMany({
-            columns: { conclusionDirectives: true, id: true },
-            limit: options.limit,
-            orderBy: [asc(userMemoriesPreferences.createdAt)],
-            where,
-          });
-
-          let succeeded = 0;
-          let failed = 0;
-          let skipped = 0;
-
-          await pMap(
-            rows,
-            async (row) => {
-              const directives = normalizeEmbeddable(row.conclusionDirectives);
-
-              try {
-                if (!directives) {
-                  await ctx.memoryModel.updatePreferenceVectors(row.id, {
-                    conclusionDirectivesVector: null,
-                  });
-                  skipped += 1;
-                  return;
-                }
-
-                const [embedding] = await embedTexts([directives]);
-                await ctx.memoryModel.updatePreferenceVectors(row.id, {
-                  conclusionDirectivesVector: embedding ?? null,
-                });
-                succeeded += 1;
-              } catch (err) {
-                failed += 1;
-                console.error(
-                  `[memoryRouter.reEmbed] Failed to re-embed preference ${row.id}`,
-                  err,
-                );
-              }
-            },
-            { concurrency },
-          );
-
-          return {
-            failed,
-            skipped,
-            succeeded,
-            total: rows.length,
-          } satisfies ReEmbedStats;
-        });
-
-        // Re-embed identities
-        await run('identities', async () => {
-          const where = combineConditions([
-            eq(userMemoriesIdentities.userId, ctx.userId),
-            options.startDate
-              ? gte(userMemoriesIdentities.createdAt, options.startDate)
-              : undefined,
-            options.endDate ? lte(userMemoriesIdentities.createdAt, options.endDate) : undefined,
-          ]);
-
-          const rows = await ctx.serverDB.query.userMemoriesIdentities.findMany({
-            columns: { description: true, id: true },
-            limit: options.limit,
-            orderBy: [asc(userMemoriesIdentities.createdAt)],
-            where,
-          });
-
-          let succeeded = 0;
-          let failed = 0;
-          let skipped = 0;
-
-          await pMap(
-            rows,
-            async (row) => {
-              const description = normalizeEmbeddable(row.description);
-
-              try {
-                if (!description) {
-                  await ctx.memoryModel.updateIdentityVectors(row.id, {
-                    descriptionVector: null,
-                  });
-                  skipped += 1;
-                  return;
-                }
-
-                const [embedding] = await embedTexts([description]);
-                await ctx.memoryModel.updateIdentityVectors(row.id, {
-                  descriptionVector: embedding ?? null,
-                });
-                succeeded += 1;
-              } catch (err) {
-                failed += 1;
-                console.error(`[memoryRouter.reEmbed] Failed to re-embed identity ${row.id}`, err);
-              }
-            },
-            { concurrency },
-          );
-
-          return {
-            failed,
-            skipped,
-            succeeded,
-            total: rows.length,
-          } satisfies ReEmbedStats;
-        });
-
-        // Re-embed experiences
-        await run('experiences', async () => {
-          const where = combineConditions([
-            eq(userMemoriesExperiences.userId, ctx.userId),
-            options.startDate
-              ? gte(userMemoriesExperiences.createdAt, options.startDate)
-              : undefined,
-            options.endDate ? lte(userMemoriesExperiences.createdAt, options.endDate) : undefined,
-          ]);
-
-          const rows = await ctx.serverDB.query.userMemoriesExperiences.findMany({
-            columns: { action: true, id: true, keyLearning: true, situation: true },
-            limit: options.limit,
-            orderBy: [asc(userMemoriesExperiences.createdAt)],
-            where,
-          });
-
-          let succeeded = 0;
-          let failed = 0;
-          let skipped = 0;
-
-          await pMap(
-            rows,
-            async (row) => {
-              const situation = normalizeEmbeddable(row.situation);
-              const action = normalizeEmbeddable(row.action);
-              const keyLearning = normalizeEmbeddable(row.keyLearning);
-
-              try {
-                if (!situation && !action && !keyLearning) {
-                  await ctx.memoryModel.updateExperienceVectors(row.id, {
-                    actionVector: null,
-                    keyLearningVector: null,
-                    situationVector: null,
-                  });
-                  skipped += 1;
-                  return;
-                }
-
-                const inputs: string[] = [];
-                if (situation) inputs.push(situation);
-                if (action) inputs.push(action);
-                if (keyLearning) inputs.push(keyLearning);
-
-                const embeddings = await embedTexts(inputs);
-                let embedIndex = 0;
-
-                const situationVector = situation ? (embeddings[embedIndex++] ?? null) : null;
-                const actionVector = action ? (embeddings[embedIndex++] ?? null) : null;
-                const keyLearningVector = keyLearning ? (embeddings[embedIndex++] ?? null) : null;
-
-                await ctx.memoryModel.updateExperienceVectors(row.id, {
-                  actionVector,
-                  keyLearningVector,
-                  situationVector,
-                });
-                succeeded += 1;
-              } catch (err) {
-                failed += 1;
-                console.error(
-                  `[memoryRouter.reEmbed] Failed to re-embed experience ${row.id}`,
-                  err,
-                );
-              }
-            },
-            { concurrency },
-          );
-
-          return {
-            failed,
-            skipped,
-            succeeded,
-            total: rows.length,
-          } satisfies ReEmbedStats;
-        });
-
-        const processedEntries = Object.entries(results) as Array<[ReEmbedTableKey, ReEmbedStats]>;
-
-        if (processedEntries.length === 0) {
-          return {
-            message: 'No memory records matched re-embed criteria',
-            results,
-            success: true,
-          };
-        }
-
-        const aggregate = processedEntries.reduce(
-          (acc, [, stats]) => {
-            acc.failed += stats.failed;
-            acc.skipped += stats.skipped;
-            acc.succeeded += stats.succeeded;
-            acc.total += stats.total;
-
-            return acc;
-          },
-          { failed: 0, skipped: 0, succeeded: 0, total: 0 },
-        );
-
-        const message =
-          aggregate.total === 0
-            ? 'No memory records required re-embedding'
-            : `Re-embedded ${aggregate.succeeded} of ${aggregate.total} records`;
-
-        return {
-          aggregate,
-          message,
-          results,
-          success: true,
-        };
-      } catch (error) {
-        console.error('Failed to re-embed memories:', error);
-        return {
-          message: `Failed to re-embed memories: ${(error as Error).message}`,
-          success: false,
-        };
-      }
-    }),
-});

package/src/server/routers/lambda/userMemories/search.ts

@@ -1,117 +0,0 @@
-import { type LobeChatDatabase } from '@lobechat/database';
-import { type z } from 'zod';
-
-import { DEFAULT_FILE_EMBEDDING_MODEL_ITEM } from '@/const/settings/knowledge';
-import { type UserMemoryModel } from '@/database/models/userMemory';
-import { getServerDefaultFilesConfig } from '@/server/globalConfig';
-import { initModelRuntimeFromDB } from '@/server/modules/ModelRuntime';
-import { type SearchMemoryResult, searchMemorySchema } from '@/types/userMemory';
-
-import { EMBEDDING_VECTOR_DIMENSION, memoryProcedure, router } from './shared';
-
-type MemorySearchContext = {
-  memoryModel: UserMemoryModel;
-  serverDB: LobeChatDatabase;
-  userId: string;
-};
-
-type MemorySearchResult = Awaited<ReturnType<UserMemoryModel['searchWithEmbedding']>>;
-
-const EMPTY_SEARCH_RESULT: SearchMemoryResult = {
-  contexts: [],
-  experiences: [],
-  preferences: [],
-};
-
-const mapMemorySearchResult = (layeredResults: MemorySearchResult): SearchMemoryResult => {
-  return {
-    contexts: layeredResults.contexts.map((context) => ({
-      accessedAt: context.accessedAt,
-      associatedObjects: context.associatedObjects,
-      associatedSubjects: context.associatedSubjects,
-      createdAt: context.createdAt,
-      currentStatus: context.currentStatus,
-      description: context.description,
-      id: context.id,
-      metadata: context.metadata,
-      scoreImpact: context.scoreImpact,
-      scoreUrgency: context.scoreUrgency,
-      tags: context.tags,
-      title: context.title,
-      type: context.type,
-      updatedAt: context.updatedAt,
-      userMemoryIds: Array.isArray(context.userMemoryIds)
-        ? (context.userMemoryIds as string[])
-        : null,
-    })),
-    experiences: layeredResults.experiences.map((experience) => ({
-      accessedAt: experience.accessedAt,
-      action: experience.action,
-      createdAt: experience.createdAt,
-      id: experience.id,
-      keyLearning: experience.keyLearning,
-      metadata: experience.metadata,
-      possibleOutcome: experience.possibleOutcome,
-      reasoning: experience.reasoning,
-      scoreConfidence: experience.scoreConfidence,
-      situation: experience.situation,
-      tags: experience.tags,
-      type: experience.type,
-      updatedAt: experience.updatedAt,
-      userMemoryId: experience.userMemoryId,
-    })),
-    preferences: layeredResults.preferences.map((preference) => ({
-      accessedAt: preference.accessedAt,
-      conclusionDirectives: preference.conclusionDirectives,
-      createdAt: preference.createdAt,
-      id: preference.id,
-      metadata: preference.metadata,
-      scorePriority: preference.scorePriority,
-      suggestions: preference.suggestions,
-      tags: preference.tags,
-      type: preference.type,
-      updatedAt: preference.updatedAt,
-      userMemoryId: preference.userMemoryId,
-    })),
-  } satisfies SearchMemoryResult;
-};
-
-export const searchUserMemories = async (
-  ctx: MemorySearchContext,
-  input: z.infer<typeof searchMemorySchema>,
-): Promise<SearchMemoryResult> => {
-  const { provider, model: embeddingModel } =
-    getServerDefaultFilesConfig().embeddingModel || DEFAULT_FILE_EMBEDDING_MODEL_ITEM;
-  // Read user's provider config from database
-  const agentRuntime = await initModelRuntimeFromDB(ctx.serverDB, ctx.userId, provider);
-
-  const queryEmbeddings = await agentRuntime.embeddings({
-    dimensions: EMBEDDING_VECTOR_DIMENSION,
-    input: input.query,
-    model: embeddingModel,
-  });
-
-  const limits = {
-    contexts: input.topK?.contexts,
-    experiences: input.topK?.experiences,
-    preferences: input.topK?.preferences,
-  };
-
-  const layeredResults = await ctx.memoryModel.searchWithEmbedding({
-    embedding: queryEmbeddings?.[0],
-    limits,
-  });
-
-  return mapMemorySearchResult(layeredResults);
-};
-
-export const searchRouter = router({
-  searchMemory: memoryProcedure.input(searchMemorySchema).query(async ({ input, ctx }) => {
-    try {
-      return await searchUserMemories(ctx, input);
-    } catch (error) {
-      console.error('Failed to retrieve memories:', error);
-      return EMPTY_SEARCH_RESULT;
-    }
-  }),
-});

package/src/server/routers/lambda/userMemories/shared.ts

@@ -1,63 +0,0 @@
-import { type LobeChatDatabase } from '@lobechat/database';
-import { type SQL, and } from 'drizzle-orm';
-
-import { DEFAULT_FILE_EMBEDDING_MODEL_ITEM } from '@/const/settings/knowledge';
-import { UserMemoryModel } from '@/database/models/userMemory';
-import { authedProcedure } from '@/libs/trpc/lambda';
-import { keyVaults, serverDatabase } from '@/libs/trpc/lambda/middleware';
-import { getServerDefaultFilesConfig } from '@/server/globalConfig';
-import { initModelRuntimeFromDB } from '@/server/modules/ModelRuntime';
-
-export const EMBEDDING_VECTOR_DIMENSION = 1024;
-
-export const memoryProcedure = authedProcedure
-  .use(serverDatabase)
-  .use(keyVaults)
-  .use(async (opts) => {
-    const { ctx } = opts;
-    return opts.next({
-      ctx: {
-        memoryModel: new UserMemoryModel(ctx.serverDB, ctx.userId),
-      },
-    });
-  });
-
-export const getEmbeddingRuntime = async (serverDB: LobeChatDatabase, userId: string) => {
-  const { provider, model: embeddingModel } =
-    getServerDefaultFilesConfig().embeddingModel || DEFAULT_FILE_EMBEDDING_MODEL_ITEM;
-  // Read user's provider config from database
-  const agentRuntime = await initModelRuntimeFromDB(serverDB, userId, provider);
-
-  return { agentRuntime, embeddingModel };
-};
-
-export const createEmbedder = (agentRuntime: any, embeddingModel: string) => {
-  return async (value?: string | null): Promise<number[] | undefined> => {
-    if (!value || value.trim().length === 0) return undefined;
-
-    const embeddings = await agentRuntime.embeddings({
-      dimensions: EMBEDDING_VECTOR_DIMENSION,
-      input: value,
-      model: embeddingModel,
-    });
-
-    return embeddings?.[0];
-  };
-};
-
-export const combineConditions = (conditions: Array<SQL | undefined>): SQL | undefined => {
-  const filtered = conditions.filter((condition): condition is SQL => condition !== undefined);
-  if (filtered.length === 0) return undefined;
-  if (filtered.length === 1) return filtered[0];
-
-  return and(...filtered);
-};
-
-export const normalizeEmbeddable = (value?: string | null): string | undefined => {
-  if (typeof value !== 'string') return undefined;
-  const trimmed = value.trim();
-
-  return trimmed.length > 0 ? trimmed : undefined;
-};
-
-export { router } from '@/libs/trpc/lambda';