@lobehub/lobehub 2.0.0-next.294 → 2.0.0-next.295

package/src/app/(backend)/api/workflows/memory-user-memory/pipelines/chat-topic/process-topics/route.ts

@@ -4,143 +4,145 @@ import {
   tracer as upstashWorkflowTracer,
 } from '@lobechat/observability-otel/modules/upstash-workflow';
 import { LayersEnum, MemorySourceType } from '@lobechat/types';
-import { serve } from '@upstash/workflow/nextjs';
+import { Client } from '@upstash/qstash';
 import { WorkflowAbort } from '@upstash/workflow';
+import { serve } from '@upstash/workflow/nextjs';
 
+import { parseMemoryExtractionConfig } from '@/server/globalConfig/parseMemoryExtractionConfig';
 import {
   MemoryExtractionExecutor,
   type MemoryExtractionPayloadInput,
   normalizeMemoryExtractionPayload,
 } from '@/server/services/memory/userMemory/extract';
-import { Client } from '@upstash/qstash'
-import { parseMemoryExtractionConfig } from '@/server/globalConfig/parseMemoryExtractionConfig';
 
 const { upstashWorkflowExtraHeaders } = parseMemoryExtractionConfig();
 
 const CEP_LAYERS: LayersEnum[] = [LayersEnum.Context, LayersEnum.Experience, LayersEnum.Preference];
 const IDENTITY_LAYERS: LayersEnum[] = [LayersEnum.Identity];
 
-export const { POST } = serve<MemoryExtractionPayloadInput>((context) =>
-  upstashWorkflowTracer.startActiveSpan(
-    'workflow:memory-user-memory:process-topics',
-    async (span) => {
-      const payload = normalizeMemoryExtractionPayload(context.requestPayload || {});
-
-      span.setAttributes({
-        ...buildUpstashWorkflowAttributes(context),
-        'workflow.memory_user_memory.force_all': payload.forceAll,
-        'workflow.memory_user_memory.force_topics': payload.forceTopics,
-        'workflow.memory_user_memory.layers': payload.layers.join(','),
-        'workflow.memory_user_memory.source': payload.sources.join(','),
-        'workflow.memory_user_memory.topic_count': payload.topicIds.length,
-        'workflow.memory_user_memory.user_count': payload.userIds.length,
-        'workflow.name': 'memory-user-memory:process-topics',
-      });
-
-      try {
-        console.log('[chat-topic][batch] Starting batch topic processing workflow', {
-          topicIds: payload.topicIds,
-          userIds: payload.userIds,
+export const { POST } = serve<MemoryExtractionPayloadInput>(
+  (context) =>
+    upstashWorkflowTracer.startActiveSpan(
+      'workflow:memory-user-memory:process-topics',
+      async (span) => {
+        const payload = normalizeMemoryExtractionPayload(context.requestPayload || {});
+
+        span.setAttributes({
+          ...buildUpstashWorkflowAttributes(context),
+          'workflow.memory_user_memory.force_all': payload.forceAll,
+          'workflow.memory_user_memory.force_topics': payload.forceTopics,
+          'workflow.memory_user_memory.layers': payload.layers.join(','),
+          'workflow.memory_user_memory.source': payload.sources.join(','),
+          'workflow.memory_user_memory.topic_count': payload.topicIds.length,
+          'workflow.memory_user_memory.user_count': payload.userIds.length,
+          'workflow.name': 'memory-user-memory:process-topics',
         });
 
-        if (!payload.userIds.length) {
-          span.setStatus({ code: SpanStatusCode.OK });
-
-          return {
-            message: 'No user id provided for topic batch.',
-            processedTopics: 0,
-            processedUsers: 0,
-          };
-        }
-        if (!payload.topicIds.length) {
-          span.setStatus({ code: SpanStatusCode.OK });
-
-          return {
-            message: 'No topic ids provided for extraction.',
-            processedTopics: 0,
-            processedUsers: 0,
-          };
-        }
-        if (!payload.sources.includes(MemorySourceType.ChatTopic)) {
-          span.setStatus({ code: SpanStatusCode.OK });
-
-          return {
-            message: 'Source not supported in topic batch.',
-            processedTopics: 0,
-            processedUsers: 0,
-          };
-        }
-
-        const userId = payload.userIds[0];
-        const executor = await MemoryExtractionExecutor.create();
+        try {
+          console.log('[chat-topic][batch] Starting batch topic processing workflow', {
+            topicIds: payload.topicIds,
+            userIds: payload.userIds,
+          });
+
+          if (!payload.userIds.length) {
+            span.setStatus({ code: SpanStatusCode.OK });
+
+            return {
+              message: 'No user id provided for topic batch.',
+              processedTopics: 0,
+              processedUsers: 0,
+            };
+          }
+          if (!payload.topicIds.length) {
+            span.setStatus({ code: SpanStatusCode.OK });
+
+            return {
+              message: 'No topic ids provided for extraction.',
+              processedTopics: 0,
+              processedUsers: 0,
+            };
+          }
+          if (!payload.sources.includes(MemorySourceType.ChatTopic)) {
+            span.setStatus({ code: SpanStatusCode.OK });
+
+            return {
+              message: 'Source not supported in topic batch.',
+              processedTopics: 0,
+              processedUsers: 0,
+            };
+          }
+
+          const userId = payload.userIds[0];
+          const executor = await MemoryExtractionExecutor.create();
+
+          // CEP: run in parallel across the batch
+          await Promise.all(
+            payload.topicIds.map((topicId, index) =>
+              context.run(
+                `memory:user-memory:extract:users:${userId}:topics:${topicId}:cep:${index}`,
+                () =>
+                  executor.extractTopic({
+                    forceAll: payload.forceAll,
+                    forceTopics: payload.forceTopics,
+                    from: payload.from,
+                    layers: CEP_LAYERS,
+                    source: MemorySourceType.ChatTopic,
+                    to: payload.to,
+                    topicId,
+                    userId,
+                  }),
+              ),
+            ),
+          );
 
-        // CEP: run in parallel across the batch
-        await Promise.all(
-          payload.topicIds.map((topicId, index) =>
-            context.run(
-              `memory:user-memory:extract:users:${userId}:topics:${topicId}:cep:${index}`,
+          // Identity: run sequentially for the batch
+          for (const [index, topicId] of payload.topicIds.entries()) {
+            await context.run(
+              `memory:user-memory:extract:users:${userId}:topics:${topicId}:identity:${index}`,
               () =>
                 executor.extractTopic({
                   forceAll: payload.forceAll,
                   forceTopics: payload.forceTopics,
                   from: payload.from,
-                  layers: CEP_LAYERS,
+                  layers: IDENTITY_LAYERS,
                   source: MemorySourceType.ChatTopic,
                   to: payload.to,
                   topicId,
                   userId,
                 }),
-            ),
-          ),
-        );
-
-        // Identity: run sequentially for the batch
-        for (const [index, topicId] of payload.topicIds.entries()) {
-          await context.run(
-            `memory:user-memory:extract:users:${userId}:topics:${topicId}:identity:${index}`,
-            () =>
-              executor.extractTopic({
-                forceAll: payload.forceAll,
-                forceTopics: payload.forceTopics,
-                from: payload.from,
-                layers: IDENTITY_LAYERS,
-                source: MemorySourceType.ChatTopic,
-                to: payload.to,
-                topicId,
-                userId,
-              }),
-          );
-        }
+            );
+          }
 
-        console.log('[chat-topic][batch] Batch topic processing workflow completed', {
-          processedTopics: payload.topicIds.length,
-        });
+          console.log('[chat-topic][batch] Batch topic processing workflow completed', {
+            processedTopics: payload.topicIds.length,
+          });
 
-        span.setStatus({ code: SpanStatusCode.OK });
+          span.setStatus({ code: SpanStatusCode.OK });
+
+          return {
+            processedTopics: payload.topicIds.length,
+            processedUsers: payload.userIds.length,
+          };
+        } catch (error) {
+          // NOTICE: Let WorkflowAbort bubble up (used internally by Upstash); record others
+          if (error instanceof WorkflowAbort) {
+            console.warn('workflow aborted:', error.message);
+            throw error;
+          }
+
+          span.recordException(error as Error);
+          span.setStatus({
+            code: SpanStatusCode.ERROR,
+            message: error instanceof Error ? error.message : 'process-topics workflow failed',
+          });
 
-        return {
-          processedTopics: payload.topicIds.length,
-          processedUsers: payload.userIds.length,
-        };
-      } catch (error) {
-        // NOTICE: Let WorkflowAbort bubble up (used internally by Upstash); record others
-        if (error instanceof WorkflowAbort) {
-          console.warn('workflow aborted:', error.message);
           throw error;
+        } finally {
+          span.end();
         }
-
-        span.recordException(error as Error);
-        span.setStatus({
-          code: SpanStatusCode.ERROR,
-          message: error instanceof Error ? error.message : 'process-topics workflow failed',
-        });
-
-        throw error;
-      } finally {
-        span.end();
-      }
-    },
-  ), {
+      },
+    ),
+  {
     // NOTICE(@nekomeowww): Here as scenarios like Vercel Deployment Protection,
     // intermediate context.run(...) won't offer customizable headers like context.trigger(...) / client.trigger(...)
     // for passing additional headers, we have to provide a custom QStash client with the required headers here.
@@ -151,6 +153,7 @@ export const { POST } = serve<MemoryExtractionPayloadInput>((context) =>
       headers: {
         ...upstashWorkflowExtraHeaders,
       },
-      token: process.env.QSTASH_TOKEN!
-    })
-});
+      token: process.env.QSTASH_TOKEN!,
+    }),
+  },
+);

package/src/app/(backend)/api/workflows/memory-user-memory/pipelines/chat-topic/process-user-topics/route.ts

@@ -1,5 +1,9 @@
+import { MemorySourceType } from '@lobechat/types';
+import { Client } from '@upstash/qstash';
 import { serve } from '@upstash/workflow/nextjs';
+
 import type { ListTopicsForMemoryExtractorCursor } from '@/database/models/topic';
+import { parseMemoryExtractionConfig } from '@/server/globalConfig/parseMemoryExtractionConfig';
 import {
   MemoryExtractionExecutor,
   type MemoryExtractionPayloadInput,
@@ -8,135 +12,143 @@ import {
   normalizeMemoryExtractionPayload,
 } from '@/server/services/memory/userMemory/extract';
 import { forEachBatchSequential } from '@/server/services/memory/userMemory/topicBatching';
-import { MemorySourceType } from '@lobechat/types';
-import { parseMemoryExtractionConfig } from '@/server/globalConfig/parseMemoryExtractionConfig';
-import { Client } from '@upstash/qstash'
 
 const TOPIC_PAGE_SIZE = 50;
 const TOPIC_BATCH_SIZE = 4;
 
 const { upstashWorkflowExtraHeaders } = parseMemoryExtractionConfig();
 
-export const { POST } = serve<MemoryExtractionPayloadInput>(async (context) => {
-  const params = normalizeMemoryExtractionPayload(context.requestPayload || {});
-  if (!params.userIds.length) {
-    return { message: 'No user ids provided for topic processing.' };
-  }
-  if (!params.sources.includes(MemorySourceType.ChatTopic)) {
-    return { message: 'No supported sources requested, skip topic processing.' };
-  }
-
-  const executor = await MemoryExtractionExecutor.create();
-
-  const scheduleNextPage = async (userId: string, cursorCreatedAt: Date, cursorId: string) => {
-    await MemoryExtractionWorkflowService.triggerProcessUserTopics({
-      ...buildWorkflowPayloadInput({
-        ...params,
-        topicCursor: {
-          createdAt: cursorCreatedAt.toISOString(),
-          id: cursorId,
-          userId,
-        },
-        topicIds: [],
-        userId,
-        userIds: [userId],
-      }),
-    }, { extraHeaders: upstashWorkflowExtraHeaders });
-  };
-
-  for (const userId of params.userIds) {
-    const topicCursor =
-      params.topicCursor && params.topicCursor.userId === userId
-        ? {
-            createdAt: new Date(params.topicCursor.createdAt),
-            id: params.topicCursor.id,
-          }
-        : undefined;
+export const { POST } = serve<MemoryExtractionPayloadInput>(
+  async (context) => {
+    const params = normalizeMemoryExtractionPayload(context.requestPayload || {});
+    if (!params.userIds.length) {
+      return { message: 'No user ids provided for topic processing.' };
+    }
+    if (!params.sources.includes(MemorySourceType.ChatTopic)) {
+      return { message: 'No supported sources requested, skip topic processing.' };
+    }
 
-    const topicsFromPayload =
-      params.topicIds && params.topicIds.length > 0
-        ? await context.run(
-            `memory:user-memory:extract:users:${userId}:filter-topic-ids`,
-            async () => {
-              const filtered = await executor.filterTopicIdsForUser(userId, params.topicIds);
-              return filtered.length > 0 ? filtered : undefined;
-            },
-          )
-        : undefined;
+    const executor = await MemoryExtractionExecutor.create();
 
-    const topicBatch = await context.run<{
-      cursor?: ListTopicsForMemoryExtractorCursor;
-      ids: string[];
-    }>(`memory:user-memory:extract:users:${userId}:list-topics:${topicCursor?.id || 'root'}`, () =>
-      topicsFromPayload && topicsFromPayload.length > 0
-        ? Promise.resolve({ ids: topicsFromPayload })
-        : executor.getTopicsForUser(
-            {
-              cursor: topicCursor,
-              forceAll: params.forceAll,
-              forceTopics: params.forceTopics,
-              from: params.from,
-              to: params.to,
+    const scheduleNextPage = async (userId: string, cursorCreatedAt: Date, cursorId: string) => {
+      await MemoryExtractionWorkflowService.triggerProcessUserTopics(
+        {
+          ...buildWorkflowPayloadInput({
+            ...params,
+            topicCursor: {
+              createdAt: cursorCreatedAt.toISOString(),
+              id: cursorId,
               userId,
             },
-            TOPIC_PAGE_SIZE,
-          ),
-    );
+            topicIds: [],
+            userId,
+            userIds: [userId],
+          }),
+        },
+        { extraHeaders: upstashWorkflowExtraHeaders },
+      );
+    };
 
-    const ids = topicBatch.ids;
-    if (!ids.length) {
-      continue;
-    }
+    for (const userId of params.userIds) {
+      const topicCursor =
+        params.topicCursor && params.topicCursor.userId === userId
+          ? {
+              createdAt: new Date(params.topicCursor.createdAt),
+              id: params.topicCursor.id,
+            }
+          : undefined;
 
-    const cursor = 'cursor' in topicBatch ? topicBatch.cursor : undefined;
+      const topicsFromPayload =
+        params.topicIds && params.topicIds.length > 0
+          ? await context.run(
+              `memory:user-memory:extract:users:${userId}:filter-topic-ids`,
+              async () => {
+                const filtered = await executor.filterTopicIdsForUser(userId, params.topicIds);
+                return filtered.length > 0 ? filtered : undefined;
+              },
+            )
+          : undefined;
 
-    await forEachBatchSequential(ids, TOPIC_BATCH_SIZE, async (topicIds, batchIndex) => {
-      // NOTICE: We trigger via QStash instead of context.invoke because invoke only swaps the last path
-      // segment with the workflowId. If we invoked directly from /process-user-topics, child workflow
-      // URLs would inherit that base and lose the desired /process-topics/workflows prefix.
-      await context.run(
-        `memory:user-memory:extract:users:${userId}:process-topics-batch:${batchIndex}`,
+      const topicBatch = await context.run<{
+        cursor?: ListTopicsForMemoryExtractorCursor;
+        ids: string[];
+      }>(
+        `memory:user-memory:extract:users:${userId}:list-topics:${topicCursor?.id || 'root'}`,
         () =>
-          MemoryExtractionWorkflowService.triggerProcessTopics({
-            ...buildWorkflowPayloadInput(params),
-            topicCursor: undefined,
-            topicIds,
-            userId,
-            userIds: [userId],
-          }, { extraHeaders: upstashWorkflowExtraHeaders }),
+          topicsFromPayload && topicsFromPayload.length > 0
+            ? Promise.resolve({ ids: topicsFromPayload })
+            : executor.getTopicsForUser(
+                {
+                  cursor: topicCursor,
+                  forceAll: params.forceAll,
+                  forceTopics: params.forceTopics,
+                  from: params.from,
+                  to: params.to,
+                  userId,
+                },
+                TOPIC_PAGE_SIZE,
+              ),
       );
-    });
 
-    if (!topicsFromPayload && cursor) {
-      await context.run(
-        `memory:user-memory:extract:users:${userId}:topics:${cursor.id}:schedule-next-batch`,
-        () => {
-          // NOTICE: Upstash Workflow only supports serializable data into plain JSON,
-          // this causes the Date object to be converted into string when passed as parameter from
-          // context to child workflow. So we need to convert it back to Date object here.
-          const createdAt = new Date(cursor.createdAt);
-          if (Number.isNaN(createdAt.getTime())) {
-            throw new Error('Invalid cursor date when scheduling next topic page');
-          }
+      const ids = topicBatch.ids;
+      if (!ids.length) {
+        continue;
+      }
 
-          scheduleNextPage(userId, createdAt, cursor.id);
-        },
-      );
+      const cursor = 'cursor' in topicBatch ? topicBatch.cursor : undefined;
+
+      await forEachBatchSequential(ids, TOPIC_BATCH_SIZE, async (topicIds, batchIndex) => {
+        // NOTICE: We trigger via QStash instead of context.invoke because invoke only swaps the last path
+        // segment with the workflowId. If we invoked directly from /process-user-topics, child workflow
+        // URLs would inherit that base and lose the desired /process-topics/workflows prefix.
+        await context.run(
+          `memory:user-memory:extract:users:${userId}:process-topics-batch:${batchIndex}`,
+          () =>
+            MemoryExtractionWorkflowService.triggerProcessTopics(
+              {
+                ...buildWorkflowPayloadInput(params),
+                topicCursor: undefined,
+                topicIds,
+                userId,
+                userIds: [userId],
+              },
+              { extraHeaders: upstashWorkflowExtraHeaders },
+            ),
+        );
+      });
+
+      if (!topicsFromPayload && cursor) {
+        await context.run(
+          `memory:user-memory:extract:users:${userId}:topics:${cursor.id}:schedule-next-batch`,
+          () => {
+            // NOTICE: Upstash Workflow only supports serializable data into plain JSON,
+            // this causes the Date object to be converted into string when passed as parameter from
+            // context to child workflow. So we need to convert it back to Date object here.
+            const createdAt = new Date(cursor.createdAt);
+            if (Number.isNaN(createdAt.getTime())) {
+              throw new Error('Invalid cursor date when scheduling next topic page');
+            }
+
+            scheduleNextPage(userId, createdAt, cursor.id);
+          },
+        );
+      }
     }
-  }
 
-  return { processedUsers: params.userIds.length };
-}, {
-  // NOTICE(@nekomeowww): Here as scenarios like Vercel Deployment Protection,
-  // intermediate context.run(...) won't offer customizable headers like context.trigger(...) / client.trigger(...)
-  // for passing additional headers, we have to provide a custom QStash client with the required headers here.
-  //
-  // Refer to the doc for more details:
-  // https://upstash.com/docs/workflow/troubleshooting/vercel#step-2-pass-header-when-triggering
-  qstashClient: new Client({
-    headers: {
-      ...upstashWorkflowExtraHeaders,
-    },
-    token: process.env.QSTASH_TOKEN!
-  })
-});
+    return { processedUsers: params.userIds.length };
+  },
+  {
+    // NOTICE(@nekomeowww): Here as scenarios like Vercel Deployment Protection,
+    // intermediate context.run(...) won't offer customizable headers like context.trigger(...) / client.trigger(...)
+    // for passing additional headers, we have to provide a custom QStash client with the required headers here.
+    //
+    // Refer to the doc for more details:
+    // https://upstash.com/docs/workflow/troubleshooting/vercel#step-2-pass-header-when-triggering
+    qstashClient: new Client({
+      headers: {
+        ...upstashWorkflowExtraHeaders,
+      },
+      token: process.env.QSTASH_TOKEN!,
+    }),
+  },
+);