@lobehub/lobehub 2.0.0-next.360 → 2.0.0-next.362

package/packages/memory-user-memory/promptfoo/evals/activity/basic/buildMessages.ts

@@ -0,0 +1,40 @@
+import { renderPlaceholderTemplate } from '@lobechat/context-engine';
+
+import { activityPrompt } from '../../../../src/prompts';
+import type { ExtractorTemplateProps } from '../../../../src/types';
+
+export interface PromptVars extends ExtractorTemplateProps {
+  conversation: string;
+}
+
+export const buildActivityMessages = (vars: PromptVars) => {
+  const retrievedContext =
+    Array.isArray(vars.retrievedContexts) && vars.retrievedContexts.length > 0
+      ? vars.retrievedContexts.join('\n\n')
+      : typeof vars.retrievedContexts === 'string'
+        ? vars.retrievedContexts
+        : 'No similar memories retrieved.';
+
+  const rendered = renderPlaceholderTemplate(activityPrompt, {
+    availableCategories: vars.availableCategories,
+    language: vars.language || 'English',
+    retrievedContext,
+    sessionDate: vars.sessionDate || new Date().toISOString(),
+    topK: vars.topK ?? 5,
+    username: vars.username || 'User',
+  });
+
+  const messages = [
+    { content: rendered, role: 'system' as const },
+    { content: rendered, role: 'user' as const },
+  ];
+
+  if (vars.conversation) {
+    messages.push({
+      content: `Conversation:\n${vars.conversation}`,
+      role: 'user' as const,
+    });
+  }
+
+  return messages;
+};

package/packages/memory-user-memory/promptfoo/evals/activity/basic/eval.yaml

@@ -0,0 +1,13 @@
+description: Regression benchmark for activity layer structured extraction
+
+providers:
+  - id: openai:chat:gpt-5-mini
+    config:
+      response_format: file://../../../response-formats/activity.json
+      temperature: 0
+
+prompts:
+  - file://./prompt.ts
+
+tests:
+  - file://./tests/cases.ts

package/packages/memory-user-memory/promptfoo/evals/activity/basic/prompt.ts

@@ -0,0 +1,5 @@
+import { buildActivityMessages, PromptVars } from './buildMessages';
+
+export default function generatePrompt({ vars }: { vars: PromptVars }) {
+  return buildActivityMessages(vars);
+}

package/packages/memory-user-memory/promptfoo/evals/activity/basic/tests/cases.ts

@@ -0,0 +1,106 @@
+type PromptfooAssert =
+  | { type: 'javascript'; value: string }
+  | { provider?: string; type: 'llm-rubric'; value: string };
+
+interface PromptfooTestCase {
+  assert: PromptfooAssert[];
+  description?: string;
+  vars: Record<string, unknown>;
+}
+
+const baseSchemaAssert: PromptfooAssert = {
+  type: 'javascript',
+  value: `
+    let parsed;
+    try {
+      parsed = JSON.parse(output);
+    } catch (error) {
+      console.error('Failed to parse JSON output', error);
+      return false;
+    }
+
+    if (!parsed || !Array.isArray(parsed.memories)) return false;
+
+    return parsed.memories.every((memory) => {
+      return (
+        memory.memoryType === 'activity' &&
+        memory.title &&
+        memory.summary &&
+        memory.withActivity?.type &&
+        memory.withActivity?.narrative
+      );
+    });
+  `,
+};
+
+const baseVars = {
+  availableCategories: ['work', 'health', 'personal'],
+  language: 'English',
+  topK: 5,
+  username: 'User',
+};
+
+const testCases: PromptfooTestCase[] = [
+  {
+    assert: [
+      baseSchemaAssert,
+      {
+        type: 'javascript',
+        value: `
+          const data = JSON.parse(output);
+          const first = data.memories?.[0];
+          if (!first) return false;
+
+          const activity = first.withActivity || {};
+          return Boolean(activity.startsAt && activity.endsAt && activity.timezone && activity.associatedLocations?.[0]?.name);
+        `,
+      },
+      {
+        provider: 'openai:gpt-5-mini',
+        type: 'llm-rubric',
+        value:
+          'Should extract a meeting activity including timing (start/end/timezone), location name ACME HQ, status completed when implied, and feedback reflecting the positive tone.',
+      },
+    ],
+    description: 'Meeting with explicit time and location',
+    vars: {
+      ...baseVars,
+      conversation:
+        'User: I met with Alice at ACME HQ on 2024-05-03 from 14:00-15:00 America/New_York. We reviewed Q2 renewal scope and agreed to send revised pricing next week. I felt positive and collaborative about the call.',
+      retrievedContexts: ['Previous similar memory: met with Alice about renewal last month.'],
+      sessionDate: '2024-05-03',
+    },
+  },
+  {
+    assert: [
+      baseSchemaAssert,
+      {
+        type: 'javascript',
+        value: `
+          const data = JSON.parse(output);
+          const first = data.memories?.[0];
+          if (!first) return false;
+
+          const activity = first.withActivity || {};
+          return Boolean(activity.narrative && activity.feedback);
+        `,
+      },
+      {
+        provider: 'openai:gpt-5-mini',
+        type: 'llm-rubric',
+        value:
+          'Should capture an exercise activity without inventing exact timestamps or timezones; keep the narrative and feedback about the yoga session at home and omit temporal fields that were not provided.',
+      },
+    ],
+    description: 'Exercise without explicit time or timezone',
+    vars: {
+      ...baseVars,
+      conversation:
+        'User: Over the weekend I did a 30-minute yoga session at home with my roommate. No specific time was set, it was just a casual stretch and it left me feeling calm.',
+      retrievedContexts: [],
+      sessionDate: '2025-05-05 10:02:00',
+    },
+  },
+];
+
+export default testCases;

package/packages/memory-user-memory/promptfoo/evals/activity/locomo/buildMessages.ts

@@ -0,0 +1,104 @@
+import { renderPlaceholderTemplate } from '@lobechat/context-engine';
+import { MemorySourceType } from '@lobechat/types';
+import { readFile } from 'node:fs/promises';
+import { isAbsolute, join } from 'node:path';
+
+import { BenchmarkLocomoContextProvider, BenchmarkLocomoPart } from '../../../../src/providers';
+import type { IngestPayload } from '../../../../src/converters/locomo';
+import { activityPrompt } from '../../../../src/prompts';
+import type { ExtractorTemplateProps, MemoryExtractionJob } from '../../../../src/types';
+
+export interface PromptVars extends ExtractorTemplateProps {
+  payloadPath: string;
+  sessionId?: string;
+  userId?: string;
+}
+
+const resolvePath = (payloadPath: string) =>
+  isAbsolute(payloadPath) ? payloadPath : join(process.cwd(), payloadPath);
+
+const buildParts = (payload: IngestPayload, sessionId?: string): BenchmarkLocomoPart[] => {
+  let partIndex = 0;
+  const sessions = payload.sessions.filter(
+    (session) => !sessionId || session.sessionId === sessionId,
+  );
+
+  return sessions.flatMap((session) =>
+    session.turns.map((turn) => {
+      const metadata = {
+        diaId: turn.diaId,
+        imageCaption: turn.imageCaption,
+        imageUrls: turn.imageUrls,
+        sessionId: session.sessionId,
+      };
+
+      return {
+        content: turn.text,
+        createdAt: turn.createdAt || session.timestamp,
+        metadata,
+        partIndex: partIndex++,
+        sessionId: session.sessionId,
+        speaker: turn.speaker,
+      };
+    }),
+  );
+};
+
+const resolveSessionDate = (payload: IngestPayload, parts: BenchmarkLocomoPart[], sessionId?: string) => {
+  const sessionDate =
+    payload.sessions.find((session) => session.sessionId === sessionId)?.timestamp ||
+    payload.sessions[0]?.timestamp;
+
+  if (sessionDate) return sessionDate;
+
+  const latestCreatedAt = parts
+    .map((part) => (part.createdAt ? new Date(part.createdAt) : null))
+    .filter(Boolean)
+    .sort((a, b) => (a!.getTime() > b!.getTime() ? 1 : -1))
+    .at(-1);
+
+  return latestCreatedAt ? latestCreatedAt.toISOString() : new Date().toISOString();
+};
+
+export const buildLocomoActivityMessages = async (vars: PromptVars) => {
+  const payloadPath = resolvePath(vars.payloadPath);
+  const payloadRaw = await readFile(payloadPath, 'utf8');
+  const payload = JSON.parse(payloadRaw) as IngestPayload;
+
+  const parts = buildParts(payload, vars.sessionId);
+  if (parts.length === 0) {
+    throw new Error(`No matching parts found in ${payload.sampleId} for session ${vars.sessionId || 'all'}`);
+  }
+  const userId = vars.userId || `locomo-user-${payload.sampleId}`;
+  const sourceId = payload.topicId || `sample_${payload.sampleId}`;
+  const sessionDate = vars.sessionDate || resolveSessionDate(payload, parts, vars.sessionId);
+
+  const provider = new BenchmarkLocomoContextProvider({
+    parts,
+    sampleId: payload.sampleId,
+    sourceId,
+    userId,
+  });
+
+  const extractionJob: MemoryExtractionJob = {
+    source: MemorySourceType.BenchmarkLocomo,
+    sourceId,
+    userId,
+  };
+
+  const { context } = await provider.buildContext(extractionJob);
+
+  const rendered = renderPlaceholderTemplate(activityPrompt, {
+    availableCategories: vars.availableCategories,
+    language: vars.language || 'English',
+    retrievedContext: context || 'No similar memories retrieved.',
+    sessionDate,
+    topK: vars.topK ?? 5,
+    username: vars.username || 'User',
+  });
+
+  return [
+    { content: rendered, role: 'system' as const },
+    { content: rendered, role: 'user' as const },
+  ];
+};

package/packages/memory-user-memory/promptfoo/evals/activity/locomo/eval.yaml

@@ -0,0 +1,13 @@
+description: LoCoMo regression for activity layer with relative time resolution
+
+providers:
+  - id: openai:chat:gpt-5-mini
+    config:
+      response_format: file://../../../response-formats/activity.json
+      temperature: 0
+
+prompts:
+  - file://./prompt.ts
+
+tests:
+  - file://./tests/cases.ts

package/packages/memory-user-memory/promptfoo/evals/activity/locomo/prompt.ts

@@ -0,0 +1,5 @@
+import { buildLocomoActivityMessages, PromptVars } from './buildMessages';
+
+export default async function generatePrompt({ vars }: { vars: PromptVars }) {
+  return buildLocomoActivityMessages(vars);
+}

package/packages/memory-user-memory/promptfoo/evals/activity/locomo/tests/benchmark-locomo-payload-conv-26.json

@@ -0,0 +1,149 @@
+{
+  "force": true,
+  "layers": [],
+  "sampleId": "conv-26",
+  "sessions": [
+    {
+      "sessionId": "session_1",
+      "timestamp": "2023-05-08T13:56:00.000Z",
+      "turns": [
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:1",
+          "role": "user",
+          "speaker": "Caroline",
+          "text": "Hey Mel! Good to see you! How have you been?"
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:2",
+          "role": "assistant",
+          "speaker": "Melanie",
+          "text": "Hey Caroline! Good to see you! I'm swamped with the kids & work. What's up with you? Anything new?"
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:3",
+          "role": "user",
+          "speaker": "Caroline",
+          "text": "I went to a LGBTQ support group yesterday and it was so powerful."
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:4",
+          "role": "assistant",
+          "speaker": "Melanie",
+          "text": "Wow, that's cool, Caroline! What happened that was so awesome? Did you hear any inspiring stories?"
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:5",
+          "imageCaption": "a photo of a dog walking past a wall with a painting of a woman",
+          "imageUrls": [
+            "https://i.redd.it/l7hozpetnhlb1.jpg"
+          ],
+          "role": "user",
+          "speaker": "Caroline",
+          "text": "The transgender stories were so inspiring! I was so happy and thankful for all the support.\n[Image: a photo of a dog walking past a wall with a painting of a woman]"
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:6",
+          "role": "assistant",
+          "speaker": "Melanie",
+          "text": "Wow, love that painting! So cool you found such a helpful group. What's it done for you?"
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:7",
+          "role": "user",
+          "speaker": "Caroline",
+          "text": "The support group has made me feel accepted and given me courage to embrace myself."
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:8",
+          "role": "assistant",
+          "speaker": "Melanie",
+          "text": "That's really cool. You've got guts. What now?"
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:9",
+          "role": "user",
+          "speaker": "Caroline",
+          "text": "Gonna continue my edu and check out career options, which is pretty exciting!"
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:10",
+          "role": "assistant",
+          "speaker": "Melanie",
+          "text": "Wow, Caroline! What kinda jobs are you thinkin' of? Anything that stands out?"
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:11",
+          "role": "user",
+          "speaker": "Caroline",
+          "text": "I'm keen on counseling or working in mental health - I'd love to support those with similar issues."
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:12",
+          "imageCaption": "a photo of a painting of a sunset over a lake",
+          "imageUrls": [
+            "http://candicealexander.com/cdn/shop/products/IMG_7269_a49d5af8-c76c-4ecd-ae20-48c08cb11dec.jpg"
+          ],
+          "role": "assistant",
+          "speaker": "Melanie",
+          "text": "You'd be a great counselor! Your empathy and understanding will really help the people you work with. By the way, take a look at this.\n[Image: a photo of a painting of a sunset over a lake]"
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:13",
+          "role": "user",
+          "speaker": "Caroline",
+          "text": "Thanks, Melanie! That's really sweet. Is this your own painting?"
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:14",
+          "role": "assistant",
+          "speaker": "Melanie",
+          "text": "Yeah, I painted that lake sunrise last year! It's special to me."
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:15",
+          "role": "user",
+          "speaker": "Caroline",
+          "text": "Wow, Melanie! The colors really blend nicely. Painting looks like a great outlet for expressing yourself."
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:16",
+          "role": "assistant",
+          "speaker": "Melanie",
+          "text": "Thanks, Caroline! Painting's a fun way to express my feelings and get creative. It's a great way to relax after a long day."
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:17",
+          "role": "user",
+          "speaker": "Caroline",
+          "text": "Totally agree, Mel. Relaxing and expressing ourselves is key. Well, I'm off to go do some research."
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:18",
+          "role": "assistant",
+          "speaker": "Melanie",
+          "text": "Yep, Caroline. Taking care of ourselves is vital. I'm off to go swimming with the kids. Talk to you soon!"
+        }
+      ]
+    }
+  ],
+  "source": "benchmark_locomo",
+  "topicId": "sample_conv-26"
+}

package/packages/memory-user-memory/promptfoo/evals/activity/locomo/tests/cases.ts

@@ -0,0 +1,72 @@
+type PromptfooAssert =
+  | { type: 'javascript'; value: string }
+  | { provider?: string; type: 'llm-rubric'; value: string };
+
+interface PromptfooTestCase {
+  assert: PromptfooAssert[];
+  description?: string;
+  vars: Record<string, unknown>;
+}
+
+const baseSchemaAssert: PromptfooAssert = {
+  type: 'javascript',
+  value: `
+    let parsed;
+    try {
+      parsed = JSON.parse(output);
+    } catch (error) {
+      console.error('Failed to parse JSON output', error);
+      return false;
+    }
+
+    if (!parsed || !Array.isArray(parsed.memories)) return false;
+
+    return parsed.memories.every((memory) => {
+      return memory.memoryType === 'activity' && memory.withActivity?.type;
+    });
+  `,
+};
+
+const testCases: PromptfooTestCase[] = [
+  {
+    assert: [
+      baseSchemaAssert,
+      {
+        type: 'javascript',
+        value: `
+          const data = JSON.parse(output);
+          const target = data.memories?.find((memory) => {
+            const text = [memory.title, memory.summary, memory.withActivity?.narrative]
+              .filter(Boolean)
+              .join(' ')
+              .toLowerCase();
+            return text.includes('support group');
+          });
+
+          if (!target) return false;
+          const startsAt = target.withActivity?.startsAt;
+          if (!startsAt) return false;
+
+          return String(startsAt).startsWith('2023-05-07');
+        `,
+      },
+      {
+        provider: 'openai:gpt-5-mini',
+        type: 'llm-rubric',
+        value:
+          'Should extract the LGBTQ support group activity from session_1 diaId D1:3, convert "yesterday" relative to the 2023-05-08 session anchor into 2023-05-07, and include a narrative about feeling supported/accepted.',
+      },
+    ],
+    description: 'LoCoMo conv-26 session_1 resolves relative date',
+    vars: {
+      availableCategories: ['personal'],
+      language: 'English',
+      payloadPath: './promptfoo/evals/activity/locomo/tests/benchmark-locomo-payload-conv-26.json',
+      sessionId: 'session_1',
+      topK: 3,
+      username: 'Caroline',
+    },
+  },
+];
+
+export default testCases;