@lobehub/lobehub 2.0.0-next.360 → 2.0.0-next.361

package/packages/memory-user-memory/promptfoo/evals/activity/locomo/tests/benchmark-locomo-payload-conv-26.json

@@ -0,0 +1,149 @@
+{
+  "force": true,
+  "layers": [],
+  "sampleId": "conv-26",
+  "sessions": [
+    {
+      "sessionId": "session_1",
+      "timestamp": "2023-05-08T13:56:00.000Z",
+      "turns": [
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:1",
+          "role": "user",
+          "speaker": "Caroline",
+          "text": "Hey Mel! Good to see you! How have you been?"
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:2",
+          "role": "assistant",
+          "speaker": "Melanie",
+          "text": "Hey Caroline! Good to see you! I'm swamped with the kids & work. What's up with you? Anything new?"
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:3",
+          "role": "user",
+          "speaker": "Caroline",
+          "text": "I went to a LGBTQ support group yesterday and it was so powerful."
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:4",
+          "role": "assistant",
+          "speaker": "Melanie",
+          "text": "Wow, that's cool, Caroline! What happened that was so awesome? Did you hear any inspiring stories?"
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:5",
+          "imageCaption": "a photo of a dog walking past a wall with a painting of a woman",
+          "imageUrls": [
+            "https://i.redd.it/l7hozpetnhlb1.jpg"
+          ],
+          "role": "user",
+          "speaker": "Caroline",
+          "text": "The transgender stories were so inspiring! I was so happy and thankful for all the support.\n[Image: a photo of a dog walking past a wall with a painting of a woman]"
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:6",
+          "role": "assistant",
+          "speaker": "Melanie",
+          "text": "Wow, love that painting! So cool you found such a helpful group. What's it done for you?"
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:7",
+          "role": "user",
+          "speaker": "Caroline",
+          "text": "The support group has made me feel accepted and given me courage to embrace myself."
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:8",
+          "role": "assistant",
+          "speaker": "Melanie",
+          "text": "That's really cool. You've got guts. What now?"
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:9",
+          "role": "user",
+          "speaker": "Caroline",
+          "text": "Gonna continue my edu and check out career options, which is pretty exciting!"
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:10",
+          "role": "assistant",
+          "speaker": "Melanie",
+          "text": "Wow, Caroline! What kinda jobs are you thinkin' of? Anything that stands out?"
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:11",
+          "role": "user",
+          "speaker": "Caroline",
+          "text": "I'm keen on counseling or working in mental health - I'd love to support those with similar issues."
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:12",
+          "imageCaption": "a photo of a painting of a sunset over a lake",
+          "imageUrls": [
+            "http://candicealexander.com/cdn/shop/products/IMG_7269_a49d5af8-c76c-4ecd-ae20-48c08cb11dec.jpg"
+          ],
+          "role": "assistant",
+          "speaker": "Melanie",
+          "text": "You'd be a great counselor! Your empathy and understanding will really help the people you work with. By the way, take a look at this.\n[Image: a photo of a painting of a sunset over a lake]"
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:13",
+          "role": "user",
+          "speaker": "Caroline",
+          "text": "Thanks, Melanie! That's really sweet. Is this your own painting?"
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:14",
+          "role": "assistant",
+          "speaker": "Melanie",
+          "text": "Yeah, I painted that lake sunrise last year! It's special to me."
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:15",
+          "role": "user",
+          "speaker": "Caroline",
+          "text": "Wow, Melanie! The colors really blend nicely. Painting looks like a great outlet for expressing yourself."
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:16",
+          "role": "assistant",
+          "speaker": "Melanie",
+          "text": "Thanks, Caroline! Painting's a fun way to express my feelings and get creative. It's a great way to relax after a long day."
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:17",
+          "role": "user",
+          "speaker": "Caroline",
+          "text": "Totally agree, Mel. Relaxing and expressing ourselves is key. Well, I'm off to go do some research."
+        },
+        {
+          "createdAt": "2023-05-08T13:56:00.000Z",
+          "diaId": "D1:18",
+          "role": "assistant",
+          "speaker": "Melanie",
+          "text": "Yep, Caroline. Taking care of ourselves is vital. I'm off to go swimming with the kids. Talk to you soon!"
+        }
+      ]
+    }
+  ],
+  "source": "benchmark_locomo",
+  "topicId": "sample_conv-26"
+}

package/packages/memory-user-memory/promptfoo/evals/activity/locomo/tests/cases.ts

@@ -0,0 +1,72 @@
+type PromptfooAssert =
+  | { type: 'javascript'; value: string }
+  | { provider?: string; type: 'llm-rubric'; value: string };
+
+interface PromptfooTestCase {
+  assert: PromptfooAssert[];
+  description?: string;
+  vars: Record<string, unknown>;
+}
+
+const baseSchemaAssert: PromptfooAssert = {
+  type: 'javascript',
+  value: `
+    let parsed;
+    try {
+      parsed = JSON.parse(output);
+    } catch (error) {
+      console.error('Failed to parse JSON output', error);
+      return false;
+    }
+
+    if (!parsed || !Array.isArray(parsed.memories)) return false;
+
+    return parsed.memories.every((memory) => {
+      return memory.memoryType === 'activity' && memory.withActivity?.type;
+    });
+  `,
+};
+
+const testCases: PromptfooTestCase[] = [
+  {
+    assert: [
+      baseSchemaAssert,
+      {
+        type: 'javascript',
+        value: `
+          const data = JSON.parse(output);
+          const target = data.memories?.find((memory) => {
+            const text = [memory.title, memory.summary, memory.withActivity?.narrative]
+              .filter(Boolean)
+              .join(' ')
+              .toLowerCase();
+            return text.includes('support group');
+          });
+
+          if (!target) return false;
+          const startsAt = target.withActivity?.startsAt;
+          if (!startsAt) return false;
+
+          return String(startsAt).startsWith('2023-05-07');
+        `,
+      },
+      {
+        provider: 'openai:gpt-5-mini',
+        type: 'llm-rubric',
+        value:
+          'Should extract the LGBTQ support group activity from session_1 diaId D1:3, convert "yesterday" relative to the 2023-05-08 session anchor into 2023-05-07, and include a narrative about feeling supported/accepted.',
+      },
+    ],
+    description: 'LoCoMo conv-26 session_1 resolves relative date',
+    vars: {
+      availableCategories: ['personal'],
+      language: 'English',
+      payloadPath: './promptfoo/evals/activity/locomo/tests/benchmark-locomo-payload-conv-26.json',
+      sessionId: 'session_1',
+      topK: 3,
+      username: 'Caroline',
+    },
+  },
+];
+
+export default testCases;

package/packages/memory-user-memory/promptfoo/response-formats/activity.json

@@ -0,0 +1,370 @@
+{
+  "json_schema": {
+    "name": "activity_extraction",
+    "schema": {
+      "additionalProperties": false,
+      "properties": {
+        "memories": {
+          "description": "Array of extracted activity memories. Use an empty array when no activity should be captured.",
+          "items": {
+            "additionalProperties": false,
+            "description": "Self-contained activity memory describing what happened, when, where, with whom, and how it felt.",
+            "examples": [
+              {
+                "details": "Talked through renewal scope, confirmed timeline flexibility, and captured follow-ups.",
+                "memoryCategory": "work",
+                "memoryType": "activity",
+                "summary": "Client Q2 renewal meeting with Alice (ACME)",
+                "tags": [
+                  "meeting",
+                  "client",
+                  "renewal"
+                ],
+                "title": "ACME Q2 renewal meeting",
+                "withActivity": {
+                  "type": "meeting",
+                  "associatedLocations": [
+                    {
+                      "address": "123 Main St, New York, NY",
+                      "name": "ACME HQ"
+                    }
+                  ],
+                  "associatedSubjects": [
+                    {
+                      "name": "Alice Smith",
+                      "type": "person"
+                    }
+                  ],
+                  "endsAt": "2024-05-03T15:00:00-04:00",
+                  "feedback": "Positive momentum; Alice felt heard and open to renewal.",
+                  "narrative": "Alice and User reviewed Q2 renewal scope, aligned on reduced deliverables, and agreed to share revised pricing next week.",
+                  "notes": "Agenda: renewal scope, pricing, next steps.",
+                  "startsAt": "2024-05-03T14:00:00-04:00",
+                  "status": "completed",
+                  "timezone": "America/New_York"
+                }
+              },
+              {
+                "details": "Routine check-up; discussed migraines and sleep habits.",
+                "memoryCategory": "health",
+                "memoryType": "activity",
+                "summary": "Doctor appointment with Dr. Kim about migraines",
+                "tags": [
+                  "appointment",
+                  "health"
+                ],
+                "title": "Neurology follow-up",
+                "withActivity": {
+                  "type": "appointment",
+                  "associatedLocations": [
+                    {
+                      "name": "City Neurology Clinic"
+                    }
+                  ],
+                  "associatedSubjects": [
+                    {
+                      "name": "Dr. Kim",
+                      "type": "person"
+                    }
+                  ],
+                  "feedback": "Felt reassured; plan seems manageable.",
+                  "narrative": "User saw Dr. Kim to review migraine frequency; decided to track sleep, hydration, and start a low-dose preventive.",
+                  "notes": "Discussed triggers, hydration, and medication side effects.",
+                  "status": "completed"
+                }
+              }
+            ],
+            "properties": {
+              "details": {
+                "description": "Optional detailed information or longer notes supporting the summary and narrative.",
+                "type": "string"
+              },
+              "memoryCategory": {
+                "description": "Memory category best matching the activity (e.g., work, health, travel, relationships).",
+                "type": "string"
+              },
+              "memoryType": {
+                "const": "activity",
+                "description": "Memory type; always activity.",
+                "type": "string"
+              },
+              "summary": {
+                "description": "Concise overview of this activity.",
+                "type": "string"
+              },
+              "tags": {
+                "description": "Model-generated tags summarizing key facets of the activity.",
+                "items": {
+                  "type": "string"
+                },
+                "type": "array"
+              },
+              "title": {
+                "description": "Brief descriptive title for the activity, e.g., \"Dinner with friends at Marina\".",
+                "type": "string"
+              },
+              "withActivity": {
+                "additionalProperties": false,
+                "description": "Structured activity fields. Temporal and association values are optional—include only when the user mentioned them.",
+                "properties": {
+                  "type": {
+                    "description": "Activity type enum. Choose the closest match; fall back to \"other\" when unclear.",
+                    "enum": [
+                      "appointment",
+                      "call",
+                      "celebration",
+                      "class",
+                      "conference",
+                      "errand",
+                      "event",
+                      "exercise",
+                      "meal",
+                      "meeting",
+                      "other",
+                      "project-session",
+                      "social",
+                      "task",
+                      "trip",
+                      "workshop"
+                    ],
+                    "type": "string"
+                  },
+                  "associatedLocations": {
+                    "description": "Places linked to this activity. Capture any mentioned venue, address, or setting.",
+                    "items": {
+                      "additionalProperties": false,
+                      "properties": {
+                        "address": {
+                          "description": "Free-form address or directions if provided.",
+                          "type": [
+                            "string",
+                            "null"
+                          ]
+                        },
+                        "extra": {
+                          "description": "Optional key-value metadata related to the location.",
+                          "type": [
+                            "string",
+                            "null"
+                          ]
+                        },
+                        "name": {
+                          "description": "Place name or venue label.",
+                          "type": "string"
+                        },
+                        "tags": {
+                          "description": "Place-related tags (e.g., indoor, outdoor, virtual).",
+                          "items": {
+                            "type": "string"
+                          },
+                          "type": [
+                            "array",
+                            "null"
+                          ]
+                        },
+                        "type": {
+                          "description": "Place type or category (office, clinic, restaurant, virtual).",
+                          "type": "string"
+                        }
+                      },
+                      "required": [
+                        "type",
+                        "name",
+                        "address",
+                        "tags",
+                        "extra"
+                      ],
+                      "type": "object"
+                    },
+                    "type": "array"
+                  },
+                  "associatedObjects": {
+                    "description": "Non-living entities or items tied to the activity (e.g., transportation for trips, devices, tools).",
+                    "items": {
+                      "additionalProperties": false,
+                      "properties": {
+                        "extra": {
+                          "description": "Optional key-value metadata related to the object.",
+                          "type": [
+                            "string",
+                            "null"
+                          ]
+                        },
+                        "name": {
+                          "description": "Name or label of the object (e.g., “MacBook”, “flight UA123”).",
+                          "type": "string"
+                        },
+                        "type": {
+                          "description": "Object category (e.g., transportation, device, document).",
+                          "enum": [
+                            "application",
+                            "item",
+                            "knowledge",
+                            "other",
+                            "person",
+                            "place"
+                          ],
+                          "type": "string"
+                        }
+                      },
+                      "required": [
+                        "type",
+                        "name",
+                        "extra"
+                      ],
+                      "type": "object"
+                    },
+                    "type": "array"
+                  },
+                  "associatedSubjects": {
+                    "description": "Living beings involved (people, pets, groups). Use when the subject lacks a known identity ID.",
+                    "items": {
+                      "additionalProperties": false,
+                      "properties": {
+                        "extra": {
+                          "description": "Optional key-value metadata related to the subject.",
+                          "type": [
+                            "string",
+                            "null"
+                          ]
+                        },
+                        "name": {
+                          "description": "Name or short label of the subject.",
+                          "type": "string"
+                        },
+                        "type": {
+                          "description": "Subject category (e.g., person, pet, group).",
+                          "enum": [
+                            "person",
+                            "pet",
+                            "group",
+                            "other"
+                          ],
+                          "type": "string"
+                        }
+                      },
+                      "required": [
+                        "type",
+                        "name",
+                        "extra"
+                      ],
+                      "type": "object"
+                    },
+                    "type": "array"
+                  },
+                  "endsAt": {
+                    "description": "ISO 8601 end time for the activity when specified. Omit if not explicitly provided.",
+                    "format": "date-time",
+                    "type": [
+                      "string",
+                      "null"
+                    ]
+                  },
+                  "feedback": {
+                    "description": "Subjective feelings or evaluation of how the activity went (mood, satisfaction, effort).",
+                    "type": [
+                      "string",
+                      "null"
+                    ]
+                  },
+                  "metadata": {
+                    "additionalProperties": false,
+                    "description": "Additional structured metadata to keep raw hints (JSON object). Use sparingly.",
+                    "type": [
+                      "object",
+                      "null"
+                    ]
+                  },
+                  "narrative": {
+                    "description": "Factual story of what happened (chronology, participants, outcomes). Required for recall.",
+                    "type": "string"
+                  },
+                  "notes": {
+                    "description": "Short annotations such as agenda, preparation, or quick bullets distinct from narrative.",
+                    "type": [
+                      "string",
+                      "null"
+                    ]
+                  },
+                  "startsAt": {
+                    "description": "ISO 8601 start time for the activity when specified. Omit if not explicitly provided.",
+                    "format": "date-time",
+                    "type": [
+                      "string",
+                      "null"
+                    ]
+                  },
+                  "status": {
+                    "description": "Lifecycle status when mentioned. Use planned/completed/cancelled/ongoing/on_hold/pending. Omit if unclear.",
+                    "enum": [
+                      "planned",
+                      "completed",
+                      "cancelled",
+                      "ongoing",
+                      "on_hold",
+                      "pending"
+                    ],
+                    "type": [
+                      "string",
+                      "null"
+                    ]
+                  },
+                  "tags": {
+                    "description": "Optional activity-specific tags or facets.",
+                    "items": {
+                      "type": "string"
+                    },
+                    "type": [
+                      "array",
+                      "null"
+                    ]
+                  },
+                  "timezone": {
+                    "description": "IANA timezone string for the start/end times when provided (e.g., \"America/New_York\").",
+                    "type": [
+                      "string",
+                      "null"
+                    ]
+                  }
+                },
+                "required": [
+                  "type",
+                  "narrative",
+                  "feedback",
+                  "notes",
+                  "associatedLocations",
+                  "associatedSubjects",
+                  "associatedObjects",
+                  "startsAt",
+                  "endsAt",
+                  "status",
+                  "tags",
+                  "timezone",
+                  "metadata"
+                ],
+                "type": "object"
+              }
+            },
+            "required": [
+              "title",
+              "summary",
+              "details",
+              "memoryType",
+              "memoryCategory",
+              "tags",
+              "withActivity"
+            ],
+            "type": "object"
+          },
+          "type": "array"
+        }
+      },
+      "required": [
+        "memories"
+      ],
+      "type": "object"
+    },
+    "strict": true
+  },
+  "type": "json_schema"
+}

package/packages/memory-user-memory/promptfoo/response-formats/experience.json

@@ -62,6 +62,12 @@
                   "type": "string",
                   "description": "Narrative describing key insights or lessons learned"
                 },
+                "knowledgeValueScore": {
+                  "type": "number",
+                  "minimum": 0,
+                  "maximum": 1,
+                  "description": "Numeric score (0-1) describing how reusable and shareable this experience is"
+                },
                 "labels": {
                   "type": "array",
                   "items": {
@@ -73,6 +79,12 @@
                   "type": "string",
                   "description": "Narrative describing potential outcomes or learnings"
                 },
+                "problemSolvingScore": {
+                  "type": "number",
+                  "minimum": 0,
+                  "maximum": 1,
+                  "description": "Numeric score (0-1) describing how effectively the problem was solved"
+                },
                 "reasoning": {
                   "type": "string",
                   "description": "Narrative describing the thought process or motivations"
@@ -95,8 +107,10 @@
               "required": [
                 "action",
                 "keyLearning",
+                "knowledgeValueScore",
                 "labels",
                 "possibleOutcome",
+                "problemSolvingScore",
                 "reasoning",
                 "scoreConfidence",
                 "situation",