@elizaos/ui 2.0.0-alpha.385 → 2.0.0-alpha.392

package/packages/typescript/src/features/advanced-capabilities/experience/evaluators/experienceEvaluator.js

@@ -3,11 +3,8 @@ import { ModelType } from "../../../../types/model.js";
 import { composePrompt } from "../../../../utils.js";
 import { EXTRACT_EXPERIENCES_TEMPLATE } from "../generated/prompts/typescript/prompts.js";
 import { ExperienceType, OutcomeType } from "../types";
+import { detectExperienceDomain, findDuplicateExperienceByLearning, sanitizeExperienceText, } from "../utils/experienceText.js";
 const EXISTING_EXPERIENCE_LIMIT = 5;
-const DUPLICATE_EXPERIENCE_LIMIT = 5;
-const DUPLICATE_JACCARD_THRESHOLD = 0.45;
-const DUPLICATE_CONTAINMENT_THRESHOLD = 0.65;
-const DUPLICATE_SHARED_TERM_THRESHOLD = 4;
 const PASSIVE_ACTIONS = new Set([
     "REPLY",
     "NONE",
@@ -15,25 +12,6 @@ const PASSIVE_ACTIONS = new Set([
     "IGNORE",
     "WAIT",
 ]);
-const STOP_WORDS = new Set([
-    "about",
-    "after",
-    "again",
-    "before",
-    "being",
-    "from",
-    "into",
-    "that",
-    "their",
-    "them",
-    "then",
-    "there",
-    "these",
-    "this",
-    "when",
-    "with",
-    "without",
-]);
 export const experienceEvaluator = {
     name: "EXPERIENCE_EVALUATOR",
     similes: ["experience recorder", "learning evaluator", "self-reflection"],
@@ -128,6 +106,7 @@ export const experienceEvaluator = {
             .map((m) => m.content.text)
             .filter(Boolean)
             .join(" ");
+        const provenance = buildExperienceProvenance(message, recentMessages);
         const existingExperiences = await experienceService.findSimilarExperiences(conversationContext, EXISTING_EXPERIENCE_LIMIT);
         const extractionPrompt = composePrompt({
             state: {
@@ -162,7 +141,7 @@ export const experienceEvaluator = {
                 continue;
             }
             const sanitizedLearning = sanitizeContext(rawLearning);
-            const duplicate = await findDuplicateExperience(experienceService, sanitizedLearning);
+            const duplicate = await findDuplicateExperienceByLearning(experienceService, sanitizedLearning);
             if (duplicate) {
                 skippedDuplicateCount++;
                 logger.debug(`[experienceEvaluator] Skipping duplicate experience: "${sanitizedLearning.substring(0, 80)}..."`);
@@ -172,6 +151,9 @@ export const experienceEvaluator = {
             const experienceType = experienceTypeMap[normalizedType] ?? ExperienceType.LEARNING;
             const experienceTag = experienceType;
             const sanitizedContext = sanitizeContext(exp.context || "Conversation analysis");
+            const sanitizedReason = typeof exp.reasoning === "string" && exp.reasoning.trim().length > 0
+                ? sanitizeContext(exp.reasoning)
+                : undefined;
             await experienceService.recordExperience({
                 type: experienceType,
                 outcome: experienceType === ExperienceType.CORRECTION
@@ -181,10 +163,17 @@ export const experienceEvaluator = {
                 action: "pattern_recognition",
                 result: sanitizedLearning,
                 learning: sanitizedLearning,
-                domain: detectDomain(sanitizedLearning),
+                domain: detectExperienceDomain(sanitizedLearning),
                 tags: ["extracted", "novel", experienceTag],
                 confidence: Math.min(exp.confidence, 0.9), // Cap confidence
                 importance: 0.8, // High importance for extracted experiences
+                sourceMessageIds: provenance.sourceMessageIds,
+                sourceRoomId: provenance.sourceRoomId,
+                sourceTriggerMessageId: provenance.sourceTriggerMessageId,
+                sourceTrajectoryId: provenance.sourceTrajectoryId,
+                sourceTrajectoryStepId: provenance.sourceTrajectoryStepId,
+                extractionMethod: "experience_evaluator",
+                extractionReason: sanitizedReason,
             });
             recordedCount++;
             logger.info(`[experienceEvaluator] Recorded novel experience: ${sanitizedLearning.substring(0, 100)}...`);
@@ -248,55 +237,12 @@ function formatExistingExperiences(experiences) {
     }
     return experiences
         .map((experience, index) => {
-        const learning = sanitizeContext(experience.learning);
-        const context = sanitizeContext(experience.context);
+        const learning = sanitizeExperienceText(experience.learning);
+        const context = sanitizeExperienceText(experience.context);
         return `${index + 1}. (${experience.type}/${experience.domain}, confidence ${experience.confidence.toFixed(2)}) When ${context}, learned: ${learning}`;
     })
         .join("\n");
 }
-async function findDuplicateExperience(experienceService, learning) {
-    const similar = await experienceService.findSimilarExperiences(learning, DUPLICATE_EXPERIENCE_LIMIT);
-    return (similar.find((experience) => isDuplicateLearning(learning, experience.learning)) ?? null);
-}
-function isDuplicateLearning(a, b) {
-    const normalizedA = normalizeTextForDuplicateComparison(a);
-    const normalizedB = normalizeTextForDuplicateComparison(b);
-    if (!normalizedA || !normalizedB) {
-        return false;
-    }
-    if (normalizedA === normalizedB) {
-        return true;
-    }
-    if (Math.min(normalizedA.length, normalizedB.length) >= 24 &&
-        (normalizedA.includes(normalizedB) || normalizedB.includes(normalizedA))) {
-        return true;
-    }
-    const aTokens = tokenizeForDuplicateComparison(normalizedA);
-    const bTokens = tokenizeForDuplicateComparison(normalizedB);
-    if (aTokens.size < 4 || bTokens.size < 4) {
-        return false;
-    }
-    const overlap = [...aTokens].filter((token) => bTokens.has(token)).length;
-    const union = new Set([...aTokens, ...bTokens]).size;
-    const jaccard = union > 0 ? overlap / union : 0;
-    const containment = overlap / Math.min(aTokens.size, bTokens.size);
-    return (jaccard >= DUPLICATE_JACCARD_THRESHOLD ||
-        containment >= DUPLICATE_CONTAINMENT_THRESHOLD ||
-        (overlap >= DUPLICATE_SHARED_TERM_THRESHOLD && containment >= 0.4));
-}
-function normalizeTextForDuplicateComparison(text) {
-    return text
-        .toLowerCase()
-        .replace(/[^a-z0-9]+/g, " ")
-        .replace(/\s+/g, " ")
-        .trim();
-}
-function tokenizeForDuplicateComparison(text) {
-    return new Set(text
-        .split(" ")
-        .map((token) => token.trim())
-        .filter((token) => token.length > 3 && !STOP_WORDS.has(token)));
-}
 function parseExtractedExperiences(response) {
     const jsonMatch = response.match(/\[[\s\S]*\]/);
     if (!jsonMatch)
@@ -322,60 +268,24 @@ function getNumberSetting(runtime, key, fallback) {
     return fallback;
 }
 function sanitizeContext(text) {
-    if (!text)
-        return "Unknown context";
-    // Remove user-specific details while preserving technical context
-    return text
-        .replace(/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g, "[EMAIL]") // emails
-        .replace(/\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/g, "[IP]") // IP addresses
-        .replace(/\/Users\/[^/\s]+/g, "/Users/[USER]") // user directories
-        .replace(/\/home\/[^/\s]+/g, "/home/[USER]") // home directories
-        .replace(/\b(?:sk|pk|rk|gsk|ghp|gho|ghu|ghs|github_pat|xox[baprs])-?[A-Za-z0-9_-]{12,}\b/gi, "[TOKEN]") // common API keys/tokens
-        .replace(/\b[A-Za-z0-9_-]{32,}\b/g, "[TOKEN]") // long opaque tokens
-        .replace(/\b(user|person|someone|they)\s+(said|asked|told|mentioned)/gi, "when asked") // personal references
-        .substring(0, 200); // limit length
+    return sanitizeExperienceText(text);
 }
-function detectDomain(text) {
-    const domains = {
-        shell: ["command", "terminal", "bash", "shell", "execute", "script", "cli"],
-        coding: [
-            "code",
-            "function",
-            "variable",
-            "syntax",
-            "programming",
-            "debug",
-            "typescript",
-            "javascript",
-        ],
-        system: [
-            "file",
-            "directory",
-            "process",
-            "memory",
-            "cpu",
-            "system",
-            "install",
-            "package",
-        ],
-        network: [
-            "http",
-            "api",
-            "request",
-            "response",
-            "url",
-            "network",
-            "fetch",
-            "curl",
-        ],
-        data: ["json", "csv", "database", "query", "data", "sql", "table"],
-        ai: ["model", "llm", "embedding", "prompt", "token", "inference"],
+function buildExperienceProvenance(triggerMessage, recentMessages) {
+    const sourceMessageIds = recentMessages
+        .map((recentMessage) => recentMessage.id)
+        .filter((id) => typeof id === "string");
+    return {
+        sourceMessageIds: sourceMessageIds.length > 0 ? sourceMessageIds : undefined,
+        sourceRoomId: triggerMessage.roomId,
+        sourceTriggerMessageId: triggerMessage.id,
+        sourceTrajectoryId: readMetadataString(triggerMessage, "trajectoryId"),
+        sourceTrajectoryStepId: readMetadataString(triggerMessage, "trajectoryStepId"),
     };
-    const lowerText = text.toLowerCase();
-    for (const [domain, keywords] of Object.entries(domains)) {
-        if (keywords.some((keyword) => lowerText.includes(keyword))) {
-            return domain;
-        }
-    }
-    return "general";
+}
+function readMetadataString(message, key) {
+    const metadata = asRecord(message.metadata);
+    const value = metadata?.[key];
+    return typeof value === "string" && value.trim().length > 0
+        ? value.trim()
+        : undefined;
 }

package/packages/typescript/src/features/advanced-capabilities/experience/service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"service.d.ts","sourceRoot":"","sources":["../../../../../../../../typescript/src/features/advanced-capabilities/experience/service.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAa,IAAI,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,EAAE,OAAO,EAAE,KAAK,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC1E,OAAO,EACN,KAAK,UAAU,EACf,KAAK,kBAAkB,EACvB,KAAK,eAAe,EAEpB,cAAc,EAEd,MAAM,YAAY,CAAC;AAIpB,qBAAa,iBAAkB,SAAQ,OAAO;IAC7C,OAAgB,WAAW,EAAE,eAAe,CACS;IAC5C,qBAAqB,SACiE;IAE/F,OAAO,CAAC,WAAW,CAAoC;IACvD,OAAO,CAAC,mBAAmB,CAAqC;IAChE,OAAO,CAAC,iBAAiB,CAA6C;IACtE,OAAO,CAAC,gBAAgB,CAAwB;IAChD,OAAO,CAAC,YAAY,CAA+C;IACnE,OAAO,CAAC,YAAY,CAAyB;IAC7C,OAAO,CAAC,mBAAmB,CAAgC;gBAE/C,OAAO,EAAE,aAAa;WAarB,KAAK,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAMtE,OAAO,CAAC,WAAW;IAUnB,OAAO,CAAC,mBAAmB;IAM3B,OAAO,CAAC,aAAa;IAQrB,OAAO,CAAC,mBAAmB;IAK3B,OAAO,CAAC,mBAAmB;IAY3B,OAAO,CAAC,UAAU;IAQlB,OAAO,CAAC,gBAAgB;IAIxB,OAAO,CAAC,aAAa;IAIrB,OAAO,CAAC,eAAe;IAWvB,OAAO,CAAC,eAAe;IAYvB,OAAO,CAAC,iBAAiB;IAczB,OAAO,CAAC,aAAa;IAUrB,OAAO,CAAC,qBAAqB;YA4Ff,iBAAiB;IAiC/B,OAAO,CAAC,qBAAqB;IAkD7B,OAAO,CAAC,gBAAgB;YASV,eAAe;IAmBvB,gBAAgB,CACrB,cAAc,EAAE,OAAO,CAAC,UAAU,CAAC,GACjC,OAAO,CAAC,UAAU,CAAC;YAmER,sBAAsB;YAOtB,uBAAuB;IAyB/B,aAAa,CAAC,EAAE,EAAE,IAAI,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAKnD,eAAe,CAAC,KAAK,GAAE,eAAoB,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;IAInE,gBAAgB,CACrB,EAAE,EAAE,IAAI,EACR,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,GAC1B,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IA8DvB,gBAAgB,CAAC,EAAE,EAAE,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC;IAc5C,gBAAgB,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;YAIvD,kBAAkB;IAgEhC,4FAA4F;IAC5F,OAAO,CAAC,YAAY;IAiDpB;;;;;;;;;;;OAWG;IACG,sBAAsB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,SAAI,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;IAoF5E,yFAAyF;IACzF,OAAO,CAAC,YAAY;IAUd,kBAAkB,CACvB,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,cAAc,GACnB,OAAO,CAAC,kBAAkB,CAAC;IA4C9B,OAAO,CAAC,gBAAgB;IAmBxB,OAAO,CAAC,kBAAkB;IAoB1B,OAAO,CAAC,2BAA2B;IAYnC,OAAO,CAAC,mBAAmB;IAsB3B,OAAO,CAAC,uBAAuB;IAyDzB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CA2B3B"}
+{"version":3,"file":"service.d.ts","sourceRoot":"","sources":["../../../../../../../../typescript/src/features/advanced-capabilities/experience/service.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAa,IAAI,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,EAAE,OAAO,EAAE,KAAK,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC1E,OAAO,EACN,KAAK,UAAU,EACf,KAAK,kBAAkB,EACvB,KAAK,eAAe,EAEpB,cAAc,EAEd,MAAM,YAAY,CAAC;AAIpB,qBAAa,iBAAkB,SAAQ,OAAO;IAC7C,OAAgB,WAAW,EAAE,eAAe,CACS;IAC5C,qBAAqB,SACiE;IAE/F,OAAO,CAAC,WAAW,CAAoC;IACvD,OAAO,CAAC,mBAAmB,CAAqC;IAChE,OAAO,CAAC,iBAAiB,CAA6C;IACtE,OAAO,CAAC,gBAAgB,CAAwB;IAChD,OAAO,CAAC,YAAY,CAA+C;IACnE,OAAO,CAAC,YAAY,CAAyB;IAC7C,OAAO,CAAC,mBAAmB,CAAgC;gBAE/C,OAAO,EAAE,aAAa;WAarB,KAAK,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAMtE,OAAO,CAAC,WAAW;IAUnB,OAAO,CAAC,mBAAmB;IAM3B,OAAO,CAAC,aAAa;IAQrB,OAAO,CAAC,mBAAmB;IAK3B,OAAO,CAAC,mBAAmB;IAY3B,OAAO,CAAC,UAAU;IAQlB,OAAO,CAAC,gBAAgB;IAIxB,OAAO,CAAC,aAAa;IAIrB,OAAO,CAAC,eAAe;IAWvB,OAAO,CAAC,eAAe;IAYvB,OAAO,CAAC,iBAAiB;IAczB,OAAO,CAAC,aAAa;IAUrB,OAAO,CAAC,qBAAqB;YAqHf,iBAAiB;IAiC/B,OAAO,CAAC,qBAAqB;IAuE7B,OAAO,CAAC,gBAAgB;YASV,eAAe;IAmBvB,gBAAgB,CACrB,cAAc,EAAE,OAAO,CAAC,UAAU,CAAC,GACjC,OAAO,CAAC,UAAU,CAAC;YA4ER,sBAAsB;YAOtB,uBAAuB;IAyB/B,aAAa,CAAC,EAAE,EAAE,IAAI,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAKnD,eAAe,CAAC,KAAK,GAAE,eAAoB,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;IAInE,gBAAgB,CACrB,EAAE,EAAE,IAAI,EACR,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,GAC1B,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAsEvB,gBAAgB,CAAC,EAAE,EAAE,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC;IAc5C,gBAAgB,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;YAIvD,kBAAkB;IAgEhC,4FAA4F;IAC5F,OAAO,CAAC,YAAY;IAiDpB;;;;;;;;;;;OAWG;IACG,sBAAsB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,SAAI,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;IAoF5E,yFAAyF;IACzF,OAAO,CAAC,YAAY;IAUd,kBAAkB,CACvB,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,cAAc,GACnB,OAAO,CAAC,kBAAkB,CAAC;IA4C9B,OAAO,CAAC,gBAAgB;IAmBxB,OAAO,CAAC,kBAAkB;IAoB1B,OAAO,CAAC,2BAA2B;IAYnC,OAAO,CAAC,mBAAmB;IAsB3B,OAAO,CAAC,uBAAuB;IAyDzB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CA2B3B"}

package/packages/typescript/src/features/advanced-capabilities/experience/service.js

@@ -180,6 +180,25 @@ export class ExperienceService extends Service {
             correctedBelief: typeof rawData?.correctedBelief === "string"
                 ? rawData.correctedBelief
                 : undefined,
+            sourceMessageIds: this.asOptionalUuidArray(rawData?.sourceMessageIds),
+            sourceRoomId: typeof rawData?.sourceRoomId === "string"
+                ? rawData.sourceRoomId
+                : undefined,
+            sourceTriggerMessageId: typeof rawData?.sourceTriggerMessageId === "string"
+                ? rawData.sourceTriggerMessageId
+                : undefined,
+            sourceTrajectoryId: typeof rawData?.sourceTrajectoryId === "string"
+                ? rawData.sourceTrajectoryId
+                : undefined,
+            sourceTrajectoryStepId: typeof rawData?.sourceTrajectoryStepId === "string"
+                ? rawData.sourceTrajectoryStepId
+                : undefined,
+            extractionMethod: typeof rawData?.extractionMethod === "string"
+                ? rawData.extractionMethod
+                : undefined,
+            extractionReason: typeof rawData?.extractionReason === "string"
+                ? rawData.extractionReason
+                : undefined,
         };
     }
     async generateEmbedding(experienceData) {
@@ -234,6 +253,27 @@ export class ExperienceService extends Service {
         if (experience.correctedBelief !== undefined) {
             data.correctedBelief = experience.correctedBelief;
         }
+        if (experience.sourceMessageIds !== undefined) {
+            data.sourceMessageIds = experience.sourceMessageIds;
+        }
+        if (experience.sourceRoomId !== undefined) {
+            data.sourceRoomId = experience.sourceRoomId;
+        }
+        if (experience.sourceTriggerMessageId !== undefined) {
+            data.sourceTriggerMessageId = experience.sourceTriggerMessageId;
+        }
+        if (experience.sourceTrajectoryId !== undefined) {
+            data.sourceTrajectoryId = experience.sourceTrajectoryId;
+        }
+        if (experience.sourceTrajectoryStepId !== undefined) {
+            data.sourceTrajectoryStepId = experience.sourceTrajectoryStepId;
+        }
+        if (experience.extractionMethod !== undefined) {
+            data.extractionMethod = experience.extractionMethod;
+        }
+        if (experience.extractionReason !== undefined) {
+            data.extractionReason = experience.extractionReason;
+        }
         return {
             id: experience.id,
             unique: true,
@@ -307,6 +347,15 @@ export class ExperienceService extends Service {
             supersedes: experienceData.supersedes,
             previousBelief: experienceData.previousBelief,
             correctedBelief: experienceData.correctedBelief,
+            sourceMessageIds: experienceData.sourceMessageIds
+                ? [...experienceData.sourceMessageIds]
+                : undefined,
+            sourceRoomId: experienceData.sourceRoomId,
+            sourceTriggerMessageId: experienceData.sourceTriggerMessageId,
+            sourceTrajectoryId: experienceData.sourceTrajectoryId,
+            sourceTrajectoryStepId: experienceData.sourceTrajectoryStepId,
+            extractionMethod: experienceData.extractionMethod,
+            extractionReason: experienceData.extractionReason,
         };
         this.setExperience(experience);
         // Save to memory service
@@ -401,6 +450,13 @@ export class ExperienceService extends Service {
                 : existing.relatedExperiences
                     ? [...existing.relatedExperiences]
                     : undefined,
+            sourceMessageIds: "sourceMessageIds" in updates
+                ? updates.sourceMessageIds
+                    ? [...updates.sourceMessageIds]
+                    : undefined
+                : existing.sourceMessageIds
+                    ? [...existing.sourceMessageIds]
+                    : undefined,
             embedding,
             updatedAt: Date.now(),
         };

package/packages/typescript/src/features/advanced-capabilities/experience/types.d.ts

@@ -54,6 +54,13 @@ export interface Experience {
     correctedBelief?: string;
     embedding?: number[];
     memoryIds?: UUID[];
+    sourceMessageIds?: UUID[];
+    sourceRoomId?: UUID;
+    sourceTriggerMessageId?: UUID;
+    sourceTrajectoryId?: string;
+    sourceTrajectoryStepId?: string;
+    extractionMethod?: string;
+    extractionReason?: string;
 }
 export interface ExperienceQuery {
     query?: string;

package/packages/typescript/src/features/advanced-capabilities/experience/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../../../../../typescript/src/features/advanced-capabilities/experience/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,8BAA8B,CAAC;AAGzD,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,CAAC;AAC7D,MAAM,MAAM,SAAS,GAClB,aAAa,GACb,SAAS,EAAE,GACX;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,CAAC;AAChC,MAAM,MAAM,UAAU,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,CAAC;AAEtD,OAAO,QAAQ,2BAA2B,CAAC;IAC1C,UAAU,mBAAmB;QAC5B,UAAU,EAAE,YAAY,CAAC;KACzB;CACD;AAGD,eAAO,MAAM,qBAAqB;;CAEM,CAAC;AAEzC,oBAAY,cAAc;IACzB,OAAO,YAAY,CAAE,+BAA+B;IACpD,OAAO,YAAY,CAAE,4BAA4B;IACjD,SAAS,cAAc,CAAE,mCAAmC;IAC5D,UAAU,eAAe,CAAE,4BAA4B;IACvD,QAAQ,aAAa,CAAE,8BAA8B;IACrD,UAAU,eAAe,CAAE,4BAA4B;IACvD,UAAU,eAAe,CAAE,+BAA+B;IAC1D,OAAO,YAAY;CACnB;AAED,oBAAY,WAAW;IACtB,QAAQ,aAAa;IACrB,QAAQ,aAAa;IACrB,OAAO,YAAY;IACnB,KAAK,UAAU;CACf;AAED,MAAM,WAAW,UAAU;IAC1B,EAAE,EAAE,IAAI,CAAC;IACT,OAAO,EAAE,IAAI,CAAC;IACd,IAAI,EAAE,cAAc,CAAC;IACrB,OAAO,EAAE,WAAW,CAAC;IAGrB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IAGjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IAGf,kBAAkB,CAAC,EAAE,IAAI,EAAE,CAAC;IAC5B,UAAU,CAAC,EAAE,IAAI,CAAC;IAGlB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IAGnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IAGpB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IAGzB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,SAAS,CAAC,EAAE,IAAI,EAAE,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,cAAc,GAAG,cAAc,EAAE,CAAC;IACzC,OAAO,CAAC,EAAE,WAAW,GAAG,WAAW,EAAE,CAAC;IACtC,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE;QACX,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,GAAG,CAAC,EAAE,MAAM,CAAC;KACb,CAAC;IACF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,WAAW,kBAAkB;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,eAAe;IAC/B,YAAY,EAAE,IAAI,CAAC;IACnB,SAAS,EAAE,SAAS,GAAG,UAAU,GAAG,SAAS,GAAG,YAAY,CAAC;IAC7D,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,UAAU,CAAC;CACtB;AAED,MAAM,WAAW,gBAAiB,SAAQ,MAAM;IAC/C,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,cAAc,CAAC;CAC/B"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../../../../../typescript/src/features/advanced-capabilities/experience/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,8BAA8B,CAAC;AAGzD,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,CAAC;AAC7D,MAAM,MAAM,SAAS,GAClB,aAAa,GACb,SAAS,EAAE,GACX;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,CAAC;AAChC,MAAM,MAAM,UAAU,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,CAAC;AAEtD,OAAO,QAAQ,2BAA2B,CAAC;IAC1C,UAAU,mBAAmB;QAC5B,UAAU,EAAE,YAAY,CAAC;KACzB;CACD;AAGD,eAAO,MAAM,qBAAqB;;CAEM,CAAC;AAEzC,oBAAY,cAAc;IACzB,OAAO,YAAY,CAAE,+BAA+B;IACpD,OAAO,YAAY,CAAE,4BAA4B;IACjD,SAAS,cAAc,CAAE,mCAAmC;IAC5D,UAAU,eAAe,CAAE,4BAA4B;IACvD,QAAQ,aAAa,CAAE,8BAA8B;IACrD,UAAU,eAAe,CAAE,4BAA4B;IACvD,UAAU,eAAe,CAAE,+BAA+B;IAC1D,OAAO,YAAY;CACnB;AAED,oBAAY,WAAW;IACtB,QAAQ,aAAa;IACrB,QAAQ,aAAa;IACrB,OAAO,YAAY;IACnB,KAAK,UAAU;CACf;AAED,MAAM,WAAW,UAAU;IAC1B,EAAE,EAAE,IAAI,CAAC;IACT,OAAO,EAAE,IAAI,CAAC;IACd,IAAI,EAAE,cAAc,CAAC;IACrB,OAAO,EAAE,WAAW,CAAC;IAGrB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IAGjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IAGf,kBAAkB,CAAC,EAAE,IAAI,EAAE,CAAC;IAC5B,UAAU,CAAC,EAAE,IAAI,CAAC;IAGlB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IAGnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IAGpB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IAGzB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,SAAS,CAAC,EAAE,IAAI,EAAE,CAAC;IAGnB,gBAAgB,CAAC,EAAE,IAAI,EAAE,CAAC;IAC1B,YAAY,CAAC,EAAE,IAAI,CAAC;IACpB,sBAAsB,CAAC,EAAE,IAAI,CAAC;IAC9B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,eAAe;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,cAAc,GAAG,cAAc,EAAE,CAAC;IACzC,OAAO,CAAC,EAAE,WAAW,GAAG,WAAW,EAAE,CAAC;IACtC,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE;QACX,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,GAAG,CAAC,EAAE,MAAM,CAAC;KACb,CAAC;IACF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,WAAW,kBAAkB;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,eAAe;IAC/B,YAAY,EAAE,IAAI,CAAC;IACnB,SAAS,EAAE,SAAS,GAAG,UAAU,GAAG,SAAS,GAAG,YAAY,CAAC;IAC7D,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,UAAU,CAAC;CACtB;AAED,MAAM,WAAW,gBAAiB,SAAQ,MAAM;IAC/C,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,cAAc,CAAC;CAC/B"}

package/packages/typescript/src/features/advanced-capabilities/experience/utils/experienceText.d.ts

@@ -0,0 +1,7 @@
+import type { ExperienceService } from "../service.ts";
+import type { Experience } from "../types.ts";
+export declare function sanitizeExperienceText(text: string): string;
+export declare function detectExperienceDomain(text: string): string;
+export declare function findDuplicateExperienceByLearning(experienceService: ExperienceService, learning: string): Promise<Experience | null>;
+export declare function isDuplicateLearning(a: string, b: string): boolean;
+//# sourceMappingURL=experienceText.d.ts.map

package/packages/typescript/src/features/advanced-capabilities/experience/utils/experienceText.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"experienceText.d.ts","sourceRoot":"","sources":["../../../../../../../../../typescript/src/features/advanced-capabilities/experience/utils/experienceText.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACvD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AA0B9C,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAkB3D;AAED,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CA8C3D;AAED,wBAAsB,iCAAiC,CACtD,iBAAiB,EAAE,iBAAiB,EACpC,QAAQ,EAAE,MAAM,GACd,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAW5B;AAED,wBAAgB,mBAAmB,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,OAAO,CAgCjE"}

package/packages/typescript/src/features/advanced-capabilities/experience/utils/experienceText.js

@@ -0,0 +1,123 @@
+const DUPLICATE_EXPERIENCE_LIMIT = 5;
+const DUPLICATE_JACCARD_THRESHOLD = 0.45;
+const DUPLICATE_CONTAINMENT_THRESHOLD = 0.65;
+const DUPLICATE_SHARED_TERM_THRESHOLD = 4;
+const STOP_WORDS = new Set([
+    "about",
+    "after",
+    "again",
+    "before",
+    "being",
+    "from",
+    "into",
+    "that",
+    "their",
+    "them",
+    "then",
+    "there",
+    "these",
+    "this",
+    "when",
+    "with",
+    "without",
+]);
+export function sanitizeExperienceText(text) {
+    if (!text)
+        return "Unknown context";
+    return text
+        .replace(/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g, "[EMAIL]")
+        .replace(/\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/g, "[IP]")
+        .replace(/\/Users\/[^/\s]+/g, "/Users/[USER]")
+        .replace(/\/home\/[^/\s]+/g, "/home/[USER]")
+        .replace(/\b(?:sk|pk|rk|gsk|ghp|gho|ghu|ghs|github_pat|xox[baprs])-?[A-Za-z0-9_-]{12,}\b/gi, "[TOKEN]")
+        .replace(/\b[A-Za-z0-9_-]{32,}\b/g, "[TOKEN]")
+        .replace(/\b(user|person|someone|they)\s+(said|asked|told|mentioned)/gi, "when asked")
+        .substring(0, 200);
+}
+export function detectExperienceDomain(text) {
+    const domains = {
+        shell: ["command", "terminal", "bash", "shell", "execute", "script", "cli"],
+        coding: [
+            "code",
+            "function",
+            "variable",
+            "syntax",
+            "programming",
+            "debug",
+            "typescript",
+            "javascript",
+        ],
+        system: [
+            "file",
+            "directory",
+            "process",
+            "memory",
+            "cpu",
+            "system",
+            "install",
+            "package",
+        ],
+        network: [
+            "http",
+            "api",
+            "request",
+            "response",
+            "url",
+            "network",
+            "fetch",
+            "curl",
+        ],
+        data: ["json", "csv", "database", "query", "data", "sql", "table"],
+        ai: ["model", "llm", "embedding", "prompt", "token", "inference"],
+    };
+    const lowerText = text.toLowerCase();
+    for (const [domain, keywords] of Object.entries(domains)) {
+        if (keywords.some((keyword) => lowerText.includes(keyword))) {
+            return domain;
+        }
+    }
+    return "general";
+}
+export async function findDuplicateExperienceByLearning(experienceService, learning) {
+    const similar = await experienceService.findSimilarExperiences(learning, DUPLICATE_EXPERIENCE_LIMIT);
+    return (similar.find((experience) => isDuplicateLearning(learning, experience.learning)) ?? null);
+}
+export function isDuplicateLearning(a, b) {
+    const normalizedA = normalizeTextForDuplicateComparison(a);
+    const normalizedB = normalizeTextForDuplicateComparison(b);
+    if (!normalizedA || !normalizedB) {
+        return false;
+    }
+    if (normalizedA === normalizedB) {
+        return true;
+    }
+    if (Math.min(normalizedA.length, normalizedB.length) >= 24 &&
+        (normalizedA.includes(normalizedB) || normalizedB.includes(normalizedA))) {
+        return true;
+    }
+    const aTokens = tokenizeForDuplicateComparison(normalizedA);
+    const bTokens = tokenizeForDuplicateComparison(normalizedB);
+    if (aTokens.size < 4 || bTokens.size < 4) {
+        return false;
+    }
+    const overlap = [...aTokens].filter((token) => bTokens.has(token)).length;
+    const union = new Set([...aTokens, ...bTokens]).size;
+    const jaccard = union > 0 ? overlap / union : 0;
+    const containment = overlap / Math.min(aTokens.size, bTokens.size);
+    return (jaccard >= DUPLICATE_JACCARD_THRESHOLD ||
+        containment >= DUPLICATE_CONTAINMENT_THRESHOLD ||
+        (overlap >= DUPLICATE_SHARED_TERM_THRESHOLD && containment >= 0.4));
+}
+function normalizeTextForDuplicateComparison(text) {
+    return text
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, " ")
+        .replace(/\s+/g, " ")
+        .trim();
+}
+function tokenizeForDuplicateComparison(text) {
+    return new Set(text
+        .split(" ")
+        .map((token) => token.trim())
+        .filter((token) => token.length > 3 && !STOP_WORDS.has(token)));
+}

package/apps/app-training/src/ui/FineTuningView.d.ts

@@ -1,5 +0,0 @@
-import { type ReactNode } from "react";
-export declare function FineTuningView({ contentHeader, }?: {
-    contentHeader?: ReactNode;
-}): import("react/jsx-runtime").JSX.Element;
-//# sourceMappingURL=FineTuningView.d.ts.map

package/apps/app-training/src/ui/FineTuningView.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"FineTuningView.d.ts","sourceRoot":"","sources":["../../../../../../../apps/app-training/src/ui/FineTuningView.tsx"],"names":[],"mappings":"AAiBA,OAAO,EACL,KAAK,SAAS,EAKf,MAAM,OAAO,CAAC;AAoBf,wBAAgB,cAAc,CAAC,EAC7B,aAAa,GACd,GAAE;IACD,aAAa,CAAC,EAAE,SAAS,CAAC;CACtB,2CAwoBL"}