@memtensor/memos-local-openclaw-plugin 0.1.4 → 0.1.5

package/src/skill/evaluator.ts

@@ -0,0 +1,220 @@
+import type { Chunk, Task, Skill, PluginContext, SummarizerConfig } from "../types";
+import { DEFAULTS } from "../types";
+
+export interface CreateEvalResult {
+  shouldGenerate: boolean;
+  reason: string;
+  suggestedName: string;
+  suggestedTags: string[];
+  confidence: number;
+}
+
+export interface UpgradeEvalResult {
+  shouldUpgrade: boolean;
+  upgradeType: "refine" | "extend" | "fix";
+  dimensions: string[];
+  reason: string;
+  mergeStrategy: string;
+  confidence: number;
+}
+
+const CREATE_EVAL_PROMPT = `You are an experience evaluation expert. Based on the completed task record below, decide whether this task contains reusable experience worth distilling into a "skill".
+
+A skill is a reusable guide that helps an AI agent handle similar tasks better in the future.
+
+Worth distilling (any ONE qualifies):
+- Contains concrete steps, commands, code, or configuration
+- Solves a recurring problem with a specific approach/workflow
+- Went through trial-and-error (wrong approach then corrected)
+- Involves non-obvious usage of specific tools, APIs, or frameworks
+- Contains debugging/troubleshooting with diagnostic reasoning
+- Demonstrates a multi-step workflow using external tools (browser, search, file system, etc.)
+- Reveals user preferences or style requirements that should be remembered
+- Shows how to combine multiple tools/services to accomplish a goal
+- Contains a process that required specific parameter tuning or configuration
+
+NOT worth distilling:
+- Pure factual Q&A with no process ("what is TCP", "what's the capital of France")
+- Single-turn simple answers with no workflow
+- Conversation too fragmented or incoherent to extract a clear process
+
+Task title: {TITLE}
+Task summary:
+{SUMMARY}
+
+Reply in JSON only, no extra text:
+{
+  "shouldGenerate": boolean,
+  "reason": "brief explanation",
+  "suggestedName": "kebab-case-name",
+  "suggestedTags": ["tag1", "tag2"],
+  "confidence": 0.0-1.0
+}`;
+
+const UPGRADE_EVAL_PROMPT = `You are a skill upgrade evaluation expert.
+
+Existing skill (v{VERSION}):
+Name: {SKILL_NAME}
+Content:
+{SKILL_CONTENT}
+
+Newly completed task:
+Title: {TITLE}
+Summary:
+{SUMMARY}
+
+Does the new task bring substantive improvements to the existing skill?
+
+Worth upgrading (any one qualifies):
+1. Faster — shorter path discovered
+2. More elegant — cleaner, follows best practices better
+3. More convenient — fewer dependencies or complexity
+4. Fewer tokens — less exploration/trial-and-error needed
+5. More accurate — corrects wrong parameters/steps in old skill
+6. More robust — adds edge cases, error handling
+7. New scenario — covers a variant the old skill didn't
+8. Fixes outdated info — old skill has stale information
+
+NOT worth upgrading:
+- New task is identical to existing skill
+- New task's approach is worse than existing skill
+- Differences are trivial
+
+Reply in JSON only, no extra text:
+{
+  "shouldUpgrade": boolean,
+  "upgradeType": "refine" | "extend" | "fix",
+  "dimensions": ["faster", "more_elegant", "more_convenient", "fewer_tokens", "more_accurate", "more_robust", "new_scenario", "fix_outdated"],
+  "reason": "what new value the task brings",
+  "mergeStrategy": "which specific parts need updating",
+  "confidence": 0.0-1.0
+}`;
+
+export class SkillEvaluator {
+  constructor(private ctx: PluginContext) {}
+
+  passesRuleFilter(chunks: Chunk[], task: Task): { pass: boolean; skipReason: string } {
+    const minChunks = this.ctx.config.skillEvolution?.minChunksForEval ?? DEFAULTS.skillMinChunksForEval;
+    if (chunks.length < minChunks) {
+      return { pass: false, skipReason: `chunks不足 (${chunks.length} < ${minChunks})` };
+    }
+
+    if (task.status === "skipped") {
+      return { pass: false, skipReason: "task状态为skipped" };
+    }
+
+    if (task.summary.length < 100) {
+      return { pass: false, skipReason: `summary过短 (${task.summary.length} < 100)` };
+    }
+
+    const userChunks = chunks.filter(c => c.role === "user");
+    if (userChunks.length === 0) {
+      return { pass: false, skipReason: "无用户消息" };
+    }
+
+    const assistantChunks = chunks.filter(c => c.role === "assistant");
+    if (assistantChunks.length === 0) {
+      return { pass: false, skipReason: "无助手回复" };
+    }
+
+    return { pass: true, skipReason: "" };
+  }
+
+  async evaluateCreate(task: Task): Promise<CreateEvalResult> {
+    const cfg = this.getProviderConfig();
+    if (!cfg) {
+      return { shouldGenerate: false, reason: "no LLM configured", suggestedName: "", suggestedTags: [], confidence: 0 };
+    }
+
+    const prompt = CREATE_EVAL_PROMPT
+      .replace("{TITLE}", task.title)
+      .replace("{SUMMARY}", task.summary.slice(0, 3000));
+
+    try {
+      const raw = await this.callLLM(cfg, prompt);
+      return this.parseJSON<CreateEvalResult>(raw, {
+        shouldGenerate: false, reason: "parse failed", suggestedName: "", suggestedTags: [], confidence: 0,
+      });
+    } catch (err) {
+      this.ctx.log.warn(`SkillEvaluator.evaluateCreate failed: ${err}`);
+      return { shouldGenerate: false, reason: `error: ${err}`, suggestedName: "", suggestedTags: [], confidence: 0 };
+    }
+  }
+
+  async evaluateUpgrade(task: Task, skill: Skill, skillContent: string): Promise<UpgradeEvalResult> {
+    const cfg = this.getProviderConfig();
+    if (!cfg) {
+      return { shouldUpgrade: false, upgradeType: "refine", dimensions: [], reason: "no LLM configured", mergeStrategy: "", confidence: 0 };
+    }
+
+    const prompt = UPGRADE_EVAL_PROMPT
+      .replace("{VERSION}", String(skill.version))
+      .replace("{SKILL_NAME}", skill.name)
+      .replace("{SKILL_CONTENT}", skillContent.slice(0, 4000))
+      .replace("{TITLE}", task.title)
+      .replace("{SUMMARY}", task.summary.slice(0, 3000));
+
+    try {
+      const raw = await this.callLLM(cfg, prompt);
+      return this.parseJSON<UpgradeEvalResult>(raw, {
+        shouldUpgrade: false, upgradeType: "refine", dimensions: [], reason: "parse failed", mergeStrategy: "", confidence: 0,
+      });
+    } catch (err) {
+      this.ctx.log.warn(`SkillEvaluator.evaluateUpgrade failed: ${err}`);
+      return { shouldUpgrade: false, upgradeType: "refine", dimensions: [], reason: `error: ${err}`, mergeStrategy: "", confidence: 0 };
+    }
+  }
+
+  private getProviderConfig(): SummarizerConfig | undefined {
+    return this.ctx.config.skillEvolution?.summarizer ?? this.ctx.config.summarizer;
+  }
+
+  private async callLLM(cfg: SummarizerConfig, userContent: string): Promise<string> {
+    const endpoint = this.normalizeEndpoint(cfg.endpoint ?? "https://api.openai.com/v1/chat/completions");
+    const model = cfg.model ?? "gpt-4o-mini";
+    const headers: Record<string, string> = {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${cfg.apiKey}`,
+      ...cfg.headers,
+    };
+
+    const resp = await fetch(endpoint, {
+      method: "POST",
+      headers,
+      body: JSON.stringify({
+        model,
+        temperature: cfg.temperature ?? 0.1,
+        max_tokens: 1024,
+        messages: [
+          { role: "user", content: userContent },
+        ],
+      }),
+      signal: AbortSignal.timeout(cfg.timeoutMs ?? 30_000),
+    });
+
+    if (!resp.ok) {
+      const body = await resp.text();
+      throw new Error(`LLM call failed (${resp.status}): ${body}`);
+    }
+
+    const json = (await resp.json()) as { choices: Array<{ message: { content: string } }> };
+    return json.choices[0]?.message?.content?.trim() ?? "";
+  }
+
+  private parseJSON<T>(raw: string, fallback: T): T {
+    const jsonMatch = raw.match(/\{[\s\S]*\}/);
+    if (!jsonMatch) return fallback;
+    try {
+      return JSON.parse(jsonMatch[0]) as T;
+    } catch {
+      return fallback;
+    }
+  }
+
+  private normalizeEndpoint(url: string): string {
+    const stripped = url.replace(/\/+$/, "");
+    if (stripped.endsWith("/chat/completions")) return stripped;
+    if (stripped.endsWith("/completions")) return stripped;
+    return `${stripped}/chat/completions`;
+  }
+}

package/src/skill/evolver.ts

@@ -0,0 +1,169 @@
+import * as fs from "fs";
+import * as path from "path";
+import type { SqliteStore } from "../storage/sqlite";
+import type { RecallEngine } from "../recall/engine";
+import type { Task, Skill, Chunk, PluginContext } from "../types";
+import { DEFAULTS } from "../types";
+import { SkillEvaluator } from "./evaluator";
+import { SkillGenerator } from "./generator";
+import { SkillUpgrader } from "./upgrader";
+import { SkillInstaller } from "./installer";
+
+export class SkillEvolver {
+  private evaluator: SkillEvaluator;
+  private generator: SkillGenerator;
+  private upgrader: SkillUpgrader;
+  private installer: SkillInstaller;
+  private processing = false;
+
+  constructor(
+    private store: SqliteStore,
+    private engine: RecallEngine,
+    private ctx: PluginContext,
+  ) {
+    this.evaluator = new SkillEvaluator(ctx);
+    this.generator = new SkillGenerator(store, engine, ctx);
+    this.upgrader = new SkillUpgrader(store, ctx);
+    this.installer = new SkillInstaller(store, ctx);
+  }
+
+  async onTaskCompleted(task: Task): Promise<void> {
+    const enabled = this.ctx.config.skillEvolution?.enabled ?? DEFAULTS.skillEvolutionEnabled;
+    const autoEval = this.ctx.config.skillEvolution?.autoEvaluate ?? DEFAULTS.skillAutoEvaluate;
+    if (!enabled || !autoEval) return;
+
+    if (this.processing) {
+      this.ctx.log.debug("SkillEvolver: already processing, skipping");
+      return;
+    }
+    this.processing = true;
+    try {
+      await this.process(task);
+    } catch (err) {
+      this.ctx.log.error(`SkillEvolver error: ${err}`);
+    } finally {
+      this.processing = false;
+    }
+  }
+
+  private async process(task: Task): Promise<void> {
+    const chunks = this.store.getChunksByTask(task.id);
+
+    const { pass, skipReason } = this.evaluator.passesRuleFilter(chunks, task);
+    if (!pass) {
+      this.ctx.log.debug(`SkillEvolver: task ${task.id} skipped by rule filter: ${skipReason} (chunks=${chunks.length})`);
+      this.store.setTaskSkillMeta(task.id, { skillStatus: "skipped", skillReason: skipReason });
+      return;
+    }
+
+    const relatedSkill = await this.findRelatedSkill(task);
+
+    if (relatedSkill) {
+      await this.handleExistingSkill(task, chunks, relatedSkill);
+    } else {
+      await this.handleNewSkill(task, chunks);
+    }
+  }
+
+  private async findRelatedSkill(task: Task): Promise<Skill | null> {
+    try {
+      const result = await this.engine.search({
+        query: task.summary.slice(0, 500),
+        maxResults: 10,
+        minScore: 0.5,
+      });
+
+      for (const hit of result.hits) {
+        if (hit.skillId) {
+          const skill = this.store.getSkill(hit.skillId);
+          if (skill && (skill.status === "active" || skill.status === "draft")) {
+            this.ctx.log.debug(`SkillEvolver: found related skill "${skill.name}" via memory search`);
+            return skill;
+          }
+        }
+      }
+    } catch (err) {
+      this.ctx.log.warn(`SkillEvolver: memory search for related skill failed: ${err}`);
+    }
+    return null;
+  }
+
+  private async handleExistingSkill(task: Task, chunks: Chunk[], skill: Skill): Promise<void> {
+    const skillContent = this.readSkillContent(skill);
+    if (!skillContent) {
+      this.ctx.log.warn(`SkillEvolver: cannot read skill "${skill.name}" content, treating as new`);
+      await this.handleNewSkill(task, chunks);
+      return;
+    }
+
+    const minConfidence = this.ctx.config.skillEvolution?.minConfidence ?? DEFAULTS.skillMinConfidence;
+    const evalResult = await this.evaluator.evaluateUpgrade(task, skill, skillContent);
+
+    if (evalResult.shouldUpgrade && evalResult.confidence >= minConfidence) {
+      this.ctx.log.info(`SkillEvolver: upgrading skill "${skill.name}" — ${evalResult.reason}`);
+      const { upgraded } = await this.upgrader.upgrade(task, skill, evalResult);
+
+      this.markChunksWithSkill(chunks, skill.id);
+
+      if (upgraded) {
+        this.store.linkTaskSkill(task.id, skill.id, "evolved_from", skill.version + 1);
+        this.installer.syncIfInstalled(skill.name);
+      } else {
+        this.store.linkTaskSkill(task.id, skill.id, "applied_to", skill.version);
+      }
+    } else if (evalResult.confidence < 0.3) {
+      // Low confidence means the matched skill is likely unrelated — try creating a new one
+      this.ctx.log.info(
+        `SkillEvolver: skill "${skill.name}" has low relevance (confidence=${evalResult.confidence}), ` +
+        `falling back to new skill evaluation for task "${task.title}"`,
+      );
+      await this.handleNewSkill(task, chunks);
+    } else {
+      this.ctx.log.debug(`SkillEvolver: skill "${skill.name}" not worth upgrading (confidence=${evalResult.confidence})`);
+      this.markChunksWithSkill(chunks, skill.id);
+      this.store.linkTaskSkill(task.id, skill.id, "applied_to", skill.version);
+    }
+  }
+
+  private async handleNewSkill(task: Task, chunks: Chunk[]): Promise<void> {
+    const minConfidence = this.ctx.config.skillEvolution?.minConfidence ?? DEFAULTS.skillMinConfidence;
+    const evalResult = await this.evaluator.evaluateCreate(task);
+
+    if (evalResult.shouldGenerate && evalResult.confidence >= minConfidence) {
+      this.ctx.log.info(`SkillEvolver: generating new skill "${evalResult.suggestedName}" — ${evalResult.reason}`);
+      this.store.setTaskSkillMeta(task.id, { skillStatus: "generating", skillReason: evalResult.reason });
+
+      const skill = await this.generator.generate(task, chunks, evalResult);
+      this.markChunksWithSkill(chunks, skill.id);
+      this.store.linkTaskSkill(task.id, skill.id, "generated_from", 1);
+      this.store.setTaskSkillMeta(task.id, { skillStatus: "generated", skillReason: evalResult.reason });
+
+      const autoInstall = this.ctx.config.skillEvolution?.autoInstall ?? DEFAULTS.skillAutoInstall;
+      if (autoInstall && skill.status === "active") {
+        this.installer.install(skill.id);
+      }
+    } else {
+      const reason = evalResult.reason || `confidence不足 (${evalResult.confidence} < ${minConfidence})`;
+      this.ctx.log.debug(`SkillEvolver: task "${task.title}" not worth generating skill — ${reason}`);
+      this.store.setTaskSkillMeta(task.id, { skillStatus: "not_generated", skillReason: reason });
+    }
+  }
+
+  private markChunksWithSkill(chunks: Chunk[], skillId: string): void {
+    for (const chunk of chunks) {
+      this.store.setChunkSkillId(chunk.id, skillId);
+    }
+    this.ctx.log.debug(`SkillEvolver: marked ${chunks.length} chunks with skill_id=${skillId}`);
+  }
+
+  private readSkillContent(skill: Skill): string | null {
+    const filePath = path.join(skill.dirPath, "SKILL.md");
+    try {
+      if (fs.existsSync(filePath)) {
+        return fs.readFileSync(filePath, "utf-8");
+      }
+    } catch { /* fall through */ }
+    const sv = this.store.getLatestSkillVersion(skill.id);
+    return sv?.content ?? null;
+  }
+}