@memtensor/memos-local-openclaw-plugin 1.0.4-beta.10 → 1.0.4-beta.11

package/src/skill/upgrader.ts

@@ -91,18 +91,30 @@ export class SkillUpgrader {
       return { upgraded: false, qualityScore: null };
     }
 
+    const backupDir = skill.dirPath + ".backup-" + Date.now();
+    try { fs.cpSync(skill.dirPath, backupDir, { recursive: true }); } catch { /* best-effort */ }
+
     fs.writeFileSync(path.join(skill.dirPath, "SKILL.md"), newContent, "utf-8");
 
+    await this.rebuildCompanionFiles(skill, newContent, task);
+
     const validation = await this.validator.validate(skill.dirPath, {
       previousContent: currentContent,
     });
 
     if (!validation.valid) {
       this.ctx.log.warn(`SkillUpgrader: validation failed for "${skill.name}", reverting: ${validation.errors.join("; ")}`);
-      fs.writeFileSync(path.join(skill.dirPath, "SKILL.md"), currentContent, "utf-8");
+      if (fs.existsSync(backupDir)) {
+        fs.rmSync(skill.dirPath, { recursive: true });
+        fs.renameSync(backupDir, skill.dirPath);
+      } else {
+        fs.writeFileSync(path.join(skill.dirPath, "SKILL.md"), currentContent, "utf-8");
+      }
       return { upgraded: false, qualityScore: null };
     }
 
+    try { if (fs.existsSync(backupDir)) fs.rmSync(backupDir, { recursive: true }); } catch { /* cleanup */ }
+
     const newVersion = skill.version + 1;
     const newDescription = this.parseDescription(newContent) || skill.description;
 
@@ -216,4 +228,130 @@ export class SkillUpgrader {
     if (match2) return match2[1];
     return "";
   }
+
+  private async rebuildCompanionFiles(skill: Skill, newContent: string, task: Task): Promise<void> {
+    const chain = buildSkillConfigChain(this.ctx);
+    if (chain.length === 0) return;
+
+    const chunks = this.store.getChunksByTask(task.id);
+    const conversationText = chunks
+      .filter(c => c.role === "user" || c.role === "assistant" || c.role === "tool")
+      .map(c => `[${c.role === "user" ? "User" : c.role === "assistant" ? "Assistant" : "Tool"}]: ${c.content.slice(0, 500)}`)
+      .join("\n\n")
+      .slice(0, 6000);
+
+    const scriptsPrompt = `Based on the following upgraded SKILL.md and task record, extract reusable automation scripts.
+Rules:
+- Only extract if the task record contains concrete shell commands, Python scripts, or TypeScript code that form a complete, reusable automation.
+- Each script must be self-contained and runnable.
+- If there are no automatable scripts, return an empty array.
+- Don't fabricate scripts — only extract what was actually used.
+
+SKILL.md:
+${newContent.slice(0, 4000)}
+
+Task conversation highlights:
+${conversationText}
+
+Reply with a JSON array only:
+[{"filename": "deploy.sh", "content": "#!/bin/bash\\n..."}]
+If no scripts, reply with: []`;
+
+    try {
+      const raw = await callLLMWithFallback(chain, scriptsPrompt, this.ctx.log, "SkillUpgrader.scripts", {
+        maxTokens: 3000, temperature: 0.1, timeoutMs: 60_000, openclawAPI: this.ctx.openclawAPI,
+      });
+      const scripts = this.parseJSONArray<{ filename: string; content: string }>(raw);
+
+      const scriptsDir = path.join(skill.dirPath, "scripts");
+      if (fs.existsSync(scriptsDir)) fs.rmSync(scriptsDir, { recursive: true });
+      if (scripts.length > 0) {
+        fs.mkdirSync(scriptsDir, { recursive: true });
+        for (const s of scripts) {
+          fs.writeFileSync(path.join(scriptsDir, s.filename), s.content, "utf-8");
+        }
+        this.ctx.log.info(`SkillUpgrader: rebuilt ${scripts.length} scripts for "${skill.name}"`);
+      }
+    } catch (err) {
+      this.ctx.log.warn(`SkillUpgrader: companion scripts rebuild failed: ${err}`);
+    }
+
+    try {
+      const evalsPrompt = `Based on the following skill, generate 3-4 realistic test prompts that should trigger this skill.
+Requirements:
+- Write test prompts that a real user would type, mix direct and indirect phrasings
+- LANGUAGE RULE: Write in the SAME language as the skill content.
+
+Skill:
+${newContent.slice(0, 4000)}
+
+Reply with a JSON array only:
+[{"id": 1, "prompt": "A realistic user message", "expectations": ["Expected behavior 1"], "trigger_confidence": "high"}]`;
+
+      const raw = await callLLMWithFallback(chain, evalsPrompt, this.ctx.log, "SkillUpgrader.evals", {
+        maxTokens: 2000, temperature: 0.3, timeoutMs: 60_000, openclawAPI: this.ctx.openclawAPI,
+      });
+      const evals = this.parseJSONArray<{ id: number; prompt: string; expectations: string[] }>(raw);
+
+      const evalsDir = path.join(skill.dirPath, "evals");
+      if (fs.existsSync(evalsDir)) fs.rmSync(evalsDir, { recursive: true });
+      if (evals.length > 0) {
+        fs.mkdirSync(evalsDir, { recursive: true });
+        fs.writeFileSync(
+          path.join(evalsDir, "evals.json"),
+          JSON.stringify({ skill_name: skill.name, evals }, null, 2),
+          "utf-8",
+        );
+        this.ctx.log.info(`SkillUpgrader: rebuilt ${evals.length} evals for "${skill.name}"`);
+      }
+    } catch (err) {
+      this.ctx.log.warn(`SkillUpgrader: companion evals rebuild failed: ${err}`);
+    }
+
+    try {
+      const refsPrompt = `Based on the following upgraded SKILL.md and task record, extract reference documentation worth preserving.
+Rules:
+- Only extract real reference content that appeared in the task (API docs, config examples, architecture notes).
+- Each reference should be a standalone document useful for understanding the skill's domain.
+- If there are no meaningful references, return an empty array.
+- Don't fabricate content — only extract what was actually discussed or used.
+- LANGUAGE RULE: Write in the SAME language as the skill content.
+
+SKILL.md:
+${newContent.slice(0, 4000)}
+
+Task conversation highlights:
+${conversationText}
+
+Reply with a JSON array only:
+[{"filename": "api-notes.md", "content": "# API Reference\\n..."}]
+If no references, reply with: []`;
+
+      const raw = await callLLMWithFallback(chain, refsPrompt, this.ctx.log, "SkillUpgrader.references", {
+        maxTokens: 3000, temperature: 0.1, timeoutMs: 60_000, openclawAPI: this.ctx.openclawAPI,
+      });
+      const refs = this.parseJSONArray<{ filename: string; content: string }>(raw);
+
+      const refsDir = path.join(skill.dirPath, "references");
+      if (fs.existsSync(refsDir)) fs.rmSync(refsDir, { recursive: true });
+      if (refs.length > 0) {
+        fs.mkdirSync(refsDir, { recursive: true });
+        for (const r of refs) {
+          fs.writeFileSync(path.join(refsDir, r.filename), r.content, "utf-8");
+        }
+        this.ctx.log.info(`SkillUpgrader: rebuilt ${refs.length} references for "${skill.name}"`);
+      }
+    } catch (err) {
+      this.ctx.log.warn(`SkillUpgrader: companion references rebuild failed: ${err}`);
+    }
+  }
+
+  private parseJSONArray<T>(raw: string): T[] {
+    const match = raw.match(/\[[\s\S]*\]/);
+    if (!match) return [];
+    try {
+      const arr = JSON.parse(match[0]);
+      return Array.isArray(arr) ? arr : [];
+    } catch { return []; }
+  }
 }

package/src/skill/validator.ts

@@ -31,6 +31,9 @@ export class SkillValidator {
     this.validateFormat(dirPath, result);
     if (!result.valid) return result;
 
+    this.checkCompanionConsistency(dirPath, result);
+    this.scanSecrets(dirPath, result);
+
     if (opts?.previousContent) {
       this.regressionCheck(dirPath, opts.previousContent, result);
     }
@@ -133,6 +136,82 @@ export class SkillValidator {
     }
   }
 
+  private checkCompanionConsistency(dirPath: string, result: ValidationResult): void {
+    const skillMdPath = path.join(dirPath, "SKILL.md");
+    const content = fs.readFileSync(skillMdPath, "utf-8");
+
+    const referencedScripts = [...content.matchAll(/`scripts\/([^`]+)`/g)].map(m => m[1]);
+    const referencedRefs = [...content.matchAll(/`references\/([^`]+)`/g)].map(m => m[1]);
+
+    const scriptsDir = path.join(dirPath, "scripts");
+    const refsDir = path.join(dirPath, "references");
+
+    for (const f of referencedScripts) {
+      if (!fs.existsSync(path.join(scriptsDir, f))) {
+        result.warnings.push(`SKILL.md references scripts/${f} but file does not exist`);
+      }
+    }
+    for (const f of referencedRefs) {
+      if (!fs.existsSync(path.join(refsDir, f))) {
+        result.warnings.push(`SKILL.md references references/${f} but file does not exist`);
+      }
+    }
+
+    if (fs.existsSync(scriptsDir)) {
+      try {
+        const actualScripts = fs.readdirSync(scriptsDir);
+        for (const f of actualScripts) {
+          if (!referencedScripts.includes(f)) {
+            result.warnings.push(`scripts/${f} exists but is not referenced in SKILL.md`);
+          }
+        }
+      } catch { /* best-effort */ }
+    }
+
+    const evalsPath = path.join(dirPath, "evals", "evals.json");
+    if (fs.existsSync(evalsPath)) {
+      try {
+        const evalsData = JSON.parse(fs.readFileSync(evalsPath, "utf-8"));
+        if (!Array.isArray(evalsData?.evals) && !Array.isArray(evalsData)) {
+          result.warnings.push("evals/evals.json exists but has unexpected structure");
+        }
+      } catch {
+        result.warnings.push("evals/evals.json exists but is not valid JSON");
+      }
+    }
+  }
+
+  private static readonly SECRET_PATTERNS: Array<{ label: string; regex: RegExp }> = [
+    { label: "API key (sk-...)", regex: /\bsk-[a-zA-Z0-9]{20,}\b/ },
+    { label: "Bearer token", regex: /\bBearer\s+[a-zA-Z0-9_\-.]{20,}\b/ },
+    { label: "AWS key", regex: /\bAKIA[0-9A-Z]{16}\b/ },
+    { label: "Generic secret assignment", regex: /(api[_-]?key|secret|token|password|credential)\s*[:=]\s*["'][^"']{8,}["']/i },
+    { label: "Base64 encoded secret (long)", regex: /\b[A-Za-z0-9+/]{40,}={0,2}\b/ },
+  ];
+
+  private scanSecrets(dirPath: string, result: ValidationResult): void {
+    const filesToScan = ["SKILL.md"];
+    const scriptsDir = path.join(dirPath, "scripts");
+    if (fs.existsSync(scriptsDir)) {
+      try {
+        for (const f of fs.readdirSync(scriptsDir)) filesToScan.push(path.join("scripts", f));
+      } catch { /* best-effort */ }
+    }
+
+    for (const relPath of filesToScan) {
+      const fullPath = path.join(dirPath, relPath);
+      if (!fs.existsSync(fullPath)) continue;
+      try {
+        const content = fs.readFileSync(fullPath, "utf-8");
+        for (const { label, regex } of SkillValidator.SECRET_PATTERNS) {
+          if (regex.test(content)) {
+            result.warnings.push(`Potential secret detected in ${relPath}: ${label}`);
+          }
+        }
+      } catch { /* best-effort */ }
+    }
+  }
+
   private async assessQuality(dirPath: string, result: ValidationResult): Promise<void> {
     const chain = buildSkillConfigChain(this.ctx);
     if (chain.length === 0) return;

package/src/storage/sqlite.ts

@@ -114,6 +114,8 @@ export class SqliteStore {
     this.migrateHubTables();
     this.migrateHubFtsToTrigram();
     this.migrateLocalSharedTasksOwner();
+    this.migrateHubUserIdentityFields();
+    this.migrateClientHubConnectionIdentityFields();
     this.log.debug("Database schema initialized");
   }
 
@@ -131,6 +133,49 @@ export class SqliteStore {
     } catch { /* table may not exist yet */ }
   }
 
+  private migrateHubUserIdentityFields(): void {
+    try {
+      const cols = this.db.prepare("PRAGMA table_info(hub_users)").all() as Array<{ name: string }>;
+      if (cols.length === 0) return;
+      if (!cols.some(c => c.name === "identity_key")) {
+        this.db.exec("ALTER TABLE hub_users ADD COLUMN identity_key TEXT NOT NULL DEFAULT ''");
+        this.db.exec("CREATE INDEX IF NOT EXISTS idx_hub_users_identity_key ON hub_users(identity_key)");
+        this.log.info("Migrated: added identity_key to hub_users");
+      }
+      if (!cols.some(c => c.name === "left_at")) {
+        this.db.exec("ALTER TABLE hub_users ADD COLUMN left_at INTEGER");
+        this.log.info("Migrated: added left_at to hub_users");
+      }
+      if (!cols.some(c => c.name === "removed_at")) {
+        this.db.exec("ALTER TABLE hub_users ADD COLUMN removed_at INTEGER");
+        this.log.info("Migrated: added removed_at to hub_users");
+      }
+      if (!cols.some(c => c.name === "rejected_at")) {
+        this.db.exec("ALTER TABLE hub_users ADD COLUMN rejected_at INTEGER");
+        this.log.info("Migrated: added rejected_at to hub_users");
+      }
+      if (!cols.some(c => c.name === "rejoin_requested_at")) {
+        this.db.exec("ALTER TABLE hub_users ADD COLUMN rejoin_requested_at INTEGER");
+        this.log.info("Migrated: added rejoin_requested_at to hub_users");
+      }
+    } catch { /* table may not exist yet */ }
+  }
+
+  private migrateClientHubConnectionIdentityFields(): void {
+    try {
+      const cols = this.db.prepare("PRAGMA table_info(client_hub_connection)").all() as Array<{ name: string }>;
+      if (cols.length === 0) return;
+      if (!cols.some(c => c.name === "identity_key")) {
+        this.db.exec("ALTER TABLE client_hub_connection ADD COLUMN identity_key TEXT NOT NULL DEFAULT ''");
+        this.log.info("Migrated: added identity_key to client_hub_connection");
+      }
+      if (!cols.some(c => c.name === "last_known_status")) {
+        this.db.exec("ALTER TABLE client_hub_connection ADD COLUMN last_known_status TEXT NOT NULL DEFAULT ''");
+        this.log.info("Migrated: added last_known_status to client_hub_connection");
+      }
+    } catch { /* table may not exist yet */ }
+  }
+
   private migrateOwnerFields(): void {
     const chunkCols = this.db.prepare("PRAGMA table_info(chunks)").all() as Array<{ name: string }>;
     if (!chunkCols.some((c) => c.name === "owner")) {
@@ -1731,16 +1776,18 @@ export class SqliteStore {
 
   setClientHubConnection(conn: ClientHubConnection): void {
     this.db.prepare(`
-      INSERT INTO client_hub_connection (id, hub_url, user_id, username, user_token, role, connected_at)
-      VALUES (1, ?, ?, ?, ?, ?, ?)
+      INSERT INTO client_hub_connection (id, hub_url, user_id, username, user_token, role, connected_at, identity_key, last_known_status)
+      VALUES (1, ?, ?, ?, ?, ?, ?, ?, ?)
       ON CONFLICT(id) DO UPDATE SET
         hub_url = excluded.hub_url,
         user_id = excluded.user_id,
         username = excluded.username,
         user_token = excluded.user_token,
         role = excluded.role,
-        connected_at = excluded.connected_at
-    `).run(conn.hubUrl, conn.userId, conn.username, conn.userToken, conn.role, conn.connectedAt);
+        connected_at = excluded.connected_at,
+        identity_key = excluded.identity_key,
+        last_known_status = excluded.last_known_status
+    `).run(conn.hubUrl, conn.userId, conn.username, conn.userToken, conn.role, conn.connectedAt, conn.identityKey ?? "", conn.lastKnownStatus ?? "");
   }
 
   getClientHubConnection(): ClientHubConnection | null {
@@ -1847,8 +1894,8 @@ export class SqliteStore {
 
   upsertHubUser(user: HubUserRecord): void {
     this.db.prepare(`
-      INSERT INTO hub_users (id, username, device_name, role, status, token_hash, created_at, approved_at)
-      VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+      INSERT INTO hub_users (id, username, device_name, role, status, token_hash, created_at, approved_at, identity_key, left_at, removed_at, rejected_at, rejoin_requested_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
       ON CONFLICT(id) DO UPDATE SET
         username = excluded.username,
         device_name = excluded.device_name,
@@ -1856,8 +1903,13 @@ export class SqliteStore {
         status = excluded.status,
         token_hash = excluded.token_hash,
         created_at = excluded.created_at,
-        approved_at = excluded.approved_at
-    `).run(user.id, user.username, user.deviceName ?? "", user.role, user.status, user.tokenHash, user.createdAt, user.approvedAt);
+        approved_at = excluded.approved_at,
+        identity_key = excluded.identity_key,
+        left_at = excluded.left_at,
+        removed_at = excluded.removed_at,
+        rejected_at = excluded.rejected_at,
+        rejoin_requested_at = excluded.rejoin_requested_at
+    `).run(user.id, user.username, user.deviceName ?? "", user.role, user.status, user.tokenHash, user.createdAt, user.approvedAt, user.identityKey ?? "", user.leftAt ?? null, user.removedAt ?? null, user.rejectedAt ?? null, user.rejoinRequestedAt ?? null);
   }
 
   getHubUser(userId: string): HubUserRecord | null {
@@ -1881,7 +1933,18 @@ export class SqliteStore {
       const result = this.db.prepare('DELETE FROM hub_users WHERE id = ?').run(userId);
       return result.changes > 0;
     }
-    const result = this.db.prepare("UPDATE hub_users SET status = 'removed', token_hash = '' WHERE id = ?").run(userId);
+    const result = this.db.prepare("UPDATE hub_users SET status = 'removed', token_hash = '', removed_at = ? WHERE id = ?").run(Date.now(), userId);
+    return result.changes > 0;
+  }
+
+  findHubUserByIdentityKey(identityKey: string): HubUserRecord | null {
+    if (!identityKey) return null;
+    const row = this.db.prepare('SELECT * FROM hub_users WHERE identity_key = ?').get(identityKey) as HubUserRow | undefined;
+    return row ? rowToHubUser(row) : null;
+  }
+
+  markHubUserLeft(userId: string): boolean {
+    const result = this.db.prepare("UPDATE hub_users SET status = 'left', token_hash = '', left_at = ? WHERE id = ?").run(Date.now(), userId);
     return result.changes > 0;
   }
 
@@ -2001,6 +2064,18 @@ export class SqliteStore {
     return out;
   }
 
+  getVisibleHubSkillEmbeddings(): Array<{ skillId: string; vector: Float32Array }> {
+    const rows = this.db.prepare(`
+      SELECT hse.skill_id, hse.vector, hse.dimensions
+      FROM hub_skill_embeddings hse
+      JOIN hub_skills hs ON hs.id = hse.skill_id
+    `).all() as Array<{ skill_id: string; vector: Buffer; dimensions: number }>;
+    return rows.map(r => ({
+      skillId: r.skill_id,
+      vector: new Float32Array(r.vector.buffer, r.vector.byteOffset, r.dimensions),
+    }));
+  }
+
   searchHubChunks(query: string, options?: { userId?: string; maxResults?: number }): Array<{ hit: HubSearchRow; rank: number }> {
     const limit = options?.maxResults ?? 10;
     const userId = options?.userId ?? "";
@@ -2549,6 +2624,8 @@ interface ClientHubConnection {
   userToken: string;
   role: UserRole;
   connectedAt: number;
+  identityKey?: string;
+  lastKnownStatus?: string;
 }
 
 interface ClientHubConnectionRow {
@@ -2558,6 +2635,8 @@ interface ClientHubConnectionRow {
   user_token: string;
   role: string;
   connected_at: number;
+  identity_key?: string;
+  last_known_status?: string;
 }
 
 function rowToClientHubConnection(row: ClientHubConnectionRow): ClientHubConnection {
@@ -2568,6 +2647,8 @@ function rowToClientHubConnection(row: ClientHubConnectionRow): ClientHubConnect
     userToken: row.user_token,
     role: row.role as UserRole,
     connectedAt: row.connected_at,
+    identityKey: row.identity_key || "",
+    lastKnownStatus: row.last_known_status || "",
   };
 }
 
@@ -2577,6 +2658,11 @@ interface HubUserRecord extends UserInfo {
   approvedAt: number | null;
   lastIp: string;
   lastActiveAt: number | null;
+  identityKey?: string;
+  leftAt?: number | null;
+  removedAt?: number | null;
+  rejectedAt?: number | null;
+  rejoinRequestedAt?: number | null;
 }
 
 interface HubUserRow {
@@ -2590,6 +2676,11 @@ interface HubUserRow {
   approved_at: number | null;
   last_ip: string;
   last_active_at: number | null;
+  identity_key?: string;
+  left_at?: number | null;
+  removed_at?: number | null;
+  rejected_at?: number | null;
+  rejoin_requested_at?: number | null;
 }
 
 function rowToHubUser(row: HubUserRow): HubUserRecord {
@@ -2605,6 +2696,11 @@ function rowToHubUser(row: HubUserRow): HubUserRecord {
     approvedAt: row.approved_at,
     lastIp: row.last_ip || "",
     lastActiveAt: row.last_active_at ?? null,
+    identityKey: row.identity_key || "",
+    leftAt: row.left_at ?? null,
+    removedAt: row.removed_at ?? null,
+    rejectedAt: row.rejected_at ?? null,
+    rejoinRequestedAt: row.rejoin_requested_at ?? null,
   };
 }
 

package/src/types.ts

@@ -66,6 +66,8 @@ export interface ChunkRef {
 
 // ─── Search / Recall ───
 
+export type SearchHitOrigin = "local" | "local-shared" | "hub-memory" | "hub-remote";
+
 export interface SearchHit {
   summary: string;
   original_excerpt: string;
@@ -74,6 +76,7 @@ export interface SearchHit {
   taskId: string | null;
   skillId: string | null;
   owner?: string;
+  origin?: SearchHitOrigin;
   source: {
     ts: number;
     role: Role;
@@ -249,6 +252,10 @@ export interface SkillEvolutionConfig {
   minConfidence?: number;
   maxSkillLines?: number;
   autoInstall?: boolean;
+  autoRecallSkills?: boolean;
+  autoRecallSkillLimit?: number;
+  preferUpgradeExisting?: boolean;
+  redactSensitiveInSkill?: boolean;
   /** Optional independent LLM config for skill evaluation/validation. Falls back to main summarizer if not set. */
   summarizer?: SummarizerConfig;
 }
@@ -344,6 +351,10 @@ export const DEFAULTS = {
   skillMinConfidence: 0.7,
   skillMaxLines: 400,
   skillAutoInstall: false,
+  skillAutoRecall: true,
+  skillAutoRecallLimit: 2,
+  skillPreferUpgrade: true,
+  skillRedactSensitive: true,
 } as const;
 
 // ─── Plugin Hooks (OpenClaw integration) ───