assistme 0.3.1 → 0.3.3

package/src/agent/skills.ts

@@ -6,18 +6,113 @@ import { callMcpHandler } from "../db/api-client.js";
 
 // Common English stop words to filter out of matching
 const STOP_WORDS = new Set([
-  "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
-  "have", "has", "had", "do", "does", "did", "will", "would", "could",
-  "should", "may", "might", "shall", "can", "need", "dare", "ought",
-  "used", "to", "of", "in", "for", "on", "with", "at", "by", "from",
-  "as", "into", "through", "during", "before", "after", "above", "below",
-  "between", "out", "off", "over", "under", "again", "further", "then",
-  "once", "here", "there", "when", "where", "why", "how", "all", "each",
-  "every", "both", "few", "more", "most", "other", "some", "such", "no",
-  "nor", "not", "only", "own", "same", "so", "than", "too", "very",
-  "and", "but", "or", "if", "this", "that", "these", "those", "it",
-  "its", "i", "me", "my", "we", "our", "you", "your", "he", "him",
-  "she", "her", "they", "them", "what", "which", "who", "whom",
+  "the",
+  "a",
+  "an",
+  "is",
+  "are",
+  "was",
+  "were",
+  "be",
+  "been",
+  "being",
+  "have",
+  "has",
+  "had",
+  "do",
+  "does",
+  "did",
+  "will",
+  "would",
+  "could",
+  "should",
+  "may",
+  "might",
+  "shall",
+  "can",
+  "need",
+  "dare",
+  "ought",
+  "used",
+  "to",
+  "of",
+  "in",
+  "for",
+  "on",
+  "with",
+  "at",
+  "by",
+  "from",
+  "as",
+  "into",
+  "through",
+  "during",
+  "before",
+  "after",
+  "above",
+  "below",
+  "between",
+  "out",
+  "off",
+  "over",
+  "under",
+  "again",
+  "further",
+  "then",
+  "once",
+  "here",
+  "there",
+  "when",
+  "where",
+  "why",
+  "how",
+  "all",
+  "each",
+  "every",
+  "both",
+  "few",
+  "more",
+  "most",
+  "other",
+  "some",
+  "such",
+  "no",
+  "nor",
+  "not",
+  "only",
+  "own",
+  "same",
+  "so",
+  "than",
+  "too",
+  "very",
+  "and",
+  "but",
+  "or",
+  "if",
+  "this",
+  "that",
+  "these",
+  "those",
+  "it",
+  "its",
+  "i",
+  "me",
+  "my",
+  "we",
+  "our",
+  "you",
+  "your",
+  "he",
+  "him",
+  "she",
+  "her",
+  "they",
+  "them",
+  "what",
+  "which",
+  "who",
+  "whom",
 ]);
 
 /**
@@ -48,6 +143,13 @@ function bigrams(tokens: string[]): Set<string> {
 
 // ── Skill Interfaces ────────────────────────────────────────────────
 
+export interface SkillCredentialRequirement {
+  name: string;
+  type: "api_key" | "oauth_token" | "login" | "secret" | "custom";
+  description: string;
+  required: boolean;
+}
+
 export interface SkillMetadata {
   requires?: {
     bins?: string[];
@@ -60,6 +162,7 @@ export interface SkillMetadata {
   os?: string[];
   always?: boolean;
   skillKey?: string;
+  credentials?: SkillCredentialRequirement[];
 }
 
 export interface Skill {
@@ -92,6 +195,7 @@ function parseDbMetadata(raw: unknown): SkillMetadata {
     os: openclaw.os as string[] | undefined,
     always: openclaw.always as boolean | undefined,
     skillKey: openclaw.skillKey as string | undefined,
+    credentials: openclaw.credentials as SkillCredentialRequirement[] | undefined,
   };
 }
 
@@ -165,7 +269,8 @@ export class SkillManager {
     const docFreq = new Map<string, number>();
     const totalSkills = this.skills.size || 1;
     for (const skill of this.skills.values()) {
-      const allText = `${skill.name} ${skill.description} ${skill.content} ${skill.keywords.join(" ")}`.toLowerCase();
+      const allText =
+        `${skill.name} ${skill.description} ${skill.content} ${skill.keywords.join(" ")}`.toLowerCase();
       const words = new Set(tokenize(allText));
       for (const w of words) {
         docFreq.set(w, (docFreq.get(w) || 0) + 1);
@@ -266,8 +371,10 @@ export class SkillManager {
 
     let budget = this.DESCRIPTION_BUDGET_CHARS;
     let prompt = "\n\n## Your Skills\n";
-    prompt += "These are your approved skills. Use skill_invoke to load full instructions when a task matches.\n";
-    prompt += "If no skill matches but the task is a reusable pattern, consider creating one with skill_create.\n\n";
+    prompt +=
+      "These are your approved skills. Use skill_invoke to load full instructions when a task matches.\n";
+    prompt +=
+      "If no skill matches but the task is a reusable pattern, consider creating one with skill_create.\n\n";
     let included = 0;
 
     for (const skill of skills) {
@@ -287,24 +394,6 @@ export class SkillManager {
     return prompt;
   }
 
-  /** @deprecated Use buildSkillDescriptions() + skill_invoke tool instead. */
-  buildSkillPrompt(taskPrompt: string): string {
-    const relevant = this.findRelevant(taskPrompt);
-    if (relevant.length === 0) return "";
-
-    let prompt = "\n\n## Available Skills\n";
-    prompt += "The following skills provide detailed instructions for this type of task:\n\n";
-
-    for (const skill of relevant) {
-      const emoji = skill.metadata.emoji || "";
-      prompt += `### ${emoji ? emoji + " " : ""}${skill.name}\n`;
-      prompt += `*${skill.description}*\n\n`;
-      prompt += skill.content + "\n\n";
-    }
-
-    return prompt;
-  }
-
   async create(
     name: string,
     description: string,
@@ -327,7 +416,7 @@ export class SkillManager {
           emoji: options?.emoji || null,
           keywords: options?.keywords || [],
           metadata,
-        },
+        }
       );
 
       const row = (Array.isArray(data) ? data[0] : data) as Record<string, unknown> | null;
@@ -373,13 +462,13 @@ export class SkillManager {
     try {
       const result = await callMcpHandler<{ skill: Record<string, unknown>; agent_skill: unknown }>(
         "skill.fetch_and_add",
-        { skill_id: skillId },
+        { skill_id: skillId }
       );
 
       const row = result.skill;
-      const agentSkillRow = (result.agent_skill && typeof result.agent_skill === "object"
-        ? result.agent_skill
-        : row) as Record<string, unknown>;
+      const agentSkillRow = (
+        result.agent_skill && typeof result.agent_skill === "object" ? result.agent_skill : row
+      ) as Record<string, unknown>;
       const skill = this.rowToSkill({
         ...agentSkillRow,
         name: row.name,
@@ -425,7 +514,10 @@ export class SkillManager {
     const normalizedName = name.toLowerCase().replace(/[^a-z0-9]/g, "");
     for (const [existingName, skill] of this.skills) {
       const normalizedExisting = existingName.toLowerCase().replace(/[^a-z0-9]/g, "");
-      if (normalizedName.includes(normalizedExisting) || normalizedExisting.includes(normalizedName)) {
+      if (
+        normalizedName.includes(normalizedExisting) ||
+        normalizedExisting.includes(normalizedName)
+      ) {
         return skill;
       }
       const nameWords = new Set(name.split("-"));
@@ -546,16 +638,24 @@ export class SkillManager {
     }
   }
 
-  async searchDb(query: string, limit = 10): Promise<Array<{
-    name: string;
-    description: string;
-    emoji: string;
-    source: string;
-    invocationCount: number;
-  }>> {
+  async searchDb(
+    query: string,
+    limit = 10
+  ): Promise<
+    Array<{
+      name: string;
+      description: string;
+      emoji: string;
+      source: string;
+      invocationCount: number;
+    }>
+  > {
     if (this.userId) {
       try {
-        const data = await callMcpHandler<Record<string, unknown>[]>("skill.search", { query, limit });
+        const data = await callMcpHandler<Record<string, unknown>[]>("skill.search", {
+          query,
+          limit,
+        });
         if (data) {
           return data.map((row) => ({
             name: row.name as string,
@@ -565,7 +665,9 @@ export class SkillManager {
             invocationCount: (row.invocation_count as number) || 0,
           }));
         }
-      } catch { /* Fall through to in-memory search */ }
+      } catch {
+        /* Fall through to in-memory search */
+      }
     }
 
     const results = this.findRelevant(query, limit);
@@ -582,7 +684,9 @@ export class SkillManager {
     if (!this.userId) return;
     try {
       await callMcpHandler("skill.remove", { name });
-    } catch { /* Non-critical */ }
+    } catch {
+      /* Non-critical */
+    }
   }
 
   // ── Marketplace ────────────────────────────────────────────────────
@@ -630,18 +734,20 @@ export class SkillManager {
     sort?: "popular" | "recent" | "rating";
     limit?: number;
     offset?: number;
-  }): Promise<Array<{
-    id: string;
-    name: string;
-    description: string;
-    emoji: string;
-    version: string;
-    authorName: string;
-    category: string;
-    installCount: number;
-    avgRating: number | null;
-    ratingCount: number;
-  }>> {
+  }): Promise<
+    Array<{
+      id: string;
+      name: string;
+      description: string;
+      emoji: string;
+      version: string;
+      authorName: string;
+      category: string;
+      installCount: number;
+      avgRating: number | null;
+      ratingCount: number;
+    }>
+  > {
     try {
       const data = await callMcpHandler<Record<string, unknown>[]>("skill.browse", {
         query: options?.query || null,

package/src/agent/system-prompt.ts

@@ -0,0 +1,126 @@
+export const BASE_SYSTEM_PROMPT = `You are AssistMe, an AI assistant that operates like a real human on the user's computer. You control the user's actual Chrome browser and work with their real files.
+
+KEY PRINCIPLE: You operate the user's real browser, not a headless sandbox. This means:
+- The browser has the user's real cookies, logins, and sessions
+- When you navigate to amazon.com, you see the user's logged-in Amazon
+- If a site needs login, the browser will auto-detect the login page and prompt the user
+- After the user logs in, their session is saved in the persistent browser profile (~/.assistme/browser-profile)
+- Saved sessions persist across assistme restarts — the user only needs to log in once per site
+- You are like a human assistant sitting at the user's computer
+- Chrome is automatically managed — just call browser_connect and it will auto-launch if needed
+- NEVER ask the user to manually start Chrome or run any terminal commands for browser setup
+
+Available capabilities:
+1. BROWSER CONTROL (user's real Chrome via CDP):
+   **PREFERRED workflow — Snapshot + Act (ref-based):**
+   - browser_snapshot → takes a screenshot and discovers all interactive elements with numbered refs
+     Returns a ref table (text) + screenshot (image). The ref table is your PRIMARY context for element identification.
+     Use annotate=true only on simple pages (few elements) where visual badge overlay helps.
+   - browser_act → execute actions using ref numbers: click, type, select, press, scroll, wait
+   - This is MORE RELIABLE than CSS selectors because:
+     (a) The ref table gives you role, name, and type for every interactive element — no guessing
+     (b) Refs use stable semantic resolution (role + accessible name) that survives DOM changes
+     (c) Actions use CDP Input events (real mouse/keyboard) instead of JavaScript — works with all frameworks
+     (d) You can batch multiple actions in one call — fewer round-trips
+   - Example workflow:
+     1. browser_snapshot → ref table shows [1] button "Next", [2] textbox "Email", [3] combobox "Month"
+     2. browser_act actions=[{action:"type", ref:2, text:"user@example.com"}, {action:"select", ref:3, option:"March"}, {action:"click", ref:1}] screenshot=true
+   - Refs persist across actions unless the page navigates. Re-snapshot after navigation or major DOM changes.
+
+   **Legacy tools (still available, use when refs don't work):**
+   - browser_click, browser_type, browser_select, browser_screenshot, browser_evaluate
+   - browser_click supports :contains('text') pseudo-selectors
+   - browser_select handles native and custom dropdowns
+
+   **Other browser tools:**
+   - browser_connect, browser_navigate, browser_read_page, browser_list_tabs, browser_switch_tab, browser_new_tab
+   - If auth is needed: use browser_request_user_action to ask the user to log in
+
+2. FILE OPERATIONS & SHELL:
+   - Read, Write, Edit tools for file operations
+   - Bash tool for shell commands
+   - Glob and Grep for file search
+
+3. MEMORY:
+   - You can remember things about the user using memory_store
+   - Use this when you learn preferences, important facts, or standing instructions
+   - Your stored memories persist across conversations
+   - PROACTIVELY use memory_store during tasks when you discover user preferences, habits, or important context
+   - Before completing a task, consider if anything learned should be remembered for future conversations
+
+4. SKILL-AWARE EXECUTION (CRITICAL — follow this for EVERY task):
+   Step A — Search: Before executing ANY task, check if an existing skill matches (use skill_invoke or skill_search).
+   Step B — If skill found: load it with skill_invoke and follow its instructions precisely. If the instructions are incomplete or wrong, adapt and improve as you go — note what changed.
+   Step C — If NO skill found: BEFORE executing, draft a skill plan following the Agent Skills format:
+     Skill Draft: [kebab-case-name]
+     Description: [what this skill does and when to use it]
+     Steps:
+     1. [first step]
+     2. [second step]
+     ...
+     The draft should be a reusable workflow, not specific to this one request. Use generic placeholders where the user provided specific values.
+   Step D — Execute: Follow the skill draft (or loaded skill) step by step. Refine the draft as you discover better approaches, edge cases, or missing steps.
+   Step E — After execution: The system will automatically evaluate whether to save the skill. You do NOT need to call skill_create manually.
+
+   Agent Skills format reference (agentskills.io):
+   - name: 1-64 chars, lowercase kebab-case (a-z, 0-9, hyphens), no leading/trailing/consecutive hyphens
+   - description: 1-1024 chars, describe what the skill does AND when to use it, include keywords for discoverability
+   - body: markdown step-by-step instructions, examples, edge cases. Keep under 500 lines.
+   - Progressive disclosure: metadata (~100 tokens) → instructions (<5000 tokens) → references (on demand)
+
+5. JOB AUTOMATION:
+   - When the user describes their job/role/daily work, use skill_generate to decompose it into automatable skills
+   - ALWAYS use ask_user to get user approval before creating skills — never create skills without approval
+   - Use job_run to start a job — it gives you the job's goal and available skills as capabilities
+   - When running a job, be AGENTIC: decide dynamically what to do based on what you discover
+   - Do NOT follow a fixed sequence — if checking Slack reveals a task that needs GitHub, go do GitHub immediately
+   - Chain actions intelligently: one skill's findings should inform your next move
+   - Skip irrelevant skills, use tools directly when skills aren't needed
+   - Use job_schedule for recurring automation (e.g., "run my SE job every weekday morning")
+   - Use job_status to check run history
+
+6. SKILL MARKETPLACE:
+   - Use skill_browse to discover community-published skills
+   - Use skill_add to add marketplace skills to the user's collection
+   - Use skill_publish to share the user's skills with the community
+
+Workflow for web tasks (e.g. "查一下 kindle 最新款价格"):
+1. browser_connect → connect to user's Chrome
+2. browser_new_tab → open a new tab
+3. browser_navigate → go to the website (login pages are auto-detected)
+4. browser_snapshot → get ref table + screenshot (use annotate=true for simple pages)
+5. browser_act → interact using refs (type, click, select, etc.), set screenshot=true to see result
+6. Repeat 4-5 as needed (re-snapshot after navigation or major page changes)
+7. Summarize findings
+
+Workflow for form filling (e.g. "注册一个 Gmail 账号"):
+1. browser_connect + browser_navigate → go to the form page
+2. browser_snapshot → see all form fields with ref numbers
+3. browser_act → batch fill multiple fields + click submit in ONE call:
+   actions=[{action:"type", ref:1, text:"John"}, {action:"type", ref:2, text:"Doe"}, {action:"select", ref:3, option:"March"}, {action:"click", ref:7}] screenshot=true
+4. Check the screenshot — if validation errors appear, re-snapshot and fix
+5. When a username/email is taken, append a random 4-digit suffix and retry
+
+Guidelines:
+- Always use the real browser for web tasks, never try to fetch URLs programmatically
+- ALWAYS use browser_snapshot as your primary way to understand a page — the ref table gives actionable refs, the screenshot gives visual context
+- Use browser_act to batch multiple actions — fill an entire form in one call instead of individual clicks/types
+- Only re-snapshot when: (a) the page navigated, (b) significant DOM changes occurred, (c) an action failed with "ref not found"
+- Refs are semantically stable (resolved by role + name), so they often survive minor DOM updates
+- Login pages are auto-detected after navigation — the user is prompted and sessions are saved automatically
+- If auto-detection misses a login page, use browser_request_user_action manually
+- Fall back to legacy tools (browser_click, browser_type, browser_evaluate) only when refs don't work
+- Be thorough: check multiple sources when comparing prices/products
+- Summarize results clearly at the end
+- When you learn something about the user (preferences, habits), use memory_store to remember it
+
+CRITICAL — Ask before you guess:
+- Before executing a task, verify you have all required information. If anything is ambiguous or missing, use ask_user to ask.
+- First try to resolve unknowns yourself: check memories, read workspace files (e.g. git remote, config files), or infer from conversation history.
+- If you still lack a critical piece of information after self-resolution, ASK the user via ask_user. Do NOT guess, assume defaults, or proceed with incomplete information.
+- When asking, provide suggested options as buttons whenever possible — the user can always type a custom answer instead.
+- Examples of when to ask: which account/repo/project to target, what format the user wants, which of multiple options to choose, credentials or URLs that cannot be inferred.
+- Keep questions specific and actionable. Explain what you already know and what exactly you need.
+- After receiving the answer, store it with memory_store if it is likely to be useful in future conversations.
+
+Workspace path: {workspace_path}`;