jerob 1.0.0

package/plan/planner.ts

@@ -0,0 +1,183 @@
+import {
+  Output,
+  extractJsonMiddleware,
+  generateText,
+  stepCountIs,
+  tool,
+  wrapLanguageModel,
+  type LanguageModel,
+} from 'ai';
+import { z } from 'zod';
+import chalk from 'chalk';
+import { ActionTracker } from '../agent/action-tracker.ts';
+import { ToolExecutor } from '../agent/tool-executor.ts';
+import { defaultAgentConfig } from '../agent/types.ts';
+import { createWebTools } from './web-tools.ts';
+import type { Plan, PlanStep } from './index.ts';
+import { getAgentModel } from '../config/ai.config.ts';
+import { withLLMRetry, parseLLMError } from '../utils/llm-error';
+
+const planSchema = z.object({
+  researchSummary: z.string().optional(),
+  steps: z
+    .array(
+      z.object({
+        title: z.string(),
+        description: z.string(),
+        hints: z.array(z.string()).optional(),
+        complexity: z.enum(['low', 'medium', 'high']).optional(),
+      }),
+    )
+    .min(1)
+    .max(15),
+});
+
+function readOnlyTools(executor: ToolExecutor) {
+  return {
+    read_file: tool({
+      description: 'Read a workspace file (relative path).',
+      inputSchema: z.object({ path: z.string() }),
+      execute: async ({ path: p }) => executor.readFile(p),
+    }),
+    list_files: tool({
+      description: 'List files/dirs at a path.',
+      inputSchema: z.object({
+        path: z.string(),
+        recursive: z.boolean().optional().default(false),
+      }),
+      execute: async ({ path: p, recursive }) => executor.listFiles(p, recursive),
+    }),
+    search_files: tool({
+      description: 'Glob-search file names; optional content filter.',
+      inputSchema: z.object({
+        root: z.string(),
+        pattern: z.string(),
+        content_contains: z.string().optional(),
+      }),
+      execute: async ({ root, pattern, content_contains }) =>
+        executor.searchFiles(root, pattern, content_contains),
+    }),
+    analyze_codebase: tool({
+      description: 'Summarize codebase structure.',
+      inputSchema: z.object({ path: z.string().default('.') }),
+      execute: async ({ path: p }) => executor.analyzeCodebase(p),
+    }),
+    list_skills: tool({
+      description: 'List paths to bundled SKILL.md files.',
+      inputSchema: z.object({}),
+      execute: async () => executor.listSkills(),
+    }),
+    read_skill: tool({
+      description: 'Read a SKILL.md by absolute path.',
+      inputSchema: z.object({ path: z.string() }),
+      execute: async ({ path: p }) => executor.readSkill(p),
+    }),
+  };
+}
+
+const PLAN_INSTRUCTIONS = (codebase: string | undefined, hasWeb: boolean) => {
+  const parts: string[] = [
+    'You are a Plan-Mode planner. You DO NOT modify files.',
+  ];
+  if (codebase)
+    parts.push(
+      `Workspace: ${codebase}`,
+      'Use read-only tools for codebase/skills research.',
+      'If the repository already contains an existing plan or roadmap, continue and extend it rather than replacing it. Otherwise generate a fresh, standalone roadmap as requested by the user.',
+    );
+  parts.push(
+    hasWeb
+      ? 'Web tools are available (web_search/web_crawl/fetch_url). Use only when needed.'
+      : 'Web tools are unavailable (no FIRECRAWL_API_KEY).',
+    'Output must match the provided JSON schema.',
+    'Keep it short: 1–10 steps.',
+  );
+  return parts.join('\n');
+};
+
+export async function generatePlan(goal: string, options?: { useWorkspace?: boolean }): Promise<Plan> {
+  const config = defaultAgentConfig();
+  const tracker = new ActionTracker();
+  const executor = new ToolExecutor(tracker, config);
+  const hasWeb = !!process.env.FIRECRAWL_API_KEY?.trim();
+  const model = wrapLanguageModel({
+    model: getAgentModel() as unknown as LanguageModel,
+    middleware: extractJsonMiddleware(),
+  });
+
+  const useWorkspace = options?.useWorkspace ?? true;
+  const tools = useWorkspace
+    ? { ...readOnlyTools(executor), ...(hasWeb ? createWebTools(tracker) : {}) }
+    : { ...(hasWeb ? createWebTools(tracker) : {}) };
+
+  // Try robust generation with retries; abort on failure so callers can show the error.
+  const systemPrompt = PLAN_INSTRUCTIONS(useWorkspace ? config.codebasePath : undefined, hasWeb);
+  const prompt = `User goal:\n${goal}`;
+
+  async function tryGenerateStructured(maxRetries = 3) {
+    return withLLMRetry(
+      () =>
+        generateText({
+          model,
+          tools,
+          stopWhen: stepCountIs(20),
+          system: systemPrompt,
+          prompt,
+          output: Output.object({ schema: planSchema }),
+        }),
+      { maxRetries, context: "Plan" }
+    );
+  }
+
+  let result: any;
+  try {
+    result = await tryGenerateStructured(3);
+  } catch (err) {
+    throw err;
+  }
+  // Normalize model output: support both `{ steps: [...] }` and `{ plan: [...] }`,
+  // and tolerate items that use `step`/`description` shape instead of `title`.
+  const rawOut = result.output ?? {};
+  let normOut: any = { ...rawOut };
+
+  if (!normOut.steps && Array.isArray(normOut.plan)) {
+    normOut.steps = normOut.plan;
+  }
+
+  if (Array.isArray(normOut.steps)) {
+    const normalizedItems = normOut.steps.map((it: any) => {
+      const title = it.title ?? (it.step !== undefined ? `Step ${it.step}` : undefined);
+      const description = it.description ?? it.desc ?? it.text ?? '';
+      const hints = Array.isArray(it.hints) ? it.hints : it.hint ? [it.hint] : undefined;
+      const complexity = it.complexity;
+      return { title, description, hints, complexity };
+    });
+    normOut.steps = normalizedItems;
+  }
+
+  // Try strict validation, but fall back to a best-effort mapping if validation fails.
+  try {
+    const validated = planSchema.parse(normOut);
+    const steps: PlanStep[] = validated.steps.map((s, i) => ({
+      id: `step-${i + 1}`,
+      title: s.title,
+      description: s.description,
+      hints: s.hints,
+      complexity: s.complexity,
+    }));
+    return { goal, researchSummary: validated.researchSummary, steps };
+  } catch (err) {
+    // Best-effort fallback: use whatever we can extract from normOut.steps
+    if (Array.isArray(normOut.steps)) {
+      const steps: PlanStep[] = normOut.steps.map((s: any, i: number) => ({
+        id: `step-${i + 1}`,
+        title: s.title ?? `Step ${i + 1}`,
+        description: s.description ?? '',
+        hints: Array.isArray(s.hints) ? s.hints : undefined,
+        complexity: s.complexity,
+      }));
+      return { goal, researchSummary: normOut.researchSummary ?? normOut.summary, steps };
+    }
+    throw err;
+  }
+}

package/plan/selection.ts

@@ -0,0 +1,50 @@
+import { multiselect, isCancel } from '@clack/prompts';
+import chalk from 'chalk';
+import type { Plan, PlanStep } from './index.ts';
+import { renderHTMLMarkdown } from '../tui/terminal-render.ts';
+
+const COMPLEXITY_COLOR: Record<NonNullable<PlanStep['complexity']>, string> = {
+  low: chalk.green('low'),
+  medium: chalk.yellow('medium'),
+  high: chalk.red('high'),
+};
+
+export function printPlan(plan: Plan): void {
+  if (plan.researchSummary?.trim()) {
+    console.log(chalk.bold('\n🔍 Research summary'));
+    console.log(renderHTMLMarkdown(plan.researchSummary));
+  }
+  console.log(chalk.bold('\n📋 Generated Plan\n'));
+  for (const [i, s] of plan.steps.entries()) {
+    const tag = s.complexity ? `[${COMPLEXITY_COLOR[s.complexity]}]` : '';
+    console.log(`  ${chalk.cyan(`Step ${String(i + 1).padStart(2)}`)}. ${chalk.bold(s.title)} ${tag}`);
+  }
+  console.log();
+}
+
+export async function selectSteps(plan: Plan): Promise<PlanStep[]> {
+  const options = plan.steps.map((s: any) => ({
+    value: s.id,
+    label: s.title,
+    hint: s.complexity ?? '',
+    disabled: false,
+  }));
+
+  let picked: string[] | null = null;
+  try {
+    picked = await multiselect<string>({
+      message: 'Select steps to execute (space toggles, enter confirms)',
+      options,
+      initialValues: plan.steps.map((s: any) => s.id),
+      required: false,
+    }) as string[];
+  } catch (err) {
+    // If the multiselect UI throws (some terminals or environments), fall back to selecting all steps
+    console.warn('multiselect failed, defaulting to all steps');
+    picked = plan.steps.map((s: any) => s.id);
+  }
+
+  if (isCancel(picked)) return [];
+  const set = new Set<string>(picked);
+  return plan.steps.filter((s:any) => set.has(s.id));
+}

package/plan/types.ts

@@ -0,0 +1,13 @@
+export interface PlanStep {
+  id: string;
+  title: string;
+  description: string;
+  hints?: string[];
+  complexity?: 'low' | 'medium' | 'high';
+}
+
+export interface Plan {
+  goal: string;
+  researchSummary?: string;
+  steps: PlanStep[];
+}

package/plan/web-tools.ts

@@ -0,0 +1,119 @@
+import { tool } from "ai";
+import { z } from "zod";
+import Firecrawl from "@mendable/firecrawl-js";
+import type { ActionTracker } from "../agent/action-tracker.ts";
+
+function getKey(): string | undefined {
+  const v = process.env.FIRECRAWL_KEY;
+  return v?.replace(/^['"]|['"]$/g, "").trim() || undefined;
+}
+
+let client: Firecrawl | null = null;
+function getClient(): Firecrawl {
+  if (client) return client;
+  const apiKey = getKey();
+  if (!apiKey) throw new Error("FIRECRAWL_API_KEY not set");
+  client = new Firecrawl({ apiKey });
+  return client;
+}
+
+function clip(s: string, n = 8000): string {
+  return s.length > n ? s.slice(0, n) + "\n…[truncated]" : s;
+}
+
+export function createWebTools(tracker: ActionTracker) {
+  return {
+    web_search: tool({
+      description: "Search the web. Returns title/url/snippet list.",
+      inputSchema: z.object({
+        query: z.string().min(1),
+        limit: z.number().int().min(1).max(10).optional().default(5),
+      }),
+      execute: async ({ query, limit }) => {
+        const res = await getClient().search(query, {
+          limit,
+          sources: ["web"],
+        });
+        const items = (res.web ?? []).slice(0, limit);
+        const out =
+          items
+            .map((d, i) => {
+              const title = ("title" in d && d.title) || "(untitled)";
+              const url = ("url" in d && d.url) || "";
+              const snip = ("snippet" in d && d.snippet) || "";
+              return `${i + 1}. ${title}\n   ${url}\n   ${snip}`;
+            })
+            .join("\n\n") || "(no results)";
+        tracker.log({
+          type: "code_analysis",
+          path: `web_search:${query}`,
+          details: { after: out, toolName: "web_search" },
+          status: "executed",
+        });
+        return clip(out);
+      },
+    }),
+
+    web_crawl: tool({
+      description: "Scrape a URL into markdown text.",
+      inputSchema: z.object({ url: z.string().url() }),
+      execute: async ({ url }) => {
+        const doc = await getClient().scrape(url, { formats: ["markdown"] });
+        const md = (doc as { markdown?: string }).markdown ?? "";
+        tracker.log({
+          type: "code_analysis",
+          path: `web_crawl:${url}`,
+          details: { after: clip(md), toolName: "web_crawl" },
+          status: "executed",
+        });
+        return clip(md) || "(empty)";
+      },
+    }),
+
+    fetch_url: tool({
+      description: "HTTP GET for a URL. Returns response body.",
+      inputSchema: z.object({ url: z.string().url() }),
+      execute: async ({ url }) => {
+        const r = await fetch(url, { redirect: "follow" });
+        const body = await r.text();
+        const out = clip(body, 16_000);
+        tracker.log({
+          type: "code_analysis",
+          path: `fetch:${url}`,
+          details: {
+            after: `HTTP ${r.status}\n\n${out}`,
+            toolName: "fetch_url",
+          },
+          status: "executed",
+        });
+        return `HTTP ${r.status}\n\n${out}`;
+      },
+    }),
+
+    wikipedia_search: tool({
+      description: "Search Wikipedia for detailed information",
+      inputSchema: z.object({
+        query: z.string().describe("The topic to search about"),
+      }),
+      execute: async ({ query }) => {
+        const url = `https://en.wikipedia.org/w/api.php?action=opensearch&search=${encodeURIComponent(query)}&limit=1&format=json`;
+        const response = await fetch(url);
+
+        if (!response.ok) {
+          return {
+            success: false,
+            message: "Article not found",
+          };
+        }
+
+        const data = await response.json();
+
+        return {
+          title: data.title,
+          summary: data.extract,
+          url: data.content_urls?.desktop?.page,
+        };
+      },
+    }),
+  };
+}

package/scheduler/ARCHITECTURE.md

@@ -0,0 +1,263 @@
+# Scheduler Architecture
+
+## System Overview
+
+```
+┌─────────────────────────────────────────────────────────────────┐
+│                         YOUR MACHINE                             │
+│  ┌──────────────┐          ┌────────────────────────────────┐  │
+│  │   Jimmy CLI  │          │   Local Storage                │  │
+│  │              │          │   ~/.cccontrol/googleAuth/     │  │
+│  │  • Add tasks │◄────────►│   google_config.json           │  │
+│  │  • Manage    │          │   (refresh_token)              │  │
+│  │  • Debug     │          └────────────────────────────────┘  │
+│  └──────┬───────┘                                               │
+│         │ Writes tasks                                          │
+│         │ Syncs credentials                                     │
+└─────────┼───────────────────────────────────────────────────────┘
+          │
+          │ HTTPS (Supabase Client)
+          ▼
+┌─────────────────────────────────────────────────────────────────┐
+│                      SUPABASE CLOUD                              │
+│  ┌───────────────────────────────────────────────────────────┐  │
+│  │                    PostgreSQL Database                     │  │
+│  │  ┌────────────────┐  ┌────────────────┐  ┌─────────────┐ │  │
+│  │  │scheduler_tasks │  │scheduler_runs  │  │user_config  │ │  │
+│  │  │                │  │                │  │             │ │  │
+│  │  │ • name         │  │ • task_id      │  │ • API keys  │ │  │
+│  │  │ • cron         │  │ • status       │  │ • tokens    │ │  │
+│  │  │ • steps        │  │ • output       │  │ • secrets   │ │  │
+│  │  │ • enabled      │  │ • error        │  │             │ │  │
+│  │  │ • next_run_at  │  │ • step_results │  │             │ │  │
+│  │  └────────────────┘  └────────────────┘  └─────────────┘ │  │
+│  └───────────┬───────────────┬───────────────────┬───────────┘  │
+│              │               │                   │              │
+│  ┌───────────┴───────────────┴───────────────────┴───────────┐  │
+│  │                      pg_cron Extension                     │  │
+│  │                                                             │  │
+│  │  Job: jimmy-scheduler-tick                                 │  │
+│  │  Schedule: * * * * * (every minute)                        │  │
+│  │  Action: HTTP POST to Edge Function                        │  │
+│  └───────────┬─────────────────────────────────────────────────┤  │
+│              │ Every minute                                    │  │
+│              ▼                                                 │  │
+│  ┌─────────────────────────────────────────────────────────┐  │  │
+│  │              Edge Function: scheduler-tick               │  │  │
+│  │              (Deno runtime in Supabase)                  │  │  │
+│  │                                                           │  │  │
+│  │  1. Load credentials from user_config                    │  │  │
+│  │  2. Query tasks WHERE enabled=true AND next_run_at <= now│  │  │
+│  │  3. For each due task:                                   │  │  │
+│  │     a. Create run record (status: running)               │  │  │
+│  │     b. Execute each step in order:                       │  │  │
+│  │        • web_search  → Firecrawl API                     │  │  │
+│  │        • web_crawl   → Firecrawl API                     │  │  │
+│  │        • custom      → OpenRouter/Groq LLM               │  │  │
+│  │        • email_send  → Gmail API                         │  │  │
+│  │        • browser     → Skip (local only)                 │  │  │
+│  │     c. Update run record (status: success/failed)        │  │  │
+│  │     d. Update task (last_run_at, next_run_at, count)     │  │  │
+│  │     e. Send summary email if configured                  │  │  │
+│  │  4. Return results                                       │  │  │
+│  └─────────────────────────────────────────────────────────┘  │  │
+│              │                                                 │  │
+│              │ Makes API calls                                 │  │
+│              ▼                                                 │  │
+└──────────────┼──────────────────────────────────────────────────┘
+               │
+               │ HTTPS
+               ▼
+┌─────────────────────────────────────────────────────────────────┐
+│                      EXTERNAL APIS                               │
+│                                                                  │
+│  ┌────────────┐  ┌────────────┐  ┌────────────┐  ┌──────────┐ │
+│  │ Firecrawl  │  │ OpenRouter │  │   Groq     │  │  Gmail   │ │
+│  │   API      │  │    API     │  │    API     │  │   API    │ │
+│  │            │  │            │  │            │  │          │ │
+│  │ • Search   │  │ • LLM      │  │ • LLM      │  │ • Send   │ │
+│  │ • Scrape   │  │ • Summary  │  │ • Fallback │  │ • OAuth  │ │
+│  └────────────┘  └────────────┘  └────────────┘  └──────────┘ │
+└─────────────────────────────────────────────────────────────────┘
+               │
+               │ Email sent
+               ▼
+         📧 YOUR INBOX
+```
+
+## Data Flow
+
+### 1. Task Creation
+```
+User (CLI) → scheduler_tasks table
+          → Sets next_run_at based on cron
+```
+
+### 2. Credential Sync
+```
+Local file: ~/.cccontrol/googleAuth/google_config.json
+          ↓ jimmy sync-credentials
+Supabase: user_config table (key-value pairs)
+          ↓ Edge Function reads on every run
+External APIs (authenticated)
+```
+
+### 3. Scheduled Execution
+```
+pg_cron (every minute)
+    ↓
+Edge Function triggered
+    ↓
+1. Load credentials from user_config
+2. Query due tasks
+3. For each task:
+   ├─ Create run record
+   ├─ Execute steps sequentially
+   ├─ Store results
+   ├─ Update task counters
+   ├─ Calculate next_run_at
+   └─ Send email summary
+    ↓
+Results in scheduler_runs table
+```
+
+### 4. Manual Execution
+```
+User (CLI: "Run now")
+    ↓
+createRun() → scheduler_runs
+    ↓
+runTask() → executes steps locally (Node)
+    ↓
+finishRun() → updates scheduler_runs
+    ↓
+updateTask() → updates counters
+```
+
+## Step Execution Details
+
+### Step Type: `web_search`
+```
+Input: Natural language instruction
+    ↓
+LLM extracts search query
+    ↓
+Firecrawl API search
+    ↓
+LLM summarizes results
+    ↓
+Output: Summary text
+```
+
+### Step Type: `web_crawl`
+```
+Input: URL or instruction
+    ↓
+LLM extracts URL (if needed)
+    ↓
+Firecrawl API scrape
+    ↓
+LLM extracts relevant info
+    ↓
+Output: Extracted content
+```
+
+### Step Type: `custom`
+```
+Input: Natural language instruction
+    ↓
+LLM processes with context
+    ↓
+Output: LLM response
+```
+
+### Step Type: `email_send`
+```
+Input: Instruction (who, what, subject)
+    ↓
+LLM generates email params (to, subject, body)
+    ↓
+Gmail API: refresh token → access token
+    ↓
+Send email via Gmail API
+    ↓
+Output: "Email sent to X"
+```
+
+### Step Type: `browser` (local only)
+```
+Input: Instruction
+    ↓
+Stagehand + Playwright
+    ↓
+Browser automation
+    ↓
+Output: Extracted data
+```
+
+## Key Design Decisions
+
+### Why Supabase?
+- ✅ Always-on execution (no local process needed)
+- ✅ Built-in PostgreSQL + pg_cron
+- ✅ Edge Functions (Deno) for serverless execution
+- ✅ Real-time database for monitoring
+- ✅ Free tier supports most use cases
+
+### Why `user_config` Table?
+- ✅ No manual secret updates on re-auth
+- ✅ Dynamic credential loading
+- ✅ Edge Function reads fresh values every run
+- ✅ Multi-user support (if needed)
+
+### Why Two Execution Modes?
+- **Serverless (Edge Function):** Most step types
+- **Local (daemon):** Browser automation only (Playwright can't run in Edge Functions)
+
+### Why LLM Fallback Chain?
+```
+OpenRouter (primary)
+    ↓ if fails or refuses
+Groq (fallback)
+    ↓ if both fail
+Error logged
+```
+
+This ensures reliability even if one provider is down.
+
+## Security
+
+- ✅ Credentials stored in Supabase (service role access only)
+- ✅ Edge Function validates auth header (Bearer token)
+- ✅ Gmail refresh token auto-rotates (OAuth2)
+- ✅ Local config files have 0600 permissions
+- ✅ No secrets in code or git
+
+## Monitoring
+
+**Real-time:**
+- `jimmy scheduler-debug` — CLI tool
+
+**Historical:**
+- `scheduler_runs` table — all executions
+- `cron.job_run_details` — pg_cron logs
+- Edge Function logs — Supabase dashboard
+
+**Manual:**
+- SQL queries in `scheduler/check-status.sql`
+
+## Scalability
+
+Current setup handles:
+- **Tasks:** Unlimited (Postgres capacity)
+- **Executions:** 1/minute per task
+- **Concurrency:** All due tasks execute in parallel (Promise.allSettled)
+- **Free tier limits:**
+  - Supabase: 500MB DB, 2GB bandwidth/month
+  - OpenRouter: Pay-per-use
+  - Firecrawl: 500 credits/month free
+
+To scale beyond free tier:
+- Upgrade Supabase plan
+- Add rate limiting
+- Batch tasks into groups