@nextsparkjs/ai-workflow 0.1.0-beta.166 → 0.1.0-beta.168

package/claude/skills/anthropics-skill-creator/scripts/run_loop.py

@@ -0,0 +1,328 @@
+#!/usr/bin/env python3
+"""Run the eval + improve loop until all pass or max iterations reached.
+
+Combines run_eval.py and improve_description.py in a loop, tracking history
+and returning the best description found. Supports train/test split to prevent
+overfitting.
+"""
+
+import argparse
+import json
+import random
+import sys
+import tempfile
+import time
+import webbrowser
+from pathlib import Path
+
+from scripts.generate_report import generate_html
+from scripts.improve_description import improve_description
+from scripts.run_eval import find_project_root, run_eval
+from scripts.utils import parse_skill_md
+
+
+def split_eval_set(eval_set: list[dict], holdout: float, seed: int = 42) -> tuple[list[dict], list[dict]]:
+    """Split eval set into train and test sets, stratified by should_trigger."""
+    random.seed(seed)
+
+    # Separate by should_trigger
+    trigger = [e for e in eval_set if e["should_trigger"]]
+    no_trigger = [e for e in eval_set if not e["should_trigger"]]
+
+    # Shuffle each group
+    random.shuffle(trigger)
+    random.shuffle(no_trigger)
+
+    # Calculate split points
+    n_trigger_test = max(1, int(len(trigger) * holdout))
+    n_no_trigger_test = max(1, int(len(no_trigger) * holdout))
+
+    # Split
+    test_set = trigger[:n_trigger_test] + no_trigger[:n_no_trigger_test]
+    train_set = trigger[n_trigger_test:] + no_trigger[n_no_trigger_test:]
+
+    return train_set, test_set
+
+
+def run_loop(
+    eval_set: list[dict],
+    skill_path: Path,
+    description_override: str | None,
+    num_workers: int,
+    timeout: int,
+    max_iterations: int,
+    runs_per_query: int,
+    trigger_threshold: float,
+    holdout: float,
+    model: str,
+    verbose: bool,
+    live_report_path: Path | None = None,
+    log_dir: Path | None = None,
+) -> dict:
+    """Run the eval + improvement loop."""
+    project_root = find_project_root()
+    name, original_description, content = parse_skill_md(skill_path)
+    current_description = description_override or original_description
+
+    # Split into train/test if holdout > 0
+    if holdout > 0:
+        train_set, test_set = split_eval_set(eval_set, holdout)
+        if verbose:
+            print(f"Split: {len(train_set)} train, {len(test_set)} test (holdout={holdout})", file=sys.stderr)
+    else:
+        train_set = eval_set
+        test_set = []
+
+    history = []
+    exit_reason = "unknown"
+
+    for iteration in range(1, max_iterations + 1):
+        if verbose:
+            print(f"\n{'='*60}", file=sys.stderr)
+            print(f"Iteration {iteration}/{max_iterations}", file=sys.stderr)
+            print(f"Description: {current_description}", file=sys.stderr)
+            print(f"{'='*60}", file=sys.stderr)
+
+        # Evaluate train + test together in one batch for parallelism
+        all_queries = train_set + test_set
+        t0 = time.time()
+        all_results = run_eval(
+            eval_set=all_queries,
+            skill_name=name,
+            description=current_description,
+            num_workers=num_workers,
+            timeout=timeout,
+            project_root=project_root,
+            runs_per_query=runs_per_query,
+            trigger_threshold=trigger_threshold,
+            model=model,
+        )
+        eval_elapsed = time.time() - t0
+
+        # Split results back into train/test by matching queries
+        train_queries_set = {q["query"] for q in train_set}
+        train_result_list = [r for r in all_results["results"] if r["query"] in train_queries_set]
+        test_result_list = [r for r in all_results["results"] if r["query"] not in train_queries_set]
+
+        train_passed = sum(1 for r in train_result_list if r["pass"])
+        train_total = len(train_result_list)
+        train_summary = {"passed": train_passed, "failed": train_total - train_passed, "total": train_total}
+        train_results = {"results": train_result_list, "summary": train_summary}
+
+        if test_set:
+            test_passed = sum(1 for r in test_result_list if r["pass"])
+            test_total = len(test_result_list)
+            test_summary = {"passed": test_passed, "failed": test_total - test_passed, "total": test_total}
+            test_results = {"results": test_result_list, "summary": test_summary}
+        else:
+            test_results = None
+            test_summary = None
+
+        history.append({
+            "iteration": iteration,
+            "description": current_description,
+            "train_passed": train_summary["passed"],
+            "train_failed": train_summary["failed"],
+            "train_total": train_summary["total"],
+            "train_results": train_results["results"],
+            "test_passed": test_summary["passed"] if test_summary else None,
+            "test_failed": test_summary["failed"] if test_summary else None,
+            "test_total": test_summary["total"] if test_summary else None,
+            "test_results": test_results["results"] if test_results else None,
+            # For backward compat with report generator
+            "passed": train_summary["passed"],
+            "failed": train_summary["failed"],
+            "total": train_summary["total"],
+            "results": train_results["results"],
+        })
+
+        # Write live report if path provided
+        if live_report_path:
+            partial_output = {
+                "original_description": original_description,
+                "best_description": current_description,
+                "best_score": "in progress",
+                "iterations_run": len(history),
+                "holdout": holdout,
+                "train_size": len(train_set),
+                "test_size": len(test_set),
+                "history": history,
+            }
+            live_report_path.write_text(generate_html(partial_output, auto_refresh=True, skill_name=name))
+
+        if verbose:
+            def print_eval_stats(label, results, elapsed):
+                pos = [r for r in results if r["should_trigger"]]
+                neg = [r for r in results if not r["should_trigger"]]
+                tp = sum(r["triggers"] for r in pos)
+                pos_runs = sum(r["runs"] for r in pos)
+                fn = pos_runs - tp
+                fp = sum(r["triggers"] for r in neg)
+                neg_runs = sum(r["runs"] for r in neg)
+                tn = neg_runs - fp
+                total = tp + tn + fp + fn
+                precision = tp / (tp + fp) if (tp + fp) > 0 else 1.0
+                recall = tp / (tp + fn) if (tp + fn) > 0 else 1.0
+                accuracy = (tp + tn) / total if total > 0 else 0.0
+                print(f"{label}: {tp+tn}/{total} correct, precision={precision:.0%} recall={recall:.0%} accuracy={accuracy:.0%} ({elapsed:.1f}s)", file=sys.stderr)
+                for r in results:
+                    status = "PASS" if r["pass"] else "FAIL"
+                    rate_str = f"{r['triggers']}/{r['runs']}"
+                    print(f"  [{status}] rate={rate_str} expected={r['should_trigger']}: {r['query'][:60]}", file=sys.stderr)
+
+            print_eval_stats("Train", train_results["results"], eval_elapsed)
+            if test_summary:
+                print_eval_stats("Test ", test_results["results"], 0)
+
+        if train_summary["failed"] == 0:
+            exit_reason = f"all_passed (iteration {iteration})"
+            if verbose:
+                print(f"\nAll train queries passed on iteration {iteration}!", file=sys.stderr)
+            break
+
+        if iteration == max_iterations:
+            exit_reason = f"max_iterations ({max_iterations})"
+            if verbose:
+                print(f"\nMax iterations reached ({max_iterations}).", file=sys.stderr)
+            break
+
+        # Improve the description based on train results
+        if verbose:
+            print(f"\nImproving description...", file=sys.stderr)
+
+        t0 = time.time()
+        # Strip test scores from history so improvement model can't see them
+        blinded_history = [
+            {k: v for k, v in h.items() if not k.startswith("test_")}
+            for h in history
+        ]
+        new_description = improve_description(
+            skill_name=name,
+            skill_content=content,
+            current_description=current_description,
+            eval_results=train_results,
+            history=blinded_history,
+            model=model,
+            log_dir=log_dir,
+            iteration=iteration,
+        )
+        improve_elapsed = time.time() - t0
+
+        if verbose:
+            print(f"Proposed ({improve_elapsed:.1f}s): {new_description}", file=sys.stderr)
+
+        current_description = new_description
+
+    # Find the best iteration by TEST score (or train if no test set)
+    if test_set:
+        best = max(history, key=lambda h: h["test_passed"] or 0)
+        best_score = f"{best['test_passed']}/{best['test_total']}"
+    else:
+        best = max(history, key=lambda h: h["train_passed"])
+        best_score = f"{best['train_passed']}/{best['train_total']}"
+
+    if verbose:
+        print(f"\nExit reason: {exit_reason}", file=sys.stderr)
+        print(f"Best score: {best_score} (iteration {best['iteration']})", file=sys.stderr)
+
+    return {
+        "exit_reason": exit_reason,
+        "original_description": original_description,
+        "best_description": best["description"],
+        "best_score": best_score,
+        "best_train_score": f"{best['train_passed']}/{best['train_total']}",
+        "best_test_score": f"{best['test_passed']}/{best['test_total']}" if test_set else None,
+        "final_description": current_description,
+        "iterations_run": len(history),
+        "holdout": holdout,
+        "train_size": len(train_set),
+        "test_size": len(test_set),
+        "history": history,
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Run eval + improve loop")
+    parser.add_argument("--eval-set", required=True, help="Path to eval set JSON file")
+    parser.add_argument("--skill-path", required=True, help="Path to skill directory")
+    parser.add_argument("--description", default=None, help="Override starting description")
+    parser.add_argument("--num-workers", type=int, default=10, help="Number of parallel workers")
+    parser.add_argument("--timeout", type=int, default=30, help="Timeout per query in seconds")
+    parser.add_argument("--max-iterations", type=int, default=5, help="Max improvement iterations")
+    parser.add_argument("--runs-per-query", type=int, default=3, help="Number of runs per query")
+    parser.add_argument("--trigger-threshold", type=float, default=0.5, help="Trigger rate threshold")
+    parser.add_argument("--holdout", type=float, default=0.4, help="Fraction of eval set to hold out for testing (0 to disable)")
+    parser.add_argument("--model", required=True, help="Model for improvement")
+    parser.add_argument("--verbose", action="store_true", help="Print progress to stderr")
+    parser.add_argument("--report", default="auto", help="Generate HTML report at this path (default: 'auto' for temp file, 'none' to disable)")
+    parser.add_argument("--results-dir", default=None, help="Save all outputs (results.json, report.html, log.txt) to a timestamped subdirectory here")
+    args = parser.parse_args()
+
+    eval_set = json.loads(Path(args.eval_set).read_text())
+    skill_path = Path(args.skill_path)
+
+    if not (skill_path / "SKILL.md").exists():
+        print(f"Error: No SKILL.md found at {skill_path}", file=sys.stderr)
+        sys.exit(1)
+
+    name, _, _ = parse_skill_md(skill_path)
+
+    # Set up live report path
+    if args.report != "none":
+        if args.report == "auto":
+            timestamp = time.strftime("%Y%m%d_%H%M%S")
+            live_report_path = Path(tempfile.gettempdir()) / f"skill_description_report_{skill_path.name}_{timestamp}.html"
+        else:
+            live_report_path = Path(args.report)
+        # Open the report immediately so the user can watch
+        live_report_path.write_text("<html><body><h1>Starting optimization loop...</h1><meta http-equiv='refresh' content='5'></body></html>")
+        webbrowser.open(str(live_report_path))
+    else:
+        live_report_path = None
+
+    # Determine output directory (create before run_loop so logs can be written)
+    if args.results_dir:
+        timestamp = time.strftime("%Y-%m-%d_%H%M%S")
+        results_dir = Path(args.results_dir) / timestamp
+        results_dir.mkdir(parents=True, exist_ok=True)
+    else:
+        results_dir = None
+
+    log_dir = results_dir / "logs" if results_dir else None
+
+    output = run_loop(
+        eval_set=eval_set,
+        skill_path=skill_path,
+        description_override=args.description,
+        num_workers=args.num_workers,
+        timeout=args.timeout,
+        max_iterations=args.max_iterations,
+        runs_per_query=args.runs_per_query,
+        trigger_threshold=args.trigger_threshold,
+        holdout=args.holdout,
+        model=args.model,
+        verbose=args.verbose,
+        live_report_path=live_report_path,
+        log_dir=log_dir,
+    )
+
+    # Save JSON output
+    json_output = json.dumps(output, indent=2)
+    print(json_output)
+    if results_dir:
+        (results_dir / "results.json").write_text(json_output)
+
+    # Write final HTML report (without auto-refresh)
+    if live_report_path:
+        live_report_path.write_text(generate_html(output, auto_refresh=False, skill_name=name))
+        print(f"\nReport: {live_report_path}", file=sys.stderr)
+
+    if results_dir and live_report_path:
+        (results_dir / "report.html").write_text(generate_html(output, auto_refresh=False, skill_name=name))
+
+    if results_dir:
+        print(f"Results saved to: {results_dir}", file=sys.stderr)
+
+
+if __name__ == "__main__":
+    main()

package/claude/skills/anthropics-skill-creator/scripts/utils.py

@@ -0,0 +1,47 @@
+"""Shared utilities for skill-creator scripts."""
+
+from pathlib import Path
+
+
+
+def parse_skill_md(skill_path: Path) -> tuple[str, str, str]:
+    """Parse a SKILL.md file, returning (name, description, full_content)."""
+    content = (skill_path / "SKILL.md").read_text()
+    lines = content.split("\n")
+
+    if lines[0].strip() != "---":
+        raise ValueError("SKILL.md missing frontmatter (no opening ---)")
+
+    end_idx = None
+    for i, line in enumerate(lines[1:], start=1):
+        if line.strip() == "---":
+            end_idx = i
+            break
+
+    if end_idx is None:
+        raise ValueError("SKILL.md missing frontmatter (no closing ---)")
+
+    name = ""
+    description = ""
+    frontmatter_lines = lines[1:end_idx]
+    i = 0
+    while i < len(frontmatter_lines):
+        line = frontmatter_lines[i]
+        if line.startswith("name:"):
+            name = line[len("name:"):].strip().strip('"').strip("'")
+        elif line.startswith("description:"):
+            value = line[len("description:"):].strip()
+            # Handle YAML multiline indicators (>, |, >-, |-)
+            if value in (">", "|", ">-", "|-"):
+                continuation_lines: list[str] = []
+                i += 1
+                while i < len(frontmatter_lines) and (frontmatter_lines[i].startswith("  ") or frontmatter_lines[i].startswith("\t")):
+                    continuation_lines.append(frontmatter_lines[i].strip())
+                    i += 1
+                description = " ".join(continuation_lines)
+                continue
+            else:
+                description = value.strip('"').strip("'")
+        i += 1
+
+    return name, description, content

package/claude/skills/api-bypass-layers/SKILL.md

@@ -1,11 +1,12 @@
 ---
 name: api-bypass-layers
 description: |
-  Multi-layer security architecture for API bypass (superadmin/developer).
-  Covers authentication layers, authorization, team context, RLS policies, and three-layer bypass validation.
+  Multi-layer security architecture for API bypass (superadmin/developer): can_bypass_rls(),
+  three-layer bypass validation, team context, and the RLS final-defense layer.
   Use this skill when implementing admin bypass features or validating security architecture.
+  For the runtime RLS plumbing (service connection pool, nextspark_app cutover) see rls-enforcement.
 allowed-tools: Read, Glob, Grep
-version: 1.1.0
+version: 1.2.0
 ---
 
 # API Bypass Layers Skill
@@ -18,8 +19,9 @@ Security architecture for superadmin and developer bypass access in the API laye
 |------|---------|
 | `packages/core/src/lib/api/auth/dual-auth.ts` | App-level bypass (canBypassTeamContext) |
 | `packages/core/src/lib/api/entity/generic-handler.ts` | CRUD handler (validateTeamContextWithBypass) |
-| `packages/core/migrations/001_better_auth_and_functions.sql` | get_auth_user_id() function |
-| `packages/core/migrations/010_teams_functions_triggers.sql` | can_bypass_rls(), get_user_team_ids(), RLS policies |
+| `packages/core/migrations/001_better_auth_and_functions.sql` | get_auth_user_id(), **can_bypass_rls(), is_superadmin(), auth_user_can_see_user()** (moved here in beta.167) |
+| `packages/core/migrations/010_teams_functions_triggers.sql` | get_user_team_ids(), teams RLS policies |
+| `packages/core/src/lib/db.ts` | service connection pool (RLS-bypass) for system ops — see rls-enforcement |
 | `packages/core/migrations/090_sample_data.sql` | System Admin Team (team-nextspark-001) |
 
 ## Architecture Overview
@@ -80,6 +82,16 @@ Security architecture for superadmin and developer bypass access in the API laye
 └─────────────────────────────────────────────────────────────────┘
 ```
 
+> **beta.167 additions (see the `rls-enforcement` skill):**
+> - **Service connection (RLS-bypass):** system operations without a user GUC (Better Auth,
+>   scheduler/processor, webhooks, the `checkSystemAdminMembership` bypass check, team/
+>   subscription bootstrap) run on a separate pool (`DATABASE_SERVICE_URL`). This is the
+>   plumbing that keeps the bypass paths working once the app connects as the non-owner
+>   `nextspark_app` role.
+> - **Fail-closed permission check:** in `generic-handler.ts`, LIST/READ now enforce session
+>   permissions and a thrown permission check returns `500 PERMISSION_CHECK_FAILED` (never
+>   "allow"). Admin-bypass and api-key paths are skipped.
+
 ## When to Use This Skill
 
 - Understanding the multi-layer security model
@@ -210,7 +222,7 @@ if (userId && !entityConfig.access?.shared && !skipUserFilter && !isBypass) {
 
 ## Database-Level Bypass: can_bypass_rls()
 
-**File:** `packages/core/migrations/010_teams_functions_triggers.sql:115-147`
+**File:** `packages/core/migrations/001_better_auth_and_functions.sql` (moved here in beta.167 so the hardened auth-table policies in `002` can use it; it was previously in `010`)
 
 **CRITICAL DIFFERENCE from App-Level:**
 - **Superadmin:** ALWAYS bypasses RLS (no team membership check)
@@ -418,7 +430,7 @@ const headers = {
 │  LAYER 4: Query Building (generic-handler.ts:509+)                       │
 │  └─ Add teamId/userId filters based on isBypass flag                     │
 │                                                                           │
-│  LAYER 5: Database RLS (010_teams_functions_triggers.sql:115-147)        │
+│  LAYER 5: Database RLS (can_bypass_rls() defined in 001, beta.167)       │
 │  └─ can_bypass_rls() OR teamId = ANY(get_user_team_ids())                │
 │                                                                           │
 └──────────────────────────────────────────────────────────────────────────┘
@@ -546,5 +558,6 @@ Before finalizing bypass implementation:
 - `better-auth` - Authentication patterns and session management
 - `permissions-system` - RBAC + Features + Quotas permission model
 - `database-migrations` - PostgreSQL RLS policies
+- `rls-enforcement` - Runtime RLS: service connection pool, nextspark_app cutover, fail-closed
 - `nextjs-api-development` - API route patterns with dual auth
 - `cypress-api` - API testing with bypass scenarios

package/claude/skills/database-migrations/SKILL.md

@@ -1,11 +1,12 @@
 ---
 name: database-migrations
 description: |
-  PostgreSQL migration patterns with RLS, Better Auth integration, and TIMESTAMPTZ.
+  PostgreSQL migration patterns: writing RLS policy SQL, Better Auth integration, and TIMESTAMPTZ.
   Covers main tables, meta tables, child entities, and sample data.
   Use this skill when creating migrations, validating SQL, or generating sample data.
+  For the RUNTIME RLS model (service pool, nextspark_app cutover, fail-closed) see rls-enforcement.
 allowed-tools: Read, Glob, Grep, Bash(python:*)
-version: 1.0.0
+version: 1.1.0
 ---
 
 # Database Migrations Skill
@@ -110,6 +111,15 @@ REFERENCES public."Parent"(id) ON DELETE CASCADE
 
 ## RLS Patterns (4 Cases)
 
+> **Before writing core-table policies (beta.167):**
+> - RLS primitives (`can_bypass_rls`, `auth_user_can_see_user`, `is_superadmin`) live in
+>   migration **`001`** — a policy on an early table (e.g. `002`) that references a function
+>   or table defined later fails at `CREATE POLICY` time. Wrap forward-dependent logic in a
+>   `SECURITY DEFINER` helper in `001` (table refs resolve at call time).
+> - **`team_role` is `TEXT`** (not an ENUM): `role TEXT NOT NULL DEFAULT 'member'`. Never
+>   `ALTER TYPE team_role`.
+> - For the runtime model (service pool, `nextspark_app`, cutover) see the **rls-enforcement** skill.
+
 ### Case 1: Private to Owner
 ```sql
 CREATE POLICY "Entity owner can do all"
@@ -144,19 +154,24 @@ WITH CHECK (true);
 
 ### Case 4: Public Read with Auth Write
 ```sql
--- Anonymous can read published
-CREATE POLICY "Entity public can select"
-ON public."Entity"
-FOR SELECT TO anon
-USING (published = TRUE);
-
 -- Authenticated can manage all
 CREATE POLICY "Entity auth can do all"
 ON public."Entity"
 FOR ALL TO authenticated
 USING (true)
 WITH CHECK (true);
+
+-- (Supabase Data API ONLY) anonymous can read published rows
+CREATE POLICY "Entity public can select"
+ON public."Entity"
+FOR SELECT TO anon
+USING (published = TRUE);
 ```
+> ⚠️ The **app does NOT use the `anon` role**. Public/unauthenticated reads enter with
+> `userId = null` → **service pool (bypass)** and are filtered in SQL by the handler
+> (e.g. `published = TRUE`). The `TO anon` policy above only matters if you expose the
+> **Supabase PostgREST / Data API** (migration `022` revokes `anon` as defense for that
+> surface). Don't rely on it for the app's public-read path.
 
 ## Meta Tables Pattern
 

package/claude/skills/entity-system/SKILL.md

@@ -5,7 +5,7 @@ description: |
   Includes: config, fields, types, service, messages (i18n), migrations.
   Use this skill to create, modify, or understand system entities.
 allowed-tools: Read, Glob, Grep, Bash(python:*)
-version: 1.0.0
+version: 1.1.0
 ---
 
 # Entity System Skill
@@ -313,7 +313,7 @@ interface ActionConfig {
 2. **Team**: teamId from httpOnly cookie `activeTeamId`
 3. **Permissions**: Verified against `permissions.config.ts`
 4. **Validation**: Fields validated against entity schema
-5. **RLS**: Queries executed with Row-Level Security
+5. **RLS**: Queries executed with Row-Level Security (user-scoped reads pass `userId` → app pool; see the `rls-enforcement` skill for the runtime service-pool model and LIST/READ fail-closed enforcement)
 6. **Team Isolation**: All operations filter by active teamId (prevents cross-team access for multi-team users)
 
 ### Team Isolation (Multi-Team Users)