lazycodex-ai 5.0.0-beta.9 → 5.0.0

package/packages/shared-skills/skills/browser/scripts/browser-doctor.mjs

@@ -0,0 +1,57 @@
+#!/usr/bin/env node
+import { loadOmowright } from "./omowright.mjs"
+
+const { entry, omowright } = await loadOmowright()
+const browserArg = process.argv.find((arg) => arg.startsWith("--browser="))
+const report = await omowright.bskDoctor({ waitForBrowserMs: 0, browser: browserArg ? browserArg.slice("--browser=".length) : undefined })
+
+const state = report.ready
+  ? "ready"
+  : !report.cli.installed
+    ? "no-cli"
+    : !report.daemon.running
+      ? "no-daemon"
+      : report.browsers.length === 0 && !report.primary
+        ? "no-browser-support"
+        : report.identification.needsChoice
+          ? "choose-browser"
+          : "no-extension"
+
+const REMEDIES = {
+  ready: "Attached engine is live: const { connectBrowserSkill } = await import(omowrightEntry); const session = await connectBrowserSkill({ name: \"<task>\", focused: false })",
+  "no-cli": "Run node \"<skill-root>/scripts/browser-install.mjs\" — it installs the CLI, starts the daemon and registers the extension; then tell the user the single step it prints.",
+  "no-daemon": "Run node \"<skill-root>/scripts/browser-install.mjs\" (it starts the daemon through `bsk status`), then re-run this doctor.",
+  "no-extension": report.nextStep ?? "Register the extension with browser-install.mjs, then relay its human step to the user and re-run this doctor.",
+  "choose-browser": `Do not install anything yet. First check your memory for the browser this user works in; if it names one of the candidates, re-run with --browser=<id>. Otherwise ask the user which browser they actually use (${report.identification.reason}), record the answer in memory, then run node "<skill-root>/scripts/browser-install.mjs" --browser=<id>.`,
+  "no-browser-support": "No Chromium-family browser profile was found on this machine. Say so and stop; do not substitute another engine.",
+}
+
+const summary = {
+  state,
+  omowright: entry,
+  cli: report.cli,
+  daemon: { running: report.daemon.running, version: report.daemon.version ?? null, protocolVersion: report.daemon.protocolVersion ?? null, error: report.daemon.error ?? null },
+  browser: report.primary ? { id: report.primary.id, label: report.primary.label, confidence: report.identification.confidence, reason: report.identification.reason } : null,
+  candidates: report.identification.candidates.map((c) => ({ id: c.id, label: c.label, signals: c.signals })),
+  defaultBrowser: report.identification.defaultBrowser,
+  registeredElsewhere: report.identification.registeredElsewhere,
+  browsersDetected: report.browsers.map((browser) => ({ id: browser.id, userDataDir: browser.userDataDir, extensionRegistered: browser.extensionRegistered })),
+  browsersConnected: report.browsersConnected.map((browser) => ({ instanceId: browser.instance_id, name: browser.browser_name, version: browser.browser_version })),
+  nextStep: report.nextStep,
+  remedy: REMEDIES[state],
+}
+
+if (process.argv.includes("--json")) {
+  console.log(JSON.stringify(summary, null, 2))
+} else {
+  console.log(`state: ${state}`)
+  console.log(`omowright: ${entry}`)
+  console.log(`cli: ${report.cli.installed ? `${report.cli.bskBin} (${report.cli.version})` : "not installed"}`)
+  console.log(`daemon: ${report.daemon.running ? `running (${report.daemon.version}, protocol ${report.daemon.protocolVersion})` : `not running${report.daemon.error ? ` — ${report.daemon.error}` : ""}`}`)
+  console.log(`browser the user uses: ${summary.browser ? `${summary.browser.label} [${summary.browser.id}] (${summary.browser.reason})` : `not identified (${report.identification.reason})`}`)
+  console.log(`candidates: ${summary.candidates.map((c) => `${c.id}${c.signals.length ? ` (${c.signals.join(", ")})` : ""}`).join(", ") || "none"}`)
+  if (summary.registeredElsewhere.length > 0) console.log(`extension entries left in other browsers: ${summary.registeredElsewhere.join(", ")} (not removed automatically)`)
+  console.log(`browsers detected: ${summary.browsersDetected.map((b) => `${b.id}${b.extensionRegistered ? " (extension registered)" : ""}`).join(", ") || "none"}`)
+  console.log(`browsers connected: ${summary.browsersConnected.map((b) => `${b.name} ${b.version} [${b.instanceId}]`).join(", ") || "none"}`)
+  console.log(`\n${REMEDIES[state]}`)
+}

package/packages/shared-skills/skills/browser/scripts/browser-install.mjs

@@ -0,0 +1,50 @@
+#!/usr/bin/env node
+import { loadOmowright } from "./omowright.mjs"
+
+const json = process.argv.includes("--json")
+const waitArg = process.argv.find((arg) => arg.startsWith("--wait-ms="))
+const waitTotalMs = waitArg ? Number(waitArg.slice("--wait-ms=".length)) : 0
+const browserArg = process.argv.find((arg) => arg.startsWith("--browser="))
+const browser = browserArg ? browserArg.slice("--browser=".length) : undefined
+
+const { omowright } = await loadOmowright()
+const steps = []
+const result = await omowright.bskOnboard({
+  browser,
+  onHumanStep: (step) => steps.push(step),
+  waitForBrowserMs: waitTotalMs > 0 ? Math.min(15_000, waitTotalMs) : 0,
+  waitTotalMs,
+})
+
+const summary = {
+  ready: result.ready,
+  cli: result.cli,
+  installerRan: result.install !== null,
+  daemon: { running: result.daemon.running, version: result.daemon.version ?? null, error: result.daemon.error ?? null },
+  browser: result.primary ? { id: result.primary.id, label: result.primary.label, confidence: result.identification.confidence, reason: result.identification.reason } : null,
+  needsChoice: result.needsChoice,
+  candidates: result.identification.candidates.map((c) => ({ id: c.id, label: c.label, signals: c.signals })),
+  registeredElsewhere: result.identification.registeredElsewhere,
+  registrations: result.registrations.map((r) => ({ browser: r.browser, registered: r.registered, alreadyPresent: r.alreadyPresent ?? false, reason: r.reason ?? null, needsRestart: r.needsRestart ?? null, humanStep: r.humanStep ?? null })),
+  browsersConnected: result.browsersConnected.map((b) => ({ instanceId: b.instance_id, name: b.browser_name, version: b.browser_version })),
+  humanStep: result.humanStep,
+}
+
+if (json) {
+  console.log(JSON.stringify(summary, null, 2))
+} else if (result.ready) {
+  console.log(`attached engine ready: ${summary.browsersConnected.map((b) => `${b.name} ${b.version}`).join(", ")}`)
+} else {
+  console.log(`cli: ${result.cli.installed ? `${result.cli.bskBin} (${result.cli.version})` : "install failed"}`)
+  console.log(`daemon: ${result.daemon.running ? "running" : `not running${result.daemon.error ? ` — ${result.daemon.error}` : ""}`}`)
+  console.log(summary.browser ? `browser: ${summary.browser.label} [${summary.browser.id}] (${summary.browser.reason})` : `browser: not identified (${result.identification.reason})`)
+  for (const r of summary.registrations) {
+    console.log(`extension for ${r.browser}: ${r.registered ? (r.alreadyPresent ? "already registered" : "registered") : `not registered (${r.reason})`}`)
+  }
+  if (result.needsChoice) {
+    console.log(`\nNothing was registered. ${result.humanStep}\nRe-run: node "<skill-root>/scripts/browser-install.mjs" --browser=<id>`)
+  } else {
+    console.log(`\nTell the user exactly this, then re-run browser-doctor.mjs:\n  ${result.humanStep}`)
+  }
+}
+process.exitCode = result.ready ? 0 : 2

package/packages/shared-skills/skills/browser/scripts/omowright.mjs

@@ -0,0 +1,25 @@
+import { existsSync } from "node:fs"
+import { dirname, join } from "node:path"
+import { fileURLToPath, pathToFileURL } from "node:url"
+
+const scriptDir = dirname(fileURLToPath(import.meta.url))
+const skillRoot = dirname(scriptDir)
+
+export function resolveOmowrightEntry(env = process.env) {
+  const candidates = [
+    env.OMOWRIGHT_ROOT ? join(env.OMOWRIGHT_ROOT, "index.js") : undefined,
+    join(skillRoot, "runtime", "omowright", "index.js"),
+    join(skillRoot, "..", "..", "..", "..", "node_modules", "omowright", "src", "index.js"),
+  ].filter((candidate) => candidate !== undefined)
+  return candidates.find((candidate) => existsSync(candidate))
+}
+
+export async function loadOmowright(env = process.env) {
+  const entry = resolveOmowrightEntry(env)
+  if (entry === undefined) {
+    throw new Error(
+      "omowright is not staged in this skill; run `node packages/shared-skills/stage-omowright-runtime.mjs` in a checkout, or set OMOWRIGHT_ROOT to a directory holding its bundled index.js",
+    )
+  }
+  return { entry, omowright: await import(pathToFileURL(entry).href) }
+}

package/packages/shared-skills/skills/coding-agent-sessions/AGENTS.md

@@ -0,0 +1,62 @@
+# coding-agent-sessions — Cross-Platform Session Finder (Python)
+
+**Generated:** 2026-08-24 (f3642fcda)
+
+## OVERVIEW
+
+The only shared skill besides `ultimate-browsing` carrying a real sub-project: a Python package + CLI that finds, lists, searches, and reads local coding-agent session transcripts across ~25 platforms (Codex, Claude, OpenCode, OMO/Senpi/pi, and a long tail), normalizing them into one `Session` model. Earned this file: 33 files, 23 Python modules, own test suite, own `pyrightconfig.json`, own `.npmignore` (ships in the npm package minus tests/caches).
+
+## STRUCTURE
+
+```
+coding-agent-sessions/
+├── SKILL.md                      # router: platform table → references/<platform>.md
+├── references/                   # codex, claude, opencode, senpi, all-platforms (store layouts per platform)
+├── agents/openai.yaml            # Codex agent role declaration
+├── scripts/
+│   ├── find-agent-sessions.py    # thin shebang entry (PEP 723, >=3.11); runpy → agent_sessions.cli
+│   ├── agent_sessions/           # the package (12 modules)
+│   └── tests/                    # 6 pytest modules
+└── pyrightconfig.json            # extraPaths scripts/, excludes scripts/tests
+```
+
+## PUBLIC API (`scripts/agent_sessions/`)
+
+| Module | Key exports |
+|--------|-------------|
+| `scanners.py` | `scan`, `DEFAULT_PLATFORMS`, `PLATFORM_SCANNERS`, per-platform `scan_*` (codex, claude, opencode, senpi, oh_my_pi, gajae_code, …) |
+| `types.py` | `Session`, `Options`, `Json`, `JsonMap` — imported by nearly every module and test |
+| `transcript.py` | parallel file reads, normalization, timestamps, session ids; `recent`, `MAX_PLATFORM_FILES` |
+| `cli.py` | command parsing, filtering, payload construction, JSON emission (283 LOC — behavioral hotspot) |
+| platform adapters | `file_scanners.py` (file-backed providers), `opencode.py` (CLI/SQLite/storage fallbacks), `sqlite_scanners.py`, `sqlite_optional_scanners.py`, `pi_family.py`, `aside_scanner.py`, `kiro_scanner.py` |
+
+## CLI CONTRACT
+
+`python3 scripts/find-agent-sessions.py <command>`; aliases `find`=`search`, `read`=`get`. Commands: `list`, `find`/`search` (repeated `--query` lanes, `--from 7d`, repeated `--platform`, `--workers`, `--cwd/--model`, `--include-subagents`, `--limit`), `get`/`read <session-id>`. Output is JSON with `match_reasons` and reconstructed first/last user prompts — that JSON shape is the stable contract.
+
+## CONVENTIONS
+
+- Stdlib-only Python (`>=3.11`), `from __future__ import annotations`, pathlib everywhere; no third-party deps at runtime.
+- Probe-first, bounded discovery: platform-specific roots checked before broad globbing; optional platforms excluded from default search when they cannot reconstruct user prompts.
+- Main sessions hide subagent children by default but report child counts; `--include-subagents` flips it. Parent/child linkage preserved and records deduplicated.
+- Tests are pytest with `tmp_path` + `MonkeyPatch` isolation; in-file `# pyright: ignore[reportMissingImports]`-style relaxations for import diagnostics only.
+- `.npmignore` strips `scripts/tests/`, `pyrightconfig.json`, and caches from the shipped skill.
+
+## ANTI-PATTERNS
+
+- NEVER answer from normalized previews alone — pull the raw transcript (`read`/`get`) for exact evidence.
+- Usage-only stores are NOT transcript sources.
+- A Claude subagent's embedded `sessionId` is NOT its identity — use its `agentId` + parent-directory linkage.
+- Don't add a platform by globbing blindly; register it in `PLATFORM_SCANNERS`/`DEFAULT_PLATFORMS` with a bounded root probe.
+
+## COMMANDS
+
+```bash
+# from packages/shared-skills/skills/coding-agent-sessions/
+python3 scripts/find-agent-sessions.py list --limit 20
+python3 scripts/find-agent-sessions.py find "commit" --from 7d --platform senpi --platform opencode
+python3 scripts/find-agent-sessions.py read <session-id>
+pytest scripts/tests
+```
+
+- Parent: [`packages/shared-skills/AGENTS.md`](../../AGENTS.md).

package/packages/shared-skills/skills/coding-agent-sessions/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: coding-agent-sessions
-description: "MUST USE when asked to find, read, list, search, inspect, fetch, export, or reconstruct coding-agent sessions across Codex, Claude Code/Desktop, OpenCode, OMO/Senpi/pi, oh-my-pi (omp), gajae-code (gjc), OpenClaw, Factory Droid, Amp, Gemini/Kimi/Qwen CLIs, Codebuff, Roo/Kilo/Cline, Kodu, Cursor CLI, Aider, Aside browser-agent sessions, or unknown local agent logs. Covers transcripts, session IDs, rollout JSONL, state SQLite, Claude projects/pre-compact histories, OpenCode messages/parts, child/subagent linkage, cwd/model/time/token filters, archives, and cost clues. Expands fuzzy recall into parallel query lanes and first probes known stores so absent platforms are skipped cheaply. Triggers: coding agent sessions, Codex/Claude/OpenCode/OMO/Senpi/pi/oh-my-pi/omp/gajae-code/gjc/OpenClaw/Droid/Amp/Kodu/Cursor/Aider/Aside sessions, transcript search, session history, session ID, read transcript, token usage, subagent sessions, what did I do yesterday, did we already do this."
+description: "Finds, reads, and reconstructs coding-agent sessions across Codex, Claude, OpenCode, OMO/Senpi, and other local agent logs. Use when asked to find or search past sessions, transcripts, or subagent runs, or to recover what an earlier session did."
 ---
 
 # Coding Agent Sessions

package/packages/shared-skills/skills/data-scientist/SKILL.md

@@ -1,243 +1,103 @@
 ---
 name: data-scientist
-description: "Expert data processing specialist with intelligent DuckDB/Polars selection for maximum performance. Always includes numpy, never uses pandas, runs everything through uv. Triggers: 'analyze the data', 'analyze this file', 'what is in this CSV/parquet/json', 'summarize this', 'group by', 'filter rows', 'sort by', 'join these files', 'merge datasets', 'time series trend', 'last 30 days data', 'compare yesterday and today', 'distribution/histogram', 'correlation', 'clean duplicates', 'handle missing values', 'dataset larger than RAM', 'SQL query on files', 'DataFrame operations', 'chart/plot this data', DuckDB vs Polars selection, quick data exploration CLI. NOT for plain text/code inspection, configs, or tiny inline math."
+description: "Processes and analyzes data with resident-kernel engines (DuckDB, Polars) and one-shot tools. Use for CSV/parquet/JSON analysis, group-by/join/aggregation, time series, distributions, cleaning, or plotting a dataset."
 ---
 
-# Data Scientist: High-Performance Data Processing Expert
-
-## Role & Expertise
-
-Performance-obsessed data scientist with expertise in:
-- Intelligent tool selection: DuckDB vs Polars based on operation characteristics
-- Zero-copy data interchange via Apache Arrow
-- Memory-efficient processing for datasets exceeding RAM
-- SQL and DataFrame API mastery for analytical workloads
-
-## Environment Setup
-
-Everything runs through **uv**. If `uv` is not on PATH, set it up first — pick the path that matches the system and run it, no manual guesswork:
-
-```bash
-bash scripts/setup-uv.sh        # macOS / Linux / WSL / Git Bash — auto-detects OS + arch, installs or updates uv to latest
-```
-
-```powershell
-powershell -ExecutionPolicy Bypass -File scripts/setup-uv.ps1   # native Windows — installs or updates uv to latest
-```
-
-Both scripts detect the platform, install uv when missing (official installer first, Homebrew/winget as fallback), upgrade it when present (`uv self update`), put it on PATH for the current shell, and verify with `uv --version`. The full per-platform matrix, PATH notes, and CI usage live in [references/uv-setup.md](references/uv-setup.md). Verify: `uv --version`.
-
-## Core Principles
-
-### ABSOLUTE RULES
-
-1. **ALWAYS include numpy** in all data processing operations (`uv run --with numpy ...`)
-2. **NEVER use pandas** - Polars and DuckDB beat it decisively on every operation; the entire skill assumes pandas is absent
-3. **ALWAYS use Python via `uv run`** for calculations and data processing
-4. **Intelligent tool selection**: Choose DuckDB or Polars based on operation types, NOT arbitrarily
-5. **Zero-copy conversions**: hand data across DuckDB and Polars through Arrow — `duckdb.sql(...).pl()`. Never call `.df()` (returns a pandas frame; crashes without pandas). Keep `pyarrow` in the package set or `.pl()` raises `ModuleNotFoundError`
-6. **Lazy evaluation**: Prefer `scan_csv`/`scan_parquet` and `.collect()` only when needed
-7. **Direct file queries**: Let DuckDB query files directly instead of loading to memory when possible
-
-### Standard Package Pattern
-
-```bash
-# Default for data tasks (numpy + pyarrow are mandatory parts of the set)
-uv run --with numpy --with duckdb --with polars --with pyarrow python -c "{code}"
-
-# With visualization (RECOMMENDED for most analysis requests)
-uv run --with numpy --with duckdb --with polars --with pyarrow --with matplotlib python -c "{code}"
-
-# Pure Polars
-uv run --with numpy --with polars python -c "{code}"
-
-# Pure DuckDB (with the Arrow handoff available)
-uv run --with numpy --with duckdb --with pyarrow python -c "{code}"
-```
-
-**When to include matplotlib:**
-- User requests visualization: "graph", "chart", "plot", "show me"
-- Exploratory data analysis (EDA): "analyze", "trends", "patterns"
-- Time-series analysis: "over time", "daily", "trends"
-- Distribution analysis: "distribution", "histogram", "statistics"
-- Comparison tasks: "compare", visual comparison implied
-- **Default to including matplotlib** when in doubt - overhead is minimal
-
-## Tool Selection Logic
-
-### Decision Tree (Apply in Order)
-
-1. **Is it a `.duckdb` file?** → **USE DUCKDB** (native format, optimal performance)
-2. **Simple one-off query without needing full data in memory?** → **USE DUCKDB** (direct file query, zero memory load)
-3. **Very heavy complex SQL query (multi-table joins, window functions)?** → **USE DUCKDB** (superior SQL optimizer)
-4. **Main operation is FILTERING?** → **USE POLARS** (typically the fastest by a wide margin — see benchmarks)
-5. **Main operation is SORTING?** → **USE POLARS** (typically the fastest)
-6. **Complex SQL JOINS needed?** → **USE DUCKDB** (stronger join engine, more join types)
-7. **Heavy GROUP BY AGGREGATIONS?** → **USE DUCKDB** (typically faster on large datasets)
-8. **Window functions with partitioning?** → **POLARS** (typically faster)
-9. **Complex TRANSFORMATIONS (pivot, melt, string ops)?** → **USE POLARS**
-10. **Dataset larger than available RAM?** → **USE POLARS** (streaming support) or **DUCKDB** (out-of-core)
-11. **Mixed operations?** → **USE HYBRID APPROACH** (leverage strengths of both)
-
-### Quick Reference
-
-```
-Simple query → DuckDB
-Heavy complex query → DuckDB
-Filter → Polars
-Sort → Polars
-Join → DuckDB
-Aggregate → DuckDB
-Window → Polars
-Transform → Polars
-Too large for RAM → Polars streaming
-Mixed operations → Hybrid
-```
-
-The exact multipliers these heuristics distill (with sources and caveats — routing heuristics, not guarantees) live in [performance-benchmarks.md](references/performance-benchmarks.md).
-
-## Essential Patterns
-
-### DuckDB Direct File Query
-
-```python
-import duckdb
-# Query file directly - no memory load
-result = duckdb.sql("""
-    SELECT category, SUM(amount) as total
-    FROM 'data.csv'
-    GROUP BY category
-""").pl()  # .pl() -> Polars via Arrow. Requires pyarrow. Never .df() (pandas).
-```
-
-### Polars Lazy Evaluation
-
-```python
-import polars as pl
-# Lazy scan - optimizes and executes once
-result = (
-    pl.scan_csv('data.csv')
-    .filter(pl.col('value') > 100)
-    .sort('value', descending=True)
-    .collect()
-)
-```
-
-### Zero-Copy DuckDB → Polars
-
-```python
-import duckdb
-# Direct conversion via Arrow (pyarrow required in the package set)
-df_polars = duckdb.sql("SELECT * FROM 'data.csv'").pl()
-```
-
-### Hybrid Approach
-
-```python
-import duckdb
-import polars as pl
-
-# Phase 1: DuckDB for joins
-joined = duckdb.sql(
-    "SELECT * FROM 'orders.csv' o "
-    "JOIN 'customers.csv' c ON o.customer_id = c.customer_id"
-).pl()
-
-# Phase 2: Polars for filtering
-filtered = joined.filter(pl.col('amount') > 100)
-
-# Phase 3: Back to DuckDB for aggregation
-duckdb.register('filtered_data', filtered)
-final = duckdb.sql('SELECT category, SUM(amount) FROM filtered_data GROUP BY category').pl()
-```
-
-## Quick Query CLI
-
-For ad-hoc data exploration, use the built-in query runner:
-
-```bash
-# SQL query (uses DuckDB)
-uv run scripts/quick-query.py data.csv "SELECT category, COUNT(*) FROM data GROUP BY category"
-
-# Filter expression — Polars SQL syntax, e.g. "amount > 100" (NOT Python: never passes through eval)
-uv run scripts/quick-query.py data.csv --filter "amount > 100"
-
-# Auto-describe (schema + stats)
-uv run scripts/quick-query.py data.parquet --describe
-```
-
-Supports CSV, Parquet, JSON, NDJSON. Cross-platform (macOS, Linux, Windows). Excel files are not read directly — export to CSV or Parquet first.
-
-## Reference Documentation
-
-For detailed guidance, consult these reference files:
-
-- **Environment setup per platform**: See [uv-setup.md](references/uv-setup.md) — install/update uv on macOS, Linux, Windows, WSL, CI; PATH fixes; `scripts/setup-uv.sh` / `scripts/setup-uv.ps1` automate it.
-- **Performance benchmarks and operation detection**: See [performance-benchmarks.md](references/performance-benchmarks.md)
-- **Integration patterns and best practices**: See [integration-patterns.md](references/integration-patterns.md)
-- **Execution templates**: See [execution-templates.md](references/execution-templates.md)
-- **Common scenarios**: See [common-scenarios.md](references/common-scenarios.md)
-
-## Quality Assurance Process
-
-### Before Execution
-1. **Analyze request** → Detect operation types (filter, join, aggregate, etc.)
-2. **Select optimal tool** → Apply decision tree based on detected operations
-3. **Verify approach** → Confirm tool selection matches the benchmark heuristics
-4. **Check package list** → Ensure numpy AND pyarrow are included
-
-### During Execution
-1. **Use lazy evaluation** when possible (Polars `scan_*`, DuckDB direct queries)
-2. **Monitor for errors** and have fallback strategy ready
-3. **Provide progress updates** for long operations
-
-### After Execution
-1. **Report performance** → Show processing time and row counts
-2. **Validate results** → Confirm output matches expectations
-3. **Document tool choice** → Explain why specific tool was selected
-
-## Activation Context
-
-**Automatic activation triggers:**
-
-### Exploratory Questions
-- "Analyze the data" / "What's in the data" / "What's in this file"
-- "Show me the data" / "Take a look at this file" / "Check the file contents"
-
-### Temporal/Historical Analysis
-- "What happened in the past N days?" / "How's last week's data?"
-- "What's the trend for the last 30 days?" / "Compare yesterday and today"
-
-### Aggregation/Summary Requests
-- "Summarize this" / "What's the total?" / "What's the average?"
-- "Show by category" / "Show statistics" / "How many?"
-
-### Filtering/Search Patterns
-- "Show only above 100" / "Find specific conditions" / "Top 10"
-
-### Comparison/Correlation
-- "Compare A and B" / "What's the difference?" / "Is there a correlation?" / "Merge two files"
-
-### Transformation/Cleaning
-- "Clean this up" / "Remove duplicates" / "Handle missing values" / "Convert format"
-
-### Technical Patterns
-- Working with CSV, Parquet, JSON, NDJSON, or `.duckdb` files
-- File paths ending in `.csv`, `.parquet`, `.json`, `.jsonl`, `.ndjson`, `.tsv`, `.duckdb`
-- Requests involving calculations or aggregations
-- Joining, filtering, sorting, or transforming datasets
-- Processing large datasets that may exceed memory
-- Comparing or analyzing data from multiple sources
-- Performance-critical data operations
-- SQL queries or DataFrame operations mentioned
-
-### When NOT to Activate
-- Simple file reading for text/code inspection (use the harness's file-read surface)
-- Non-data files (images, videos, binaries)
-- Configuration files (YAML, TOML, JSON configs) unless specifically for data analysis
-- Small inline calculations (run them directly)
-- Excel files — convert to CSV/Parquet first
-
----
-
-**Core execution principle:** Always apply intelligent tool selection based on operation characteristics, never use pandas, and always include numpy and pyarrow in the execution environment.
+# Data Scientist: Hybrid-Engine Data Processing
+
+Answer data questions through the cheapest engine and surface that can prove the answer, and
+decide where the computation should live before touching the data.
+
+## Execution surfaces: resident kernel first
+
+A persistent REPL/eval kernel (many harnesses expose one for JavaScript and Python) is the
+default surface. Reason: each one-shot process pays roughly a second of spawn-plus-import
+overhead and re-scans the input file, while a resident connection amortizes both — after a
+one-time load, repeat queries return in milliseconds. Exploration is repeat queries, so this
+difference dominates the session.
+
+1. **JavaScript kernel (Bun)**: run `scripts/ensure-js-deps.sh` once; it prints the absolute
+   import path for `@duckdb/node-api`. Dynamic-import it, connect once, query across cells.
+2. **Python kernel**: the default surface for Python work. duckdb/numpy/matplotlib are
+   typically resident; Polars and pyarrow come from `scripts/ensure-py-deps.sh`, which
+   installs them once into a user cache keyed to the kernel's interpreter —
+   `sys.path.insert` the printed directory and import. The interpreter itself is never
+   mutated.
+3. **uv lane** (`uv run --with ...`): isolation for a heavy or crash-prone one-shot that
+   should not take the kernel down.
+4. **No kernel** (plain-shell harness): the same engines as one-shots — `bun -e` for
+   DuckDB-js, `uv run python -c` for the Python stack — batching several questions per
+   process.
+
+Per-surface patterns and pitfalls: read `references/execution-surfaces.md` before first use.
+
+## Engine selection
+
+- **DuckDB** for SQL-shaped work: direct file queries, joins, aggregation, subqueries,
+  window functions. It queries CSV/Parquet/JSON in place without loading, spills to disk
+  past its memory limit, and reads remote files with the same syntax.
+- **Polars** when the pipeline is DataFrame-shaped: expression-chain transforms, reshapes,
+  streaming datasets past RAM — resident in the Python kernel via `ensure-py-deps.sh`.
+  Read `references/polars-lane.md` — the current 1.x API differs from widely-memorized
+  older spellings.
+- **numpy** when numeric work goes beyond SQL/DataFrame aggregation: statistical tests,
+  linear algebra, FFT, random sampling.
+- **matplotlib** for every chart — read `references/visualization.md` first; it carries the
+  quality bar and a mandatory visual check.
+
+Performance folklore ("X is Nx faster at filtering") varies with data shape, cardinality,
+and hardware. When the engine choice materially matters, measure on the actual data instead
+of trusting remembered multipliers.
+
+## Placement: decide where the computation lives
+
+Probe before you compute — one cell: file size, free RAM, and (when unclear) a row count via
+a direct scan. Then place the work:
+
+- **Load into memory** when the working set stays within roughly a quarter of free RAM AND
+  the session will run repeated queries: `CREATE TABLE t AS SELECT ...` (or a collected
+  DataFrame) once, then iterate. One scan up front converts every later query from a file
+  re-scan into milliseconds.
+- **Query in place / stream** when the question is single-pass, or the data exceeds RAM:
+  DuckDB reads files directly (`FROM 'data.csv'`); past RAM, cap DuckDB's memory and let it
+  spill, or use Polars' streaming engine in the Python kernel. NEVER load a larger-than-RAM
+  dataset fully into memory — swapping stalls the whole machine, while streaming merely
+  takes longer.
+- **Query remotely, in place** when the data lives elsewhere: DuckDB reads http(s)/S3
+  Parquet and CSV with projection and predicate pushdown, so fetch the columns and rows the
+  question needs, never the whole file. When data sits on another machine you can execute
+  on, ship the query to the data and return the small result. Rule: result much smaller
+  than data — move the query; repeated local iteration planned — move a pruned copy of the
+  data once.
+
+Sizing heuristics and recipes: `references/placement.md`.
+
+## Hard rules
+
+- **NEVER use pandas.** DuckDB and Polars beat it decisively on every workload this skill
+  covers, and the environments this skill assumes do not ship it — `.df()` on a DuckDB
+  result raises unless pandas is installed; convert with `.pl()` via Arrow instead.
+- Excel files are not read directly: export to CSV or Parquet first.
+
+## Output contract
+
+Answer the question; report row counts and timing for anything heavy; then stop — no bonus
+charts, no extra exploration passes beyond what the question needed. Chart when asked, or
+when the answer is a shape (trend, distribution, comparison) that prose cannot carry — then
+follow `references/visualization.md` including its visual QA step.
+
+## References
+
+| Read | When |
+| --- | --- |
+| `references/execution-surfaces.md` | before the first query on any surface: kernel patterns, one-shot recipes, escalation rules |
+| `references/polars-lane.md` | DataFrame-shaped pipeline or data past RAM: current API, Arrow handoff, package sets |
+| `references/placement.md` | before heavy or remote work: sizing probe, memory limits, remote reads |
+| `references/visualization.md` | before any chart: type selection, quality bar, CJK fonts, visual QA |
+| `references/uv-setup.md` | uv missing or broken on this machine |
+
+## CLI fallback
+
+When no kernel or REPL surface exists, `uv run scripts/quick-query.py <file> [SQL]`
+(`--filter <polars-sql-expr>`, `--describe`) answers ad-hoc questions with zero code.
+Supports CSV, Parquet, JSON, NDJSON.