@damiangilgonzalez/harness 1.3.7

package/skills/chrome-extension/scripts/verify.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# verify.sh — lint a Manifest V3 manifest.json against MV3 invariants.
+# Read-only. Exits 0 on a clean/empty target (no manifest = nothing to check).
+# Usage: scripts/verify.sh [target-dir]   (defaults to current directory)
+set -euo pipefail
+
+TARGET="${1:-.}"
+
+if ! command -v node >/dev/null 2>&1; then
+  echo "verify.sh: node not found; skipping manifest lint." >&2
+  exit 0
+fi
+
+# Find manifest.json files, ignoring build/vendor dirs. No matches => clean pass.
+MANIFESTS="$(
+  find "$TARGET" \
+    -type d \( -name node_modules -o -name dist -o -name .git \) -prune -o \
+    -type f -name manifest.json -print 2>/dev/null
+)"
+
+if [ -z "$MANIFESTS" ]; then
+  echo "verify.sh: no manifest.json under '$TARGET' — nothing to check."
+  exit 0
+fi
+
+FAIL=0
+while IFS= read -r m; do
+  [ -z "$m" ] && continue
+  node - "$m" <<'NODE' || FAIL=1
+const fs = require("fs");
+const path = process.argv[2];
+const errs = [];
+let mf;
+try {
+  mf = JSON.parse(fs.readFileSync(path, "utf8"));
+} catch (e) {
+  console.error(`FAIL ${path}: invalid JSON — ${e.message}`);
+  process.exit(1);
+}
+if (mf.manifest_version !== 3) errs.push(`manifest_version must be 3 (got ${JSON.stringify(mf.manifest_version)})`);
+const bg = mf.background || {};
+if ("scripts" in bg) errs.push("background.scripts is MV2; use background.service_worker (string)");
+if ("page" in bg) errs.push("background.page is MV2; use background.service_worker (string)");
+if ("persistent" in bg) errs.push("background.persistent does not exist in MV3; remove it");
+if (mf.background !== undefined) {
+  if (typeof bg.service_worker !== "string") errs.push("background.service_worker must be a string");
+}
+if (typeof mf.name !== "string" || !mf.name) errs.push("missing name");
+if (typeof mf.version !== "string" || !mf.version) errs.push("missing version");
+if (mf.icons === undefined || typeof mf.icons !== "object") errs.push("missing icons");
+if (errs.length) {
+  console.error(`FAIL ${path}:`);
+  for (const e of errs) console.error(`  - ${e}`);
+  process.exit(1);
+}
+console.log(`OK   ${path}`);
+NODE
+done <<EOF
+$MANIFESTS
+EOF
+
+exit "$FAIL"

package/skills/claude-project-setup/SKILL.md

@@ -0,0 +1,84 @@
+---
+name: claude-project-setup
+description: "Use when starting a new project, initializing a repository for Claude Code, or when the user asks to configure CLAUDE.md, rules, commands, skills, or agents for a project."
+tags: ["setup", "claude-code", "project-init"]
+profiles: [core, ui, full]
+---
+
+# Claude Project Setup
+
+## Overview
+
+This skill fetches the project setup guide from Notion and applies it to configure a new Claude Code project. The guide lives in Notion so it stays up-to-date across all projects without local file duplication. This ensures every project you create follows the official Claude Code best practices and organizational standards.
+
+## When to Use
+
+Use this skill when:
+- **Starting a new project or repo from scratch** and need to establish Claude Code structure
+- **User asks to "set up Claude"** or "configure this project for Claude Code"
+- **User asks about CLAUDE.md, rules, commands, skills, or agents** structure or organization
+- **User wants guidance on organizing** a Claude Code project
+- **Initializing a repository** and need to establish configuration files and conventions
+- **Setting up best practices** for a multi-agent or team-based project
+
+## How It Works
+
+1. **Fetch the guide from Notion** (always fresh):
+   - Use the MCP tool `mcp__claude_ai_Notion__notion-fetch` with ID: `34187201f6958042af22fd647d8e0409`
+   - **Important**: Always fetch fresh — do NOT use cached content, as the guide updates frequently
+   - This ensures you're applying the latest best practices and conventions
+
+2. **Read and understand** the full Notion page content:
+   - Review the structure recommendations
+   - Understand the components available (CLAUDE.md, rules, commands, skills, agents)
+   - Identify best practices for the specific project context
+
+3. **Apply the guide** to the current project:
+   - Ask the user which components they need (CLAUDE.md? custom rules? commands? skills? agents?)
+   - Understand the project's tech stack and purpose
+   - Create the directory structure as described in the Notion guide
+   - Generate tailored files adapted to the specific project (language, framework, domain conventions)
+   - Set up any configuration hooks or automated behaviors the user needs
+
+## Example Conversations
+
+### Example 1: New Full-Stack Project
+**User:** "I'm starting a new Next.js project with Claude Code. How should I organize it?"
+
+**What you do:**
+1. Fetch the Notion setup guide (fresh)
+2. Ask what components are needed: "Would you like a CLAUDE.md for documentation, custom rules for code style, commands for common tasks, or custom skills?"
+3. Generate a project structure with CLAUDE.md covering Next.js conventions, rules for code formatting and testing, commands for build/test/deploy, and possibly custom skills for frontend or backend tasks
+4. Provide clear file-by-file guidance on what to put in each component
+
+### Example 2: Existing Project Enhancement
+**User:** "We have a Python backend but want to set up Claude Code properly. What rules should we define?"
+
+**What you do:**
+1. Fetch the Notion setup guide (fresh)
+2. Understand the existing project structure
+3. Ask what they want to standardize: linting rules? commit conventions? testing strategy?
+4. Create a `.claude/` directory structure with custom rules matching their Python/backend conventions
+5. Provide ready-to-customize rule files tailored to their tech stack
+
+### Example 3: Team Collaboration Setup
+**User:** "How do I configure Claude Code for my team so everyone follows the same patterns?"
+
+**What you do:**
+1. Fetch the Notion setup guide (fresh)
+2. Discuss team needs: code style, shared skills, automated checks, agent workflows
+3. Create a comprehensive CLAUDE.md with team standards
+4. Set up shared rules and commands that enforce consistency
+5. Document how to extend skills and agents for team members
+
+## Important Notes
+
+- **ALWAYS fetch the Notion page fresh** every time you use this skill — the guide is a living document and may have been updated with new best practices, conventions, or improvements
+- The Notion page ID is: `34187201f6958042af22fd647d8e0409`
+- Use `mcp__claude_ai_Notion__notion-fetch` tool to retrieve the latest content
+- Adapt the guide to the specific project context:
+  - Language and framework (Python, Node.js, Go, Rust, etc.)
+  - Project domain (backend API, frontend UI, full-stack, ML pipeline, etc.)
+  - Team size and collaboration needs
+  - Existing conventions or standards to preserve
+- If the user already has partial setup, ask what's missing and add those components incrementally

package/skills/claude-project-setup/evals/cases.yaml

@@ -0,0 +1,32 @@
+skill: claude-project-setup
+
+should_trigger:
+  - prompt: "Estoy empezando un proyecto nuevo, ayudame a configurar Claude Code."
+    why: "Disparador directo de claude-project-setup."
+  - prompt: "Quiero inicializar el repo con CLAUDE.md y reglas para el agente."
+    why: "Disparador directo de claude-project-setup."
+  - prompt: "Configurame comandos y skills para este proyecto desde cero."
+    why: "Disparador directo de claude-project-setup."
+  - prompt: "Necesito montar agentes y hooks para un repo que acabo de crear."
+    why: "Disparador directo de claude-project-setup."
+  - prompt: "Ayudame a dejar listo el repositorio para trabajar con Claude Code."
+    why: "Disparador directo de claude-project-setup."
+
+should_not_trigger:
+  - prompt: "Revisa el CLAUDE.md actual y dime si sigue las buenas practicas."
+    route_to: "writing-for-agents"
+    why: "Revisar un CLAUDE.md pensando en que lo lee un agente es el trabajo de writing-for-agents, no de claude-project-setup."
+  - prompt: "Explicame que es un hook de Claude Code."
+    route_to: "none"
+    why: "Fuera del alcance de claude-project-setup."
+  - prompt: "Anade una regla nueva a un CLAUDE.md que ya existe."
+    route_to: "writing-for-agents"
+    why: "Editar un CLAUDE.md ya existente es el trabajo de writing-for-agents, no de claude-project-setup."
+  - prompt: "Documenta el dominio de este proyecto."
+    route_to: "domain-modeling"
+    why: "Documentar el dominio del proyecto es el trabajo de domain-modeling, no de claude-project-setup."
+
+capability:
+  - scenario: "El usuario pide inicializar un proyecto nuevo para Claude Code."
+    must_include:
+      - "CLAUDE.md"

package/skills/clickhouse-analytics/SKILL.md

@@ -0,0 +1,159 @@
+---
+name: clickhouse-analytics
+description: "Use when running a ClickHouse server for high-volume OLAP: choosing a MergeTree engine and ORDER BY/PARTITION BY keys, ingesting billions of event/log/metric rows, pre-aggregating with materialized views, or fixing a query that scans instead of pruning. NOT in-process file analytics (that is `duckdb`), NOT OLTP CRUD indexing (that is `postgresdb`)."
+tags: [clickhouse, olap, columnar, mergetree, analytics, materialized-views, data-ingestion, sql]
+recommends: [duckdb, postgresdb, dashboard, kpi-framework, reporting, business-intelligence]
+origin: risco
+---
+
+# ClickHouse analytics
+
+ClickHouse is a multi-user, always-on, replicated columnar server built to ingest continuous high-volume writes and answer aggregation queries over billions of rows in milliseconds. You reach for it when the workload is "append a firehose of events/logs/metrics, then GROUP BY them for dashboards." Target **26.3 LTS** (v26.3.12.3, 2026-05-22) — several defaults below changed in the 26.x line, so version matters.
+
+The fork before you write any DDL: files on a laptop, in-process, no server, no concurrent writers → `../duckdb/SKILL.md`; app CRUD, point updates, foreign keys, row locks, RLS, migrations → `../postgresdb/SKILL.md`; `clickhouse-server`, replication, concurrent writers, 100M+ rows/s ingest → this skill.
+
+Instrumenting capture (GA4/PostHog) is `../analytics/SKILL.md`; charting the result for humans is `../dashboard/SKILL.md`; deciding which metrics matter is `../kpi-framework/SKILL.md`. ClickHouse is the engine underneath all three.
+
+## Pick the engine first
+
+The engine decides dedup and merge behavior, and you cannot change `ORDER BY`/`PARTITION BY` later without a rebuild — so choose before typing `CREATE TABLE`.
+
+| Engine | Use it for | Dedup / merge behavior | Gotcha |
+|---|---|---|---|
+| `MergeTree` | Append-only events, logs, metrics | No dedup of logical rows; inserts dedup'd by block since 26.2 | The default and 90% of tables |
+| `ReplacingMergeTree(ver)` | Upserts / keep latest version per key | Collapses duplicate `ORDER BY` keys *eventually* during merges | Reads see dupes until merged; need `FINAL` to force — slow, keep off hot path |
+| `AggregatingMergeTree` | Pre-aggregated rollups fed by a materialized view | Merges `-State` partials per `ORDER BY` key | Only useful behind an MV; query with `-Merge` |
+| `SummingMergeTree` | Simple additive rollups (sum only) | Sums numeric columns per `ORDER BY` key on merge | Can't do uniq/quantile — use AggregatingMergeTree for those |
+| `Replicated*` prefix | High availability / multi-replica | Same as base engine + ZooKeeper/Keeper replication | Production HA wrapper; combine with any of the above |
+
+Default to `MergeTree`. Move to `AggregatingMergeTree` only when you are pre-aggregating through a materialized view. Full matrix and reasoning: `references/schema-and-engines.md`.
+
+## Schema rules
+
+1. **`ORDER BY` is your single biggest perf lever — a good one cuts query time ~100x.** It defines the sparse primary index that prunes which granules get read. Get this right above everything else.
+2. **Order the key low-cardinality → high-cardinality, left to right, driven by `WHERE`/`GROUP BY` — never by join keys.** 3–5 columns. The leftmost column should be the one you filter on most; cardinality rises as you go right. Timeseries: put the raw timestamp last, often `(tenant_id, toStartOfDay(ts), event_type, ts)`.
+3. **Treat `ORDER BY` and `PARTITION BY` as immutable.** Changing either almost always means a new table + `INSERT ... SELECT` migration. Decide deliberately now.
+4. **Partition coarsely — by month, or by day only at very high volume.** Partitioning is for *data lifecycle* (TTL, `DROP PARTITION`), not query speed; the sparse index does speed. Per-hour or per-`toYYYYMMDD` on a high-cardinality stream creates thousands of partitions → too many parts → merge storms.
+5. **Right-size types and use codecs.** `LowCardinality(String)` for columns under ~10k distinct values (enum-like: country, event_type, status). Smallest int that fits. `CODEC(Delta, ZSTD)` for monotonic timestamps/counters; `CODEC(ALP)` for float columns (26.3, beats Gorilla on many workloads); native `JSON` type (GA in 26.3) for semi-structured payloads instead of stringly-typed blobs.
+
+```sql
+CREATE TABLE events
+(
+    tenant_id    UInt32,
+    ts           DateTime64(3) CODEC(Delta, ZSTD),
+    event_type   LowCardinality(String),
+    user_id      UInt64,
+    country      LowCardinality(String),
+    revenue      Float64 CODEC(ALP),
+    props        JSON
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMM(ts)             -- monthly: coarse, for TTL/drops
+ORDER BY (tenant_id, toStartOfDay(ts), event_type, ts)
+TTL toDateTime(ts) + INTERVAL 18 MONTH;
+```
+
+Depth (cardinality math, codec table, type mapping, partition-count budget): `references/schema-and-engines.md`.
+
+## Ingestion rules
+
+```sql
+-- Bad: row-at-a-time. Each statement becomes its own tiny part.
+INSERT INTO events VALUES (1, now(), 'click', 42, 'ES', 0, '{}');
+INSERT INTO events VALUES (1, now(), 'view',  42, 'ES', 0, '{}');
+-- ... 10k more single inserts -> 10k parts -> merges can't keep up
+```
+
+```sql
+-- Good: one batch of many rows (aim 10k–100k+ per INSERT).
+INSERT INTO events VALUES
+  (1, now(), 'click', 42, 'ES', 0, '{}'),
+  (1, now(), 'view',  42, 'ES', 0, '{}'),
+  /* ...thousands more... */ ;
+
+-- Or load straight from object storage, no client batching at all:
+INSERT INTO events
+SELECT * FROM s3('https://bucket.s3.amazonaws.com/events/2026/*.parquet', 'Parquet');
+```
+
+- **Async inserts are enabled by default starting 26.3 LTS.** The server buffers small inserts in memory and flushes on a size/time threshold, so many client-side batchers become unnecessary. Flush fires on the *first* threshold hit: `async_insert_max_query_number` (default 450) or the adaptive busy timeout, between `async_insert_busy_timeout_min_ms` (default 50ms) and a data-rate-driven max (adaptive since 24.2).
+- **Insert deduplication is on by default for all inserts as of 26.2** (previously sync-only), and works end-to-end across async inserts and dependent materialized views since 26.1. **Net effect: retrying a failed insert is safe** — an identical block won't double-count. Pass `insert_deduplication_token` when you want explicit control over what counts as identical.
+- **Keep inserts synchronous** when you must read-your-write immediately, or when you already batch large blocks yourself and want no buffering latency.
+
+S3/Kafka/file recipes, async-insert tuning knobs, dedup tokens: `references/ingestion-and-mvs.md`.
+
+## Materialized views and pre-aggregation
+
+For anything beyond raw sum/count (uniq, quantiles, argMax), pre-aggregate incrementally with `AggregatingMergeTree` + a materialized view storing `-State` partials, queried back with `-Merge`.
+
+```sql
+CREATE TABLE events_hourly
+(
+    tenant_id  UInt32,
+    hour       DateTime,
+    users      AggregateFunction(uniq, UInt64),
+    revenue    AggregateFunction(sum, Float64)
+)
+ENGINE = AggregatingMergeTree
+PARTITION BY toYYYYMM(hour)
+ORDER BY (tenant_id, hour);            -- MV GROUP BY MUST match this
+
+CREATE MATERIALIZED VIEW events_hourly_mv TO events_hourly AS
+SELECT tenant_id,
+       toStartOfHour(ts) AS hour,
+       uniqState(user_id) AS users,
+       sumState(revenue)  AS revenue
+FROM events
+GROUP BY tenant_id, hour;              -- no POPULATE on a big base table
+```
+
+```sql
+-- Read it back: -Merge collapses the partial states.
+SELECT tenant_id, hour, uniqMerge(users) AS uniq_users, sumMerge(revenue) AS rev
+FROM events_hourly
+GROUP BY tenant_id, hour;
+```
+
+- **The MV's `GROUP BY` must match the target table's `ORDER BY`** so merges stay efficient.
+- **Never `POPULATE` a billion-row base table** — it blocks the MV and can OOM. Create the MV empty (it captures new rows immediately), then backfill history in time-bounded `INSERT ... SELECT` windows. Full backfill walkthrough: `references/ingestion-and-mvs.md`.
+
+## Query optimization
+
+The sparse index only prunes on `ORDER BY` prefix columns. When a hot query filters on a column the primary key doesn't cover, in order of reach for:
+
+1. **`PREWHERE`** — ClickHouse auto-applies it, but an explicit `PREWHERE` on a cheap, highly selective column reads that column first and skips other columns for non-matching rows. Cuts I/O.
+2. **Projection** — an alternate `ORDER BY`/pre-aggregation stored with the table; ClickHouse picks it transparently. Best when one secondary access pattern is common and worth the storage.
+3. **Data-skipping index** — `minmax` (correlated-with-PK ranges), `set` (low distinct count), `bloom_filter` (high-cardinality equality/`IN`). Cheaper than a projection, coarser pruning.
+
+Decision: PK can't prune and you query *one* alternate sort order a lot → projection. You just need to skip granules on a side column → skip index (`bloom_filter` for high-cardinality `=`/`IN`, `minmax` for ranges). Inspect with `EXPLAIN indexes = 1` and `SET send_logs_level = 'trace'` to see granules read. Walkthrough + slow-query recipes: `references/query-optimization.md`.
+
+```sql
+SELECT event_type, count() FROM events
+PREWHERE country = 'ES'                 -- cheap, selective: filter before reading the rest
+WHERE ts >= now() - INTERVAL 7 DAY
+GROUP BY event_type;
+```
+
+## Operations
+
+- **Watch part count.** `SELECT table, count() FROM system.parts WHERE active GROUP BY table` — a growing number means inserts are too small/frequent or partitioning is too fine. Fix the insert pattern, not the merge settings.
+- **Retention via TTL**, not `DELETE`. `TTL` on the table drops expired data during merges automatically.
+- **`ALTER TABLE ... DROP PARTITION` is instant and free**; row-level `DELETE`/`ALTER DELETE` is a mutation that rewrites parts — avoid it for bulk cleanup. This is the payoff of coarse partitioning.
+- **`ReplacingMergeTree` reads can see un-merged duplicates.** Use `FINAL` only on cold/admin queries, never in dashboards — it merges at query time.
+
+## Anti-patterns
+
+| Anti-pattern | Why it hurts | Do instead |
+|---|---|---|
+| `MergeTree` with no `ORDER BY` (or `ORDER BY tuple()`) on a queried table | No sparse index → every query full-scans | Pick a 3–5 col key, low→high cardinality, `WHERE`-driven |
+| `PARTITION BY` a high-cardinality col / per-hour / per-day at low volume | Thousands of partitions → too many parts → merge storms | Partition by `toYYYYMM`; the sparse index does the speed |
+| Single-row `INSERT ... VALUES` in a loop | Each becomes a tiny part; merges can't keep up | Batch 10k–100k+ rows, or rely on 26.3 async inserts |
+| `POPULATE` on a billion-row base table's MV | Blocks the MV, can OOM | Create MV empty, backfill in time windows |
+| `SELECT *` on a wide table | Reads every column, defeats columnar storage | Select only the columns you need |
+| `FINAL` in a dashboard query | Forces merge at query time → slow | Keep `FINAL` off hot paths; accept eventual dedup |
+| ClickHouse for OLTP point-updates / single-row reads by id | Wrong engine; no real updates, weak point lookups | Use `../postgresdb/SKILL.md` |
+| MV `GROUP BY` not matching target `ORDER BY` | Inefficient merges, wrong rollups | Align them exactly |
+
+## Verification
+
+`scripts/verify.sh <file.sql>` is a static linter over candidate ClickHouse DDL/queries: flags `MergeTree` without `ORDER BY`, over-fine `PARTITION BY`, single-row `INSERT ... VALUES`, `POPULATE` on materialized views, `SELECT *`, and `FINAL`. Read-only, no live cluster needed, exits 0 on clean input.

package/skills/clickhouse-analytics/evals/README.md

@@ -0,0 +1,3 @@
+# Evals — clickhouse-analytics
+
+These cases are prompts fed to the skill router. `should_trigger` entries expect this skill (`clickhouse-analytics`) to be selected; `should_not_trigger` entries each name a real sibling in `route_to` that should win instead (duckdb for file/in-process analytics, postgresdb for OLTP, analytics for event capture, dashboard for layout, kpi-framework for metric strategy). The single `capability` case is graded by a judge against its `must_include` rubric — a good answer designs a MergeTree-family table with a justified low→high-cardinality ORDER BY and coarse PARTITION BY, gives 26.x-correct batched/async ingestion with safe-retry dedup, builds an AggregatingMergeTree MV (no POPULATE) and a query-side accelerator, and names an anti-pattern avoided. Run them through the repo's eval harness the same way as other skills under `skills/*/evals/cases.yaml`; there is no live ClickHouse cluster involved — routing is judged on the prompt and capability on the produced answer.

package/skills/clickhouse-analytics/evals/cases.yaml

@@ -0,0 +1,45 @@
+skill: clickhouse-analytics
+
+should_trigger:
+  - prompt: "We're ingesting 2B clickstream events a day into ClickHouse — what ENGINE, ORDER BY and PARTITION BY should the table use?"
+    why: Core schema-design request at scale; engine + sort-key + partition choice is the heart of this skill.
+  - prompt: "Our ClickHouse dashboard queries scan the whole table, the primary key isn't pruning anything."
+    why: Non-obvious — points at sparse-index/ORDER-BY-prefix pruning, not generic SQL tuning.
+  - prompt: "After our hourly ingest job, parts keep piling up and merges can't keep up."
+    why: Non-obvious — too-many-parts from tiny/over-partitioned inserts; the merge-pressure symptom maps to ingestion + partition rules here.
+  - prompt: "Build a materialized view that keeps a rolling uniq-users-per-hour aggregate as events stream in."
+    why: AggregatingMergeTree + -State/-Merge incremental pre-aggregation, the MV section of this skill.
+  - prompt: "Montar ClickHouse para analítica de eventos a gran escala con ingesta continua y consultas sub-segundo."
+    why: Spanish trigger for standing up a high-volume ClickHouse analytics server.
+  - prompt: "Should the upserted order rows table be ReplacingMergeTree or AggregatingMergeTree?"
+    why: MergeTree-family engine choice with the FINAL/dedup tradeoff — exactly the engine decision table.
+  - prompt: "Consultes lentes a ClickHouse, l'index no poda particions — com l'optimitzo?"
+    why: Catalan trigger for query optimization / sparse-index pruning on a ClickHouse server.
+
+should_not_trigger:
+  - prompt: "Query a folder of Parquet files on my laptop with SQL, no server to run."
+    route_to: duckdb
+    why: In-process, single-machine, file-based analytics with no server — DuckDB's space, not a ClickHouse cluster.
+  - prompt: "Design the orders table with foreign keys and add an index to fix this slow OLTP point-lookup query."
+    route_to: postgresdb
+    why: Transactional CRUD schema, FKs, row-level point reads/updates — an OLTP relational job, not columnar OLAP.
+  - prompt: "Install PostHog and instrument signup and checkout events in our web app."
+    route_to: analytics
+    why: Event-capture instrumentation in the app, not designing the columnar store the events land in.
+  - prompt: "Lay out an executive KPI dashboard so the numbers fit on one screen."
+    route_to: dashboard
+    why: Visual layout/charting of metrics for humans, not the engine querying them.
+  - prompt: "Decide which five metrics the business should track this quarter and set their targets."
+    route_to: kpi-framework
+    why: Defining which metrics matter and their targets is strategy, not ClickHouse schema/query work.
+
+capability:
+  - scenario: "A team has a high-volume web-events table feeding a dashboard whose GROUP BY query over the last 7 days is now too slow. They ask how to model the table, ingest the firehose, and serve sub-second aggregates on ClickHouse 26.3 LTS."
+    must_include:
+      - Picks a MergeTree-family engine and justifies it (MergeTree for append-only events; AggregatingMergeTree only behind an MV).
+      - Proposes an ORDER BY ordered low-cardinality to high-cardinality, driven by WHERE/GROUP BY (not join keys), with the raw timestamp last; 3-5 columns.
+      - Chooses a coarse PARTITION BY (toYYYYMM monthly, or daily only at extreme volume) and explains partitioning is for data lifecycle, not query speed.
+      - Gives batched/async ingestion guidance — batches of 10k-100k+ rows or async inserts default-on since 26.3, and notes insert dedup default-on since 26.2 makes retries safe.
+      - Builds an AggregatingMergeTree materialized view with uniqState/sumState and -Merge readback, GROUP BY matching the target ORDER BY, and explicitly avoids POPULATE on the big base table (backfill in bounded windows).
+      - Adds a query-side accelerator (PREWHERE, projection, or a bloom_filter/minmax skip index) for a filter the ORDER BY prefix doesn't cover.
+      - Names at least one anti-pattern it is avoiding (e.g. single-row INSERTs, per-hour partitioning, SELECT *, or FINAL in the hot path).

package/skills/clickhouse-analytics/references/ingestion-and-mvs.md

@@ -0,0 +1,109 @@
+# Ingestion and materialized views
+
+Target version: 26.3 LTS. The 26.x line changed several insert defaults — note them per item below.
+
+## Batch sizing
+
+ClickHouse turns every `INSERT` into at least one part on disk. A part is a sorted, compressed unit that the background merge scheduler later combines. Too many small parts = the scheduler falls behind = "too many parts" errors and slow queries.
+
+- Aim for **10k–100k+ rows per `INSERT`** when you batch client-side.
+- Or rely on **async inserts** (default-on since 26.3) to batch server-side.
+- Never one row per statement in a loop.
+
+## Async inserts (default-on since 26.3 LTS)
+
+The server holds incoming small inserts in an in-memory buffer and flushes them as one part when the first threshold is reached:
+
+| Setting | Default | Meaning |
+|---|---|---|
+| `async_insert` | `1` (since 26.3) | Buffer inserts server-side |
+| `async_insert_max_query_number` | `450` | Flush after this many buffered queries |
+| `async_insert_busy_timeout_min_ms` | `50` | Lower bound of the adaptive flush timer |
+| `async_insert_busy_timeout_max_ms` | data-rate driven | Upper bound; adaptive since 24.2 |
+| `wait_for_async_insert` | `1` | Client waits for the flush to confirm durability |
+
+The busy timeout is **adaptive**: at high data rates it shortens toward the min, at low rates it lengthens toward the max, balancing latency vs part count automatically.
+
+Keep inserts **synchronous** (`async_insert=0`) when you need read-your-write consistency immediately, or when you already send large well-sized blocks and want zero buffering latency.
+
+## Deduplication (default-on for all inserts since 26.2)
+
+- Before 26.2, insert dedup was sync-only. **As of 26.2 it applies uniformly to sync and async inserts.**
+- Since 26.1 dedup works **end-to-end across async inserts and dependent materialized views**.
+- Net effect: an identical re-sent block is dropped, so **retrying a failed insert is safe** — no double counting.
+- Control identity explicitly with `insert_deduplication_token`: same token = same logical insert, so you can dedup across differing row content or split a logical batch.
+
+```sql
+INSERT INTO events SETTINGS insert_deduplication_token = 'batch-2026-06-02-0007'
+SELECT * FROM s3('https://bucket/events/2026-06-02/*.parquet', 'Parquet');
+```
+
+## Loading from external sources
+
+```sql
+-- S3 (Parquet / JSONEachRow / CSV auto-detected by extension or explicit format)
+INSERT INTO events
+SELECT * FROM s3('https://bucket.s3.amazonaws.com/events/2026/*.parquet', 'Parquet');
+
+-- Local/served files via file() or the clickhouse-client --query with FORMAT
+INSERT INTO events FROM INFILE 'events.csv.gz' FORMAT CSV;
+```
+
+```sql
+-- Kafka: a Kafka engine table + an MV that drains it into the MergeTree table.
+CREATE TABLE events_queue (raw String)
+ENGINE = Kafka
+SETTINGS kafka_broker_list = 'kafka:9092',
+         kafka_topic_list  = 'events',
+         kafka_group_name  = 'ch-ingest',
+         kafka_format      = 'JSONEachRow';
+
+CREATE MATERIALIZED VIEW events_queue_mv TO events AS
+SELECT * FROM events_queue;          -- the Kafka engine itself never stores rows
+```
+
+## AggregatingMergeTree materialized view — end to end
+
+Goal: a rolling **uniq users and revenue per tenant per hour** that stays current as events stream in.
+
+```sql
+-- 1. Target table storing partial aggregate STATES.
+CREATE TABLE events_hourly
+(
+    tenant_id  UInt32,
+    hour       DateTime,
+    users      AggregateFunction(uniq, UInt64),
+    revenue    AggregateFunction(sum, Float64)
+)
+ENGINE = AggregatingMergeTree
+PARTITION BY toYYYYMM(hour)
+ORDER BY (tenant_id, hour);
+
+-- 2. MV that writes states on every new insert into events.
+--    GROUP BY MUST equal the target ORDER BY. No POPULATE.
+CREATE MATERIALIZED VIEW events_hourly_mv TO events_hourly AS
+SELECT tenant_id,
+       toStartOfHour(ts)  AS hour,
+       uniqState(user_id) AS users,
+       sumState(revenue)  AS revenue
+FROM events
+GROUP BY tenant_id, hour;
+
+-- 3. Backfill history WITHOUT POPULATE: bounded windows so it never OOMs.
+INSERT INTO events_hourly
+SELECT tenant_id, toStartOfHour(ts), uniqState(user_id), sumState(revenue)
+FROM events
+WHERE ts >= '2026-01-01' AND ts < '2026-02-01'   -- one month at a time
+GROUP BY tenant_id, hour;
+-- repeat per month; dedup keeps reruns safe.
+
+-- 4. Read with -Merge to collapse states into final values.
+SELECT tenant_id, hour,
+       uniqMerge(users)  AS uniq_users,
+       sumMerge(revenue) AS rev
+FROM events_hourly
+GROUP BY tenant_id, hour
+ORDER BY tenant_id, hour;
+```
+
+Why no `POPULATE`: it scans the entire base table in one shot, blocks the MV from capturing concurrent inserts during that scan, and can exhaust memory on a billion-row table. Creating the MV empty captures new rows from the moment of creation; the bounded backfill fills the past safely.

package/skills/clickhouse-analytics/references/query-optimization.md

@@ -0,0 +1,76 @@
+# Query optimization
+
+## How the sparse index prunes
+
+ClickHouse reads data in **granules** (8192 rows by default). It keeps one primary-index mark per granule — the `ORDER BY` column values at the granule boundary. A query whose `WHERE` constrains a **prefix of the `ORDER BY`** lets the engine binary-search the marks and read only matching granules. Everything else is skipped without touching disk.
+
+Consequence: a filter on a column **not** in the `ORDER BY` prefix prunes nothing — the engine scans every granule. That is the usual cause of "the primary key isn't pruning." The fixes below all add a *secondary* way to prune.
+
+Inspect what actually happened:
+
+```sql
+EXPLAIN indexes = 1
+SELECT count() FROM events WHERE country = 'ES' AND ts >= now() - INTERVAL 7 DAY;
+-- shows which indexes were used and granules selected vs total
+
+-- or, run the query with trace logging to see granules/rows read:
+SET send_logs_level = 'trace';
+```
+
+## PREWHERE
+
+`PREWHERE` reads its columns *first*, evaluates the predicate, and only then reads the remaining columns for surviving rows. ClickHouse auto-moves predicates into `PREWHERE`, but an explicit `PREWHERE` on a **cheap, highly selective** column forces the order you want and cuts I/O on wide tables.
+
+```sql
+SELECT event_type, count() FROM events
+PREWHERE country = 'ES'                 -- 1 narrow column read first, filters hard
+WHERE ts >= now() - INTERVAL 7 DAY
+GROUP BY event_type;
+```
+
+Use it when one filter column is small and eliminates most rows before the expensive columns are touched.
+
+## Projections
+
+A projection is a second physical copy of the table data with a different `ORDER BY` and/or pre-aggregation, stored alongside the table. The optimizer picks it transparently when it serves the query better.
+
+```sql
+ALTER TABLE events ADD PROJECTION by_country
+( SELECT * ORDER BY (country, ts) );
+ALTER TABLE events MATERIALIZE PROJECTION by_country;   -- builds it for existing data
+```
+
+Cost: extra storage and insert work (every insert maintains the projection). Worth it when **one** secondary access pattern is frequent and latency-critical.
+
+## Data-skipping indexes
+
+Cheaper and coarser than projections — they store summaries per block of granules and skip blocks that can't match.
+
+| Index | Built for | Example |
+|---|---|---|
+| `minmax` | Columns correlated with the sort order (ranges) | `INDEX idx_amt revenue TYPE minmax GRANULARITY 4` |
+| `set(N)` | Low distinct count per block | `INDEX idx_et event_type TYPE set(100) GRANULARITY 4` |
+| `bloom_filter` | High-cardinality equality / `IN` | `INDEX idx_uid user_id TYPE bloom_filter GRANULARITY 4` |
+
+```sql
+ALTER TABLE events ADD INDEX idx_uid user_id TYPE bloom_filter GRANULARITY 4;
+ALTER TABLE events MATERIALIZE INDEX idx_uid;   -- backfill for existing parts
+```
+
+## Which accelerator, when
+
+- Query filters a column the PK prefix doesn't cover, **point/`IN` lookups**, high cardinality → `bloom_filter` skip index.
+- Query filters a **range** on a side column loosely correlated with the sort → `minmax` skip index.
+- You repeatedly query in a **whole different sort order** (and storage is acceptable) → projection.
+- You repeatedly compute the **same aggregate** at read time → pre-aggregate with an `AggregatingMergeTree` MV instead (see `ingestion-and-mvs.md`).
+- A cheap, selective filter column on a wide table → explicit `PREWHERE`.
+
+## Common slow-query fixes
+
+| Symptom | Likely cause | Fix |
+|---|---|---|
+| Full scan despite a `WHERE` | Filter not on `ORDER BY` prefix | Skip index / projection, or revisit the key |
+| Slow `SELECT *` dashboards | Reading every column | Select only needed columns |
+| `ReplacingMergeTree` query slow | `FINAL` merging at query time | Drop `FINAL` off hot path; dedup in the MV layer |
+| Query fast cold, slow under load | Too many parts | Fix insert batching/partitioning (see schema ref) |
+| Memory blows up on GROUP BY | Huge cardinality grouping | Pre-aggregate via MV; or `max_bytes_before_external_group_by` |

package/skills/clickhouse-analytics/references/schema-and-engines.md

@@ -0,0 +1,63 @@
+# Schema and engines (deep dive)
+
+## Engine matrix
+
+| Engine | Keeps | Merge action | Query pattern | When |
+|---|---|---|---|---|
+| `MergeTree` | Every inserted row | Just sorts/merges parts | Direct `SELECT ... GROUP BY` | Append-only events, logs, metrics — the default |
+| `ReplacingMergeTree([ver][, is_deleted])` | Latest row per `ORDER BY` key | Drops older versions on merge | `SELECT ... FINAL` or app-side latest | Upserts, CDC sink, "current state per id" |
+| `AggregatingMergeTree` | Partial aggregate states per key | Combines `-State` partials | `SELECT ...Merge(col)` behind an MV | Pre-aggregated uniq/quantile/argMax rollups |
+| `SummingMergeTree([cols])` | Summed numerics per key | Adds numeric columns on merge | Direct `SELECT sum-already-done` | Simple additive counters only |
+| `CollapsingMergeTree(sign)` | Rows with +1/-1 sign | Cancels +1/-1 pairs | Sum the sign column | Mutable rows with a known prior state |
+| `Replicated<X>` | Same as `<X>` | Same + cross-replica sync via Keeper | Same | Any of the above, in production HA |
+
+Rule of thumb: start at `MergeTree`. Add `AggregatingMergeTree` only behind a materialized view. Reach for `ReplacingMergeTree` only when you genuinely have upserts and can tolerate eventual dedup. Use the `Replicated` prefix in any clustered deployment.
+
+## ORDER BY — the reasoning
+
+The `ORDER BY` defines the **sparse primary index**: ClickHouse stores one index mark per granule (8192 rows by default), so the index is tiny and lives in memory. A query whose `WHERE` touches a *prefix* of the `ORDER BY` can skip whole granules.
+
+Construction:
+
+1. List the columns you filter and group by most. Ignore join keys — they don't help pruning.
+2. Sort that list **low cardinality first, high cardinality last**. Low-cardinality leading columns produce long runs of equal values, which compress hard and let the index prune large contiguous ranges.
+3. Keep it to **3–5 columns**. Extra columns past the useful prefix only cost insert-time sorting.
+4. For timeseries, bucket the timestamp early and keep the raw timestamp last: `(tenant_id, toStartOfDay(ts), event_type, ts)`. The bucket prunes by day; the trailing raw `ts` orders within a granule for range scans.
+
+A good key vs a bad key on the same data is routinely a ~100x query-time difference. This is the highest-leverage decision in the whole schema, and it is effectively immutable — changing it means a new table and an `INSERT ... SELECT` migration.
+
+`PRIMARY KEY` may be a prefix of `ORDER BY` when you want a smaller index than the sort order (e.g. sort by `(a, b, c)` but index only `(a, b)`).
+
+## PARTITION BY — sizing
+
+Partitioning is for **data management**, not speed:
+
+- TTL drops expired partitions during merges.
+- `ALTER TABLE ... DROP PARTITION` removes a chunk instantly.
+- Backfills and detaches operate per partition.
+
+Budget: aim for **tens to low hundreds of active partitions**, not thousands. Each partition holds its own parts; merges never cross partitions, so over-partitioning multiplies tiny parts and starves the merge scheduler.
+
+| Volume | Partition by |
+|---|---|
+| < ~100M rows/month | `toYYYYMM(ts)` (monthly) |
+| Billions/month, time-bounded retention | `toYYYYMMDD(ts)` (daily) — only at this scale |
+| Anything | never a raw high-cardinality column, never per-hour |
+
+## Types and codecs
+
+| Column shape | Type | Codec | Why |
+|---|---|---|---|
+| Enum-like string, < ~10k distinct | `LowCardinality(String)` | (built-in dict) | Dictionary-encodes; faster GROUP BY, smaller |
+| Free-form string, high distinct | `String` | `ZSTD` | LowCardinality hurts above ~10k distinct |
+| Monotonic timestamp / counter | `DateTime64(3)` / `UInt*` | `CODEC(Delta, ZSTD)` | Delta makes deltas tiny, ZSTD packs them |
+| Float metric | `Float64` | `CODEC(ALP)` | ALP (26.3) beats Gorilla on many float series |
+| Semi-structured payload | `JSON` (native, GA 26.3) | — | Real typed paths, not a `String` blob |
+| Bounded fixed set | `Enum8` / `Enum16` | — | Stored as int, validated on insert |
+| Integer | smallest that fits (`UInt8`…`UInt64`) | — | Narrower = less I/O |
+
+`LowCardinality` threshold: under ~10k distinct values it wins; above that the dictionary overhead can cost more than it saves — fall back to plain `String` + `ZSTD`. Measure with `SELECT uniqExact(col) FROM table` on a sample.
+
+## Migrating an ORDER BY / PARTITION BY
+
+Because both are immutable, the migration is always: create a new table with the corrected key, `INSERT INTO new SELECT * FROM old` (ideally partition-by-partition to bound memory), verify counts, then `EXCHANGE TABLES new AND old` (atomic) and drop the old one.