ctxora 6.2.2 → 6.3.0

package/src/harness_context/skills/ecc/clickhouse-io/SKILL.md

@@ -0,0 +1,445 @@
+---
+name: clickhouse-io
+description: ClickHouse database patterns, query optimization, analytics, and data engineering best practices for high-performance analytical workloads. Use when writing ClickHouse schemas or queries, or when an analytical query is too slow.
+metadata:
+  origin: ECC
+---
+
+# ClickHouse Analytics Patterns
+
+ClickHouse-specific patterns for high-performance analytics and data engineering.
+
+## When to Activate
+
+- Designing ClickHouse table schemas (MergeTree engine selection)
+- Writing analytical queries (aggregations, window functions, joins)
+- Optimizing query performance (partition pruning, projections, materialized views)
+- Ingesting large volumes of data (batch inserts, Kafka integration)
+- Migrating from PostgreSQL/MySQL to ClickHouse for analytics
+- Implementing real-time dashboards or time-series analytics
+
+## Overview
+
+ClickHouse is a column-oriented database management system (DBMS) for online analytical processing (OLAP). It's optimized for fast analytical queries on large datasets.
+
+**Key Features:**
+- Column-oriented storage
+- Data compression
+- Parallel query execution
+- Distributed queries
+- Real-time analytics
+
+## Table Design Patterns
+
+### MergeTree Engine (Most Common)
+
+```sql
+CREATE TABLE markets_analytics (
+    date Date,
+    market_id String,
+    market_name String,
+    volume UInt64,
+    trades UInt32,
+    unique_traders UInt32,
+    avg_trade_size Float64,
+    created_at DateTime
+) ENGINE = MergeTree()
+PARTITION BY toYYYYMM(date)
+ORDER BY (date, market_id)
+SETTINGS index_granularity = 8192;
+```
+
+### ReplacingMergeTree (Deduplication)
+
+```sql
+-- For data that may have duplicates (e.g., from multiple sources)
+CREATE TABLE user_events (
+    event_id String,
+    user_id String,
+    event_type String,
+    timestamp DateTime,
+    properties String
+) ENGINE = ReplacingMergeTree()
+PARTITION BY toYYYYMM(timestamp)
+ORDER BY (user_id, event_id, timestamp)
+PRIMARY KEY (user_id, event_id);
+```
+
+### AggregatingMergeTree (Pre-aggregation)
+
+```sql
+-- For maintaining aggregated metrics
+CREATE TABLE market_stats_hourly (
+    hour DateTime,
+    market_id String,
+    total_volume AggregateFunction(sum, UInt64),
+    total_trades AggregateFunction(count, UInt32),
+    unique_users AggregateFunction(uniq, String)
+) ENGINE = AggregatingMergeTree()
+PARTITION BY toYYYYMM(hour)
+ORDER BY (hour, market_id);
+
+-- Query aggregated data
+SELECT
+    hour,
+    market_id,
+    sumMerge(total_volume) AS volume,
+    countMerge(total_trades) AS trades,
+    uniqMerge(unique_users) AS users
+FROM market_stats_hourly
+WHERE hour >= toStartOfHour(now() - INTERVAL 24 HOUR)
+GROUP BY hour, market_id
+ORDER BY hour DESC;
+```
+
+## Query Optimization Patterns
+
+### Efficient Filtering
+
+```sql
+-- PASS: GOOD: Use indexed columns first
+SELECT *
+FROM markets_analytics
+WHERE date >= '2025-01-01'
+  AND market_id = 'market-123'
+  AND volume > 1000
+ORDER BY date DESC
+LIMIT 100;
+
+-- FAIL: BAD: Filter on non-indexed columns first
+SELECT *
+FROM markets_analytics
+WHERE volume > 1000
+  AND market_name LIKE '%election%'
+  AND date >= '2025-01-01';
+```
+
+### Aggregations
+
+```sql
+-- PASS: GOOD: Use ClickHouse-specific aggregation functions
+SELECT
+    toStartOfDay(created_at) AS day,
+    market_id,
+    sum(volume) AS total_volume,
+    count() AS total_trades,
+    uniq(trader_id) AS unique_traders,
+    avg(trade_size) AS avg_size
+FROM trades
+WHERE created_at >= today() - INTERVAL 7 DAY
+GROUP BY day, market_id
+ORDER BY day DESC, total_volume DESC;
+
+-- PASS: Use quantile for percentiles (more efficient than percentile)
+SELECT
+    quantile(0.50)(trade_size) AS median,
+    quantile(0.95)(trade_size) AS p95,
+    quantile(0.99)(trade_size) AS p99
+FROM trades
+WHERE created_at >= now() - INTERVAL 1 HOUR;
+```
+
+### Window Functions
+
+```sql
+-- Calculate running totals
+SELECT
+    date,
+    market_id,
+    volume,
+    sum(volume) OVER (
+        PARTITION BY market_id
+        ORDER BY date
+        ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+    ) AS cumulative_volume
+FROM markets_analytics
+WHERE date >= today() - INTERVAL 30 DAY
+ORDER BY market_id, date;
+```
+
+## Data Insertion Patterns
+
+### Bulk Insert (Recommended)
+
+```typescript
+import { createClient } from '@clickhouse/client'
+
+const clickhouse = createClient({
+  url: process.env.CLICKHOUSE_URL ?? 'http://localhost:8123',
+  username: process.env.CLICKHOUSE_USER,
+  password: process.env.CLICKHOUSE_PASSWORD
+})
+
+// PASS: Batch insert (efficient)
+async function bulkInsertTrades(trades: Trade[]) {
+  await clickhouse.insert({
+    table: 'trades',
+    values: trades.map(trade => ({
+      id: trade.id,
+      market_id: trade.market_id,
+      user_id: trade.user_id,
+      amount: trade.amount,
+      timestamp: trade.timestamp.toISOString()
+    })),
+    format: 'JSONEachRow'
+  })
+}
+
+// FAIL: Individual inserts (slow)
+async function insertTrade(trade: Trade) {
+  // Don't do this in a loop!
+  await clickhouse.insert({
+    table: 'trades',
+    values: [{
+      id: trade.id,
+      market_id: trade.market_id,
+      user_id: trade.user_id,
+      amount: trade.amount,
+      timestamp: trade.timestamp.toISOString()
+    }],
+    format: 'JSONEachRow'
+  })
+}
+```
+
+### Streaming Insert
+
+```typescript
+// For continuous data ingestion
+import { Readable } from 'node:stream'
+
+async function streamInserts(dataSource: AsyncIterable<Record<string, unknown>>) {
+  await clickhouse.insert({
+    table: 'trades',
+    values: Readable.from(dataSource, { objectMode: true }),
+    format: 'JSONEachRow'
+  })
+}
+```
+
+## Materialized Views
+
+### Real-time Aggregations
+
+```sql
+-- Create materialized view for hourly stats
+CREATE MATERIALIZED VIEW market_stats_hourly_mv
+TO market_stats_hourly
+AS SELECT
+    toStartOfHour(timestamp) AS hour,
+    market_id,
+    sumState(amount) AS total_volume,
+    countState() AS total_trades,
+    uniqState(user_id) AS unique_users
+FROM trades
+GROUP BY hour, market_id;
+
+-- Query the materialized view
+SELECT
+    hour,
+    market_id,
+    sumMerge(total_volume) AS volume,
+    countMerge(total_trades) AS trades,
+    uniqMerge(unique_users) AS users
+FROM market_stats_hourly
+WHERE hour >= now() - INTERVAL 24 HOUR
+GROUP BY hour, market_id;
+```
+
+## Performance Monitoring
+
+### Query Performance
+
+```sql
+-- Check slow queries
+SELECT
+    query_id,
+    user,
+    query,
+    query_duration_ms,
+    read_rows,
+    read_bytes,
+    memory_usage
+FROM system.query_log
+WHERE type = 'QueryFinish'
+  AND query_duration_ms > 1000
+  AND event_time >= now() - INTERVAL 1 HOUR
+ORDER BY query_duration_ms DESC
+LIMIT 10;
+```
+
+### Table Statistics
+
+```sql
+-- Check table sizes
+SELECT
+    database,
+    table,
+    formatReadableSize(sum(bytes)) AS size,
+    sum(rows) AS rows,
+    max(modification_time) AS latest_modification
+FROM system.parts
+WHERE active
+GROUP BY database, table
+ORDER BY sum(bytes) DESC;
+```
+
+## Common Analytics Queries
+
+### Time Series Analysis
+
+```sql
+-- Daily active users
+SELECT
+    toDate(timestamp) AS date,
+    uniq(user_id) AS daily_active_users
+FROM events
+WHERE timestamp >= today() - INTERVAL 30 DAY
+GROUP BY date
+ORDER BY date;
+
+-- Retention analysis
+SELECT
+    signup_date,
+    countIf(days_since_signup = 0) AS day_0,
+    countIf(days_since_signup = 1) AS day_1,
+    countIf(days_since_signup = 7) AS day_7,
+    countIf(days_since_signup = 30) AS day_30
+FROM (
+    SELECT
+        user_id,
+        min(toDate(timestamp)) AS signup_date,
+        toDate(timestamp) AS activity_date,
+        dateDiff('day', signup_date, activity_date) AS days_since_signup
+    FROM events
+    GROUP BY user_id, activity_date
+)
+GROUP BY signup_date
+ORDER BY signup_date DESC;
+```
+
+### Funnel Analysis
+
+```sql
+-- Conversion funnel
+SELECT
+    countIf(step = 'viewed_market') AS viewed,
+    countIf(step = 'clicked_trade') AS clicked,
+    countIf(step = 'completed_trade') AS completed,
+    round(clicked / viewed * 100, 2) AS view_to_click_rate,
+    round(completed / clicked * 100, 2) AS click_to_completion_rate
+FROM (
+    SELECT
+        user_id,
+        session_id,
+        event_type AS step
+    FROM events
+    WHERE event_date = today()
+)
+GROUP BY session_id;
+```
+
+### Cohort Analysis
+
+```sql
+-- User cohorts by signup month
+SELECT
+    toStartOfMonth(signup_date) AS cohort,
+    toStartOfMonth(activity_date) AS month,
+    dateDiff('month', cohort, month) AS months_since_signup,
+    count(DISTINCT user_id) AS active_users
+FROM (
+    SELECT
+        user_id,
+        min(toDate(timestamp)) OVER (PARTITION BY user_id) AS signup_date,
+        toDate(timestamp) AS activity_date
+    FROM events
+)
+GROUP BY cohort, month, months_since_signup
+ORDER BY cohort, months_since_signup;
+```
+
+## Data Pipeline Patterns
+
+### ETL Pattern
+
+```typescript
+// Extract, Transform, Load
+async function etlPipeline() {
+  // 1. Extract from source
+  const rawData = await extractFromPostgres()
+
+  // 2. Transform
+  const transformed = rawData.map(row => ({
+    date: new Date(row.created_at).toISOString().split('T')[0],
+    market_id: row.market_slug,
+    volume: parseFloat(row.total_volume),
+    trades: parseInt(row.trade_count)
+  }))
+
+  // 3. Load to ClickHouse
+  await bulkInsertToClickHouse(transformed)
+}
+
+// Run periodically
+setInterval(etlPipeline, 60 * 60 * 1000)  // Every hour
+```
+
+### Change Data Capture (CDC)
+
+```typescript
+// Listen to PostgreSQL changes and sync to ClickHouse
+import { Client } from 'pg'
+
+const pgClient = new Client({ connectionString: process.env.DATABASE_URL })
+
+pgClient.query('LISTEN market_updates')
+
+pgClient.on('notification', async (msg) => {
+  const update = JSON.parse(msg.payload)
+
+  await clickhouse.insert({
+    table: 'market_updates',
+    values: [
+      {
+        market_id: update.id,
+        event_type: update.operation,  // INSERT, UPDATE, DELETE
+        timestamp: new Date(),
+        data: JSON.stringify(update.new_data)
+      }
+    ],
+    format: 'JSONEachRow'
+  })
+})
+```
+
+## Best Practices
+
+### 1. Partitioning Strategy
+- Partition by time (usually month or day)
+- Avoid too many partitions (performance impact)
+- Use DATE type for partition key
+
+### 2. Ordering Key
+- Put most frequently filtered columns first
+- Consider cardinality (high cardinality first)
+- Order impacts compression
+
+### 3. Data Types
+- Use smallest appropriate type (UInt32 vs UInt64)
+- Use LowCardinality for repeated strings
+- Use Enum for categorical data
+
+### 4. Avoid
+- SELECT * (specify columns)
+- FINAL (merge data before query instead)
+- Too many JOINs (denormalize for analytics)
+- Small frequent inserts (batch instead)
+
+### 5. Monitoring
+- Track query performance
+- Monitor disk usage
+- Check merge operations
+- Review slow query log
+
+**Remember**: ClickHouse excels at analytical workloads. Design tables for your query patterns, batch inserts, and leverage materialized views for real-time aggregations.

package/src/harness_context/skills/ecc/code-tour/LICENSE.ecc

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Affaan Mustafa
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/src/harness_context/skills/ecc/code-tour/SKILL.md

@@ -0,0 +1,254 @@
+---
+name: code-tour
+description: Create CodeTour `.tour` files — persona-targeted, step-by-step walkthroughs with real file and line anchors. Use for onboarding tours, architecture walkthroughs, PR tours, RCA tours, and structured "explain how this works" requests. Use when the user asks for a code tour, onboarding walkthrough, PR tour, or an explanation of how a subsystem works.
+metadata:
+  origin: ECC
+---
+
+# Code Tour
+
+Create **CodeTour** `.tour` files for codebase walkthroughs that open directly to real files and line ranges. Tours live in `.tours/` and are meant for the CodeTour format, not ad hoc Markdown notes.
+
+A good tour is a narrative for a specific reader:
+- what they are looking at
+- why it matters
+- what path they should follow next
+
+Only create `.tour` JSON files. Do not modify source code as part of this skill.
+
+## When to Use
+
+Use this skill when:
+- the user asks for a code tour, onboarding tour, architecture walkthrough, or PR tour
+- the user says "explain how X works" and wants a reusable guided artifact
+- the user wants a ramp-up path for a new engineer or reviewer
+- the task is better served by a guided sequence than a flat summary
+
+Examples:
+- onboarding a new maintainer
+- architecture tour for one service or package
+- PR-review walk-through anchored to changed files
+- RCA tour showing the failure path
+- security review tour of trust boundaries and key checks
+
+## When NOT to Use
+
+| Instead of code-tour | Use |
+| --- | --- |
+| A one-off explanation in chat is enough | answer directly |
+| The user wants prose docs, not a `.tour` artifact | `documentation-lookup` or repo docs editing |
+| The task is implementation or refactoring | do the implementation work |
+| The task is broad codebase onboarding without a tour artifact | `codebase-onboarding` |
+
+## Workflow
+
+### 1. Discover
+
+Explore the repo before writing anything:
+- README and package/app entry points
+- folder structure
+- relevant config files
+- the changed files if the tour is PR-focused
+
+Do not start writing steps before you understand the shape of the code.
+
+### 2. Infer the reader
+
+Decide the persona and depth from the request.
+
+| Request shape | Persona | Suggested depth |
+| --- | --- | --- |
+| "onboarding", "new joiner" | `new-joiner` | 9-13 steps |
+| "quick tour", "vibe check" | `vibecoder` | 5-8 steps |
+| "architecture" | `architect` | 14-18 steps |
+| "tour this PR" | `pr-reviewer` | 7-11 steps |
+| "why did this break" | `rca-investigator` | 7-11 steps |
+| "security review" | `security-reviewer` | 7-11 steps |
+| "explain how this feature works" | `feature-explainer` | 7-11 steps |
+| "debug this path" | `bug-fixer` | 7-11 steps |
+
+### 3. Read and verify anchors
+
+Every file path and line anchor must be real:
+- confirm the file exists
+- confirm the line numbers are in range
+- if using a selection, verify the exact block
+- if the file is volatile, prefer a pattern-based anchor
+
+Never guess line numbers.
+
+### 4. Write the `.tour`
+
+Write to:
+
+```text
+.tours/<persona>-<focus>.tour
+```
+
+Keep the path deterministic and readable.
+
+### 5. Validate
+
+Before finishing:
+- every referenced path exists
+- every line or selection is valid
+- the first step is anchored to a real file or directory
+- the `ref` points at a branch or commit that actually has every file the tour references (see below)
+- the tour tells a coherent story rather than listing files
+
+## The `ref` Field
+
+`ref` ties the tour to a git branch or commit. It matters more than it looks: when `ref` is not the branch the reader has checked out, CodeTour opens each step's file from that revision in git, not from the files on disk. If a file is not in that revision, the step will not open — the reader sees *"The editor could not be opened because the file was not found"* even though the file is sitting right there. The tour and its comments still show, so the real cause is easy to miss.
+
+Pick `ref` by tour type:
+
+| Tour type | Set `ref` to |
+| --- | --- |
+| PR tour | the PR branch — never the base branch |
+| Onboarding / architecture | the branch the reader will be on (often `main`), or leave it out |
+| Not sure | leave `ref` out, so CodeTour reads files straight from disk |
+
+The PR case is the common trap: a PR usually adds new files, and new files do not exist on the base branch yet. Point `ref` at the base (e.g. `develop`) and every step on a new file fails to open.
+
+Before finishing, confirm each step's file actually exists at the `ref` you chose.
+
+## Step Types
+
+### Content
+
+Use sparingly, usually only for a closing step:
+
+```json
+{ "title": "Next Steps", "description": "You can now trace the request path end to end." }
+```
+
+Do not make the first step content-only.
+
+### Directory
+
+Use to orient the reader to a module:
+
+```json
+{ "directory": "src/services", "title": "Service Layer", "description": "The core orchestration logic lives here." }
+```
+
+### File + line
+
+This is the default step type:
+
+```json
+{ "file": "src/auth/middleware.ts", "line": 42, "title": "Auth Gate", "description": "Every protected request passes here first." }
+```
+
+### Selection
+
+Use when one code block matters more than the whole file:
+
+```json
+{
+  "file": "src/core/pipeline.ts",
+  "selection": {
+    "start": { "line": 15, "character": 0 },
+    "end": { "line": 34, "character": 0 }
+  },
+  "title": "Request Pipeline",
+  "description": "This block wires validation, auth, and downstream execution."
+}
+```
+
+### Pattern
+
+Use when exact lines may drift:
+
+```json
+{ "file": "src/app.ts", "pattern": "export default class App", "title": "Application Entry" }
+```
+
+### URI
+
+Use for PRs, issues, or docs when helpful:
+
+```json
+{ "uri": "https://github.com/org/repo/pull/456", "title": "The PR" }
+```
+
+## Writing Rule: SMIG
+
+Each description should answer:
+- **Situation**: what the reader is looking at
+- **Mechanism**: how it works
+- **Implication**: why it matters for this persona
+- **Gotcha**: what a smart reader might miss
+
+Keep descriptions compact, specific, and grounded in the actual code.
+
+## Narrative Shape
+
+Use this arc unless the task clearly needs something different:
+1. orientation
+2. module map
+3. core execution path
+4. edge case or gotcha
+5. closing / next move
+
+The tour should feel like a path, not an inventory.
+
+## Example
+
+```json
+{
+  "$schema": "https://aka.ms/codetour-schema",
+  "title": "API Service Tour",
+  "description": "Walkthrough of the request path for the payments service.",
+  "ref": "main",
+  "steps": [
+    {
+      "directory": "src",
+      "title": "Source Root",
+      "description": "All runtime code for the service starts here."
+    },
+    {
+      "file": "src/server.ts",
+      "line": 12,
+      "title": "Entry Point",
+      "description": "The server boots here and wires middleware before any route is reached."
+    },
+    {
+      "file": "src/routes/payments.ts",
+      "line": 8,
+      "title": "Payment Routes",
+      "description": "Every payments request enters through this router before hitting service logic."
+    },
+    {
+      "title": "Next Steps",
+      "description": "You can now follow any payment request end to end with the main anchors in place."
+    }
+  ]
+}
+```
+
+## Anti-Patterns
+
+| Anti-pattern | Fix |
+| --- | --- |
+| Flat file listing | Tell a story with dependency between steps |
+| Generic descriptions | Name the concrete code path or pattern |
+| Guessed anchors | Verify every file and line first |
+| Too many steps for a quick tour | Cut aggressively |
+| First step is content-only | Anchor the first step to a real file or directory |
+| Persona mismatch | Write for the actual reader, not a generic engineer |
+
+## Best Practices
+
+- keep step count proportional to repo size and persona depth
+- use directory steps for orientation, file steps for substance
+- for PR tours, cover changed files first
+- for monorepos, scope to the relevant packages instead of touring everything
+- close with what the reader can now do, not a recap
+
+## Related Skills
+
+- `codebase-onboarding`
+- `coding-standards`
+- `council`
+- official upstream format: `microsoft/codetour`