@aslomon/effectum 0.3.4 → 0.5.0

package/bin/lib/tool-loader.js

@@ -0,0 +1,243 @@
+/**
+ * Dynamic tool loader — loads JSON-based tool definitions from system/tools/,
+ * merges foundation + stack + community definitions, and deduplicates by key.
+ *
+ * New stacks require only a new JSON file in system/tools/ — zero code changes.
+ * Community/local overrides are loaded from .effectum/tools/ and ~/.effectum/tools/.
+ */
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+const os = require("os");
+
+// ─── JSON loading helpers ────────────────────────────────────────────────────
+
+/**
+ * Load tools from a JSON file. Returns empty array if file doesn't exist or is invalid.
+ * @param {string} filePath - absolute path to JSON file
+ * @returns {Array<object>}
+ */
+function loadJsonTools(filePath) {
+  try {
+    if (!fs.existsSync(filePath)) return [];
+    const raw = fs.readFileSync(filePath, "utf8");
+    const parsed = JSON.parse(raw);
+    if (Array.isArray(parsed.tools)) return parsed.tools;
+    if (Array.isArray(parsed)) return parsed;
+    return [];
+  } catch (_) {
+    return [];
+  }
+}
+
+/**
+ * Load all JSON files from a directory.
+ * Skips files starting with _ (e.g., _schema.json).
+ * @param {string} dirPath - directory to scan
+ * @returns {Array<object>}
+ */
+function loadToolsFromDir(dirPath) {
+  const tools = [];
+  try {
+    if (!fs.existsSync(dirPath)) return tools;
+    const files = fs
+      .readdirSync(dirPath)
+      .filter((f) => f.endsWith(".json") && !f.startsWith("_"));
+    for (const file of files) {
+      tools.push(...loadJsonTools(path.join(dirPath, file)));
+    }
+  } catch (_) {
+    // Directory doesn't exist or isn't readable
+  }
+  return tools;
+}
+
+// ─── Tool resolution ─────────────────────────────────────────────────────────
+
+/**
+ * Find the system/tools/ directory relative to this module (which lives in bin/lib/).
+ * @returns {string}
+ */
+function getSystemToolsDir() {
+  return path.resolve(__dirname, "..", "..", "system", "tools");
+}
+
+/**
+ * Get the JSON filename for a stack key.
+ * @param {string} stack - e.g., "nextjs-supabase"
+ * @returns {string} - e.g., "nextjs-supabase.json"
+ */
+function stackToFilename(stack) {
+  return `${stack}.json`;
+}
+
+// ─── System basics (pre-config) ──────────────────────────────────────────────
+
+/**
+ * Get system-level basics that must be checked before any configuration.
+ * These are Homebrew (macOS), Git, Node.js, and Claude Code.
+ * @returns {Array<object>}
+ */
+function getSystemBasics() {
+  const platform = os.platform() === "darwin" ? "darwin" : "linux";
+  const basics = [];
+
+  // Homebrew (macOS only)
+  if (platform === "darwin") {
+    basics.push({
+      key: "brew",
+      bin: "brew",
+      displayName: "Homebrew",
+      category: "system",
+      why: "Package manager for macOS — needed to install other tools",
+      priority: 0,
+      autoInstall: true,
+      install: {
+        darwin:
+          '/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"',
+      },
+      check: "brew --version",
+    });
+  }
+
+  // Git
+  basics.push({
+    key: "git",
+    bin: "git",
+    displayName: "Git",
+    category: "system",
+    why: "Version control — required for all projects",
+    priority: 0,
+    autoInstall: true,
+    install: {
+      darwin: "xcode-select --install",
+      linux: "sudo apt install -y git",
+    },
+    check: "git --version",
+  });
+
+  // Node.js
+  basics.push({
+    key: "node",
+    bin: "node",
+    displayName: "Node.js",
+    category: "system",
+    why: "JavaScript runtime — required for Claude Code and npm tools",
+    priority: 0,
+    autoInstall: true,
+    install: {
+      darwin: "brew install node",
+      linux: "sudo apt install -y nodejs npm",
+    },
+    check: "node --version",
+  });
+
+  // Claude Code
+  basics.push({
+    key: "claude",
+    bin: "claude",
+    displayName: "Claude Code",
+    category: "system",
+    why: "AI coding agent — the core of the autonomous workflow",
+    priority: 0,
+    autoInstall: true,
+    install: {
+      all: "npm i -g @anthropic-ai/claude-code",
+    },
+    check: "claude --version",
+  });
+
+  return basics;
+}
+
+// ─── Main loader ─────────────────────────────────────────────────────────────
+
+/**
+ * Load and merge tool definitions for a given stack.
+ *
+ * Merge order (last wins for duplicate keys):
+ *   1. foundation.json (always)
+ *   2. stack-specific.json (if exists)
+ *   3. Community: <targetDir>/.effectum/tools/*.json
+ *   4. Community: ~/.effectum/tools/*.json
+ *
+ * @param {string} stack - stack key (e.g., "nextjs-supabase", "generic")
+ * @param {string} [targetDir] - project directory for local community tools
+ * @returns {Array<object>} - deduplicated, priority-sorted tool list
+ */
+function loadToolDefinitions(stack, targetDir) {
+  const systemDir = getSystemToolsDir();
+  const tools = [];
+
+  // 1. Foundation (always loaded)
+  const foundationPath = path.join(systemDir, "foundation.json");
+  tools.push(...loadJsonTools(foundationPath));
+
+  // 2. Stack-specific
+  if (stack && stack !== "foundation") {
+    const stackPath = path.join(systemDir, stackToFilename(stack));
+    tools.push(...loadJsonTools(stackPath));
+  }
+
+  // 3. Community: local project overrides
+  if (targetDir) {
+    const localToolsDir = path.join(targetDir, ".effectum", "tools");
+    tools.push(...loadToolsFromDir(localToolsDir));
+  }
+
+  // 4. Community: global user overrides
+  const globalToolsDir = path.join(os.homedir(), ".effectum", "tools");
+  tools.push(...loadToolsFromDir(globalToolsDir));
+
+  return deduplicateByKey(tools);
+}
+
+// ─── Deduplication ───────────────────────────────────────────────────────────
+
+/**
+ * Deduplicate tools by key. Last occurrence wins (community overrides bundled).
+ * Result is sorted by priority (ascending).
+ * @param {Array<object>} tools
+ * @returns {Array<object>}
+ */
+function deduplicateByKey(tools) {
+  const map = new Map();
+  for (const tool of tools) {
+    map.set(tool.key, tool);
+  }
+  return Array.from(map.values()).sort(
+    (a, b) => (a.priority ?? 5) - (b.priority ?? 5),
+  );
+}
+
+// ─── List available stacks ───────────────────────────────────────────────────
+
+/**
+ * List all available stack JSON files (excluding foundation and _schema).
+ * @returns {Array<string>} - stack keys (e.g., ["nextjs-supabase", "python-fastapi"])
+ */
+function listAvailableStacks() {
+  const systemDir = getSystemToolsDir();
+  try {
+    return fs
+      .readdirSync(systemDir)
+      .filter(
+        (f) =>
+          f.endsWith(".json") && !f.startsWith("_") && f !== "foundation.json",
+      )
+      .map((f) => f.replace(".json", ""));
+  } catch (_) {
+    return [];
+  }
+}
+
+module.exports = {
+  loadJsonTools,
+  loadToolsFromDir,
+  getSystemToolsDir,
+  getSystemBasics,
+  loadToolDefinitions,
+  deduplicateByKey,
+  listAvailableStacks,
+};

package/bin/lib/ui.js

@@ -19,6 +19,17 @@ const {
   getAllMcps,
   getAllSubagents,
 } = require("./recommendation");
+const {
+  checkAllTools,
+  checkSystemBasics,
+  formatToolStatus,
+  formatInstallInstructions,
+  formatInstallPlan,
+  formatAuthStatus,
+  installTool,
+  categorizeForInstall,
+  checkAllAuth,
+} = require("./cli-tools");
 
 /** @type {import("@clack/prompts")} */
 let p;
@@ -495,6 +506,162 @@ async function askGitBranch() {
   return { create: true, name };
 }
 
+// ─── System Basics Check (Phase 1) ──────────────────────────────────────────
+
+/**
+ * Check system basics (Homebrew, Git, Node.js, Claude Code) before config.
+ * Offers to install missing basics.
+ * @returns {Promise<void>}
+ */
+async function showSystemCheck() {
+  const result = checkSystemBasics();
+
+  p.note(formatToolStatus(result.tools), "System Basics");
+
+  if (result.missing.length === 0) {
+    p.log.success("All system basics are installed.");
+    return;
+  }
+
+  p.log.warn(`${result.missing.length} system tool(s) not found.`);
+
+  for (const tool of result.missing) {
+    const name = tool.displayName || tool.key;
+    const wantInstall = await p.confirm({
+      message: `Install ${name}? (${tool.why})`,
+      initialValue: true,
+    });
+    handleCancel(wantInstall);
+
+    if (wantInstall) {
+      const s = p.spinner();
+      s.start(`Installing ${name}...`);
+      const installResult = installTool(tool);
+      if (installResult.ok) {
+        tool.installed = true;
+        s.stop(`${name} installed`);
+      } else {
+        s.stop(`${name} failed: ${installResult.error || "unknown error"}`);
+        p.log.warn(`You can install ${name} manually later.`);
+      }
+    }
+  }
+}
+
+// ─── Consolidated Installation Plan (Phase 3) ──────────────────────────────
+
+/**
+ * Show consolidated installation plan for stack tools and install with one confirmation.
+ * @param {string} stack - selected stack key
+ * @param {string} [targetDir] - project directory for community overrides
+ * @returns {Promise<{ tools: Array<object>, missing: Array<object>, installed: Array<object> }>}
+ */
+async function showInstallPlan(stack, targetDir) {
+  const result = checkAllTools(stack, targetDir);
+
+  p.note(formatToolStatus(result.tools), "Stack Tools");
+
+  if (result.missing.length === 0) {
+    p.log.success("All stack tools are installed.");
+    return result;
+  }
+
+  // Categorize missing tools
+  const plan = categorizeForInstall(result.tools);
+
+  // Show the consolidated plan
+  p.note(formatInstallPlan(plan), "Installation Plan");
+
+  // Auto-install with one confirmation
+  if (plan.autoInstall.length > 0) {
+    const names = plan.autoInstall
+      .map((t) => t.displayName || t.key)
+      .join(", ");
+    const confirm = await p.confirm({
+      message: `Install ${plan.autoInstall.length} tool(s)? (${names})`,
+      initialValue: true,
+    });
+    handleCancel(confirm);
+
+    if (confirm) {
+      for (const tool of plan.autoInstall) {
+        const name = tool.displayName || tool.key;
+        const s = p.spinner();
+        s.start(`Installing ${name}...`);
+        const installResult = installTool(tool);
+        if (installResult.ok) {
+          // Update in result too
+          const match = result.tools.find((t) => t.key === tool.key);
+          if (match) match.installed = true;
+          s.stop(`${name} installed`);
+        } else {
+          s.stop(`${name} failed: ${installResult.error || "unknown error"}`);
+        }
+      }
+    }
+  }
+
+  if (plan.manual.length > 0) {
+    p.log.info("Manual setup required for the tools listed above.");
+  }
+
+  return result;
+}
+
+// ─── Auth Flow (Phase 4) ────────────────────────────────────────────────────
+
+/**
+ * Check auth status for installed tools and guide through authentication.
+ * @param {Array<object>} tools - tools with `installed` status
+ * @returns {Promise<void>}
+ */
+async function showAuthCheck(tools) {
+  const authResults = checkAllAuth(tools);
+
+  if (authResults.length === 0) return;
+
+  p.note(formatAuthStatus(authResults), "Auth Status");
+
+  const unauthenticated = authResults.filter((t) => !t.authenticated);
+
+  if (unauthenticated.length === 0) {
+    p.log.success("All tools are authenticated.");
+    return;
+  }
+
+  p.log.warn(`${unauthenticated.length} tool(s) need authentication.`);
+
+  const runAuth = await p.confirm({
+    message: "Show auth commands for unauthenticated tools?",
+    initialValue: true,
+  });
+  handleCancel(runAuth);
+
+  if (runAuth) {
+    const lines = unauthenticated.map((t) => {
+      const name = t.displayName || t.key;
+      let line = `  ${name}: ${t.authSetupCmd}`;
+      if (t.authUrl) line += `\n    Token: ${t.authUrl}`;
+      return line;
+    });
+    p.note(lines.join("\n"), "Run these commands manually");
+  }
+}
+
+// ─── Legacy CLI Tool Check (kept for backward compat) ───────────────────────
+
+/**
+ * Run CLI tool check and offer installation/auth for missing tools.
+ * @deprecated Use showSystemCheck + showInstallPlan + showAuthCheck instead.
+ * @param {string} stack - selected stack key
+ * @returns {Promise<{ tools: Array<object>, missing: Array<object>, installed: Array<object> }>}
+ */
+async function showCliToolCheck(stack) {
+  const result = await showInstallPlan(stack);
+  await showAuthCheck(result.tools);
+  return result;
+}
+
 // ─── Display helpers ────────────────────────────────────────────────────────
 
 /**
@@ -561,6 +728,11 @@ module.exports = {
   askSetupMode,
   askCustomize,
   askManual,
+  // Tool check flow
+  showSystemCheck,
+  showInstallPlan,
+  showAuthCheck,
+  showCliToolCheck,
   // Legacy / utility prompts
   askMcpServers,
   askPlaywright,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aslomon/effectum",
-  "version": "0.3.4",
+  "version": "0.5.0",
   "description": "Autonomous development system for Claude Code \u2014 describe what you want, get production-ready code",
   "bin": {
     "effectum": "bin/effectum.js"

package/system/agents/data-engineer.md

@@ -0,0 +1,268 @@
+---
+name: data-engineer
+description: "Use this agent when building ETL pipelines, data models, data warehouses, or data-intensive applications. Invoke for SQL optimization, pandas/polars data processing, Spark jobs, schema design, data validation, and data quality engineering."
+tools: Read, Write, Edit, Bash, Glob, Grep
+model: sonnet
+---
+
+You are a senior data engineer specializing in building reliable, scalable data pipelines and data infrastructure. Your expertise spans ETL/ELT design, data modeling, SQL optimization, and modern data stack tools with deep knowledge of data quality, governance, and performance tuning.
+
+When invoked:
+
+1. Query context manager for existing data architecture and pipeline patterns
+2. Review data sources, transformations, and destination schemas
+3. Analyze data volume, velocity, and quality requirements
+4. Design following data engineering best practices and patterns
+
+Data engineering checklist:
+
+- Data sources identified and cataloged
+- Schema design normalized appropriately
+- Pipeline idempotency guaranteed
+- Data validation rules defined
+- Error handling and dead letter queues
+- Monitoring and alerting configured
+- Data lineage documented
+- SLA requirements met
+
+SQL optimization:
+
+- Query execution plan analysis
+- Index strategy design
+- Partition pruning
+- Join optimization
+- CTE vs subquery decisions
+- Window function patterns
+- Materialized view usage
+- Query parallelization
+
+Data modeling:
+
+- Dimensional modeling (star/snowflake)
+- Data vault methodology
+- Slowly changing dimensions
+- Fact table design
+- Surrogate key strategies
+- Temporal data patterns
+- Multi-tenant data isolation
+- Schema evolution management
+
+ETL/ELT pipeline design:
+
+- Incremental extraction patterns
+- Change data capture (CDC)
+- Idempotent transformations
+- Pipeline orchestration (Airflow, Dagster, Prefect)
+- Backfill strategies
+- Dependency management
+- Error recovery and retry logic
+- Pipeline monitoring
+
+Python data processing:
+
+- pandas optimization patterns
+- polars for large datasets
+- Dask for distributed processing
+- Memory-efficient transformations
+- Chunked processing for large files
+- Type-safe data operations
+- Serialization formats (Parquet, Arrow)
+- Data validation with Pandera/Great Expectations
+
+Apache Spark:
+
+- SparkSQL optimization
+- DataFrame vs RDD usage
+- Partition strategy
+- Shuffle optimization
+- Broadcast joins
+- Caching and persistence
+- Dynamic resource allocation
+- Structured Streaming
+
+Data validation:
+
+- Schema validation
+- Data type enforcement
+- Null handling policies
+- Referential integrity checks
+- Business rule validation
+- Statistical anomaly detection
+- Data freshness monitoring
+- Cross-source reconciliation
+
+Schema design:
+
+- PostgreSQL schema patterns
+- Migration strategy (forward-only)
+- Index design principles
+- Constraint enforcement
+- Enum vs lookup tables
+- JSON/JSONB column usage
+- Array and composite types
+- Full-text search configuration
+
+Data quality engineering:
+
+- Data profiling
+- Quality metrics and KPIs
+- Automated quality checks
+- Data observability
+- Anomaly detection
+- Root cause analysis
+- Quality dashboards
+- SLA tracking
+
+Performance tuning:
+
+- Batch vs streaming trade-offs
+- Compression strategies
+- Partitioning schemes
+- Connection pooling
+- Query optimization
+- Parallel processing
+- Caching layers
+- Resource allocation
+
+## Communication Protocol
+
+### Data Architecture Assessment
+
+Initialize data engineering by understanding the data landscape.
+
+Architecture context request:
+
+```json
+{
+  "requesting_agent": "data-engineer",
+  "request_type": "get_data_context",
+  "payload": {
+    "query": "Data engineering context needed: data sources, volume/velocity, transformation requirements, target schemas, quality requirements, SLA expectations, and existing pipeline infrastructure."
+  }
+}
+```
+
+## Development Workflow
+
+Execute data engineering through systematic phases:
+
+### 1. Data Discovery
+
+Understand data sources, volumes, and requirements.
+
+Discovery framework:
+
+- Source system inventory
+- Data volume assessment
+- Update frequency analysis
+- Schema documentation
+- Quality baseline measurement
+- Dependency mapping
+- SLA requirements gathering
+- Security classification
+
+Data assessment:
+
+- Source connectivity testing
+- Sample data profiling
+- Schema inference
+- Volume estimation
+- Quality scoring
+- Latency measurement
+- Format identification
+- Access pattern analysis
+
+### 2. Implementation Phase
+
+Build reliable data pipelines with proper error handling.
+
+Implementation approach:
+
+- Schema design and migration
+- Extraction logic development
+- Transformation pipeline coding
+- Loading and upsert patterns
+- Validation rule implementation
+- Error handling setup
+- Monitoring integration
+- Documentation generation
+
+Pipeline patterns:
+
+- Extract → Validate → Transform → Load
+- Idempotent operations
+- Checkpoint and resume
+- Dead letter queue for failures
+- Audit trail logging
+- Schema evolution handling
+- Backfill capability
+- Incremental processing
+
+Progress reporting:
+
+```json
+{
+  "agent": "data-engineer",
+  "status": "building",
+  "pipeline_progress": {
+    "sources_connected": 5,
+    "transformations": 12,
+    "tables_created": 8,
+    "validation_rules": 24,
+    "test_coverage": "85%"
+  }
+}
+```
+
+### 3. Quality and Operations
+
+Ensure data quality and operational excellence.
+
+Quality checklist:
+
+- All pipelines idempotent
+- Validation rules comprehensive
+- Error handling tested
+- Monitoring dashboards live
+- Alerting configured
+- Documentation complete
+- Runbooks created
+- Performance benchmarks met
+
+Delivery notification:
+"Data engineering completed. Built 5-source ETL pipeline processing 2M records/day with 99.9% reliability. Schema includes 8 tables with proper indexing, partitioning, and RLS policies. Data quality checks cover 24 validation rules with automated alerting. Pipeline is idempotent with full backfill capability."
+
+Testing strategies:
+
+- Unit tests for transformations
+- Integration tests for pipelines
+- Data quality assertions
+- Schema migration tests
+- Performance regression tests
+- Edge case validation
+- Idempotency verification
+- End-to-end pipeline tests
+
+Operational patterns:
+
+- Pipeline scheduling
+- Failure alerting
+- Automatic retries
+- Data reconciliation
+- Capacity planning
+- Cost optimization
+- Access control
+- Audit logging
+
+Integration with other agents:
+
+- Collaborate with postgres-pro on database optimization
+- Work with backend-developer on API data contracts
+- Coordinate with security-engineer on data access policies
+- Partner with performance-engineer on query optimization
+- Consult devops-engineer on pipeline infrastructure
+- Sync with api-designer on data API design
+- Align with debugger on pipeline failure diagnosis
+- Engage test-automator on data testing strategy
+
+Always prioritize data reliability, pipeline idempotency, schema integrity, and operational excellence while building scalable data infrastructure that meets SLA requirements.