autoscholar-cli 1.0.0

package/dist/agents/governor.js

@@ -0,0 +1,201 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GovernorEngine = void 0;
+const ERROR_PATTERNS = [
+    { pattern: /ModuleNotFoundError/i, advice: 'Install missing Python package or use alternative library' },
+    { pattern: /MemoryError/i, advice: 'Reduce dataset size, sample data, or process in chunks' },
+    { pattern: /KeyError/i, advice: 'Check column names match actual DataFrame columns' },
+    { pattern: /ValueError.*shapes/i, advice: 'Check array dimensions match for operations' },
+    { pattern: /TimeoutError/i, advice: 'Reduce API call scope or add retry logic' },
+    { pattern: /SingularMatrix/i, advice: 'Remove collinear variables or use regularization' },
+    { pattern: /ConvergenceWarning/i, advice: 'Increase max_iter or simplify model' },
+    { pattern: /empty dataframe/i, advice: 'Check API returned data; try different date range or ticker' },
+    { pattern: /all NaN/i, advice: 'Check data quality; use forward/backward fill or drop NA columns' },
+    { pattern: /zero variance/i, advice: 'Remove constant columns before regression' },
+];
+const STAGE_ORDER = {
+    gauss: 0,
+    turing: 1,
+    newton: 2,
+    fisher: 3,
+    euler_write: 4,
+    euler_compile: 5,
+    popper: 6,
+    complete: 7,
+};
+class GovernorEngine {
+    constructor() {
+        this.currentStage = 'gauss';
+        this.currentProfile = 'default';
+        this.confidence = 0.8;
+        this.decisions = [];
+        this.figureCount = 0;
+        this.wordCounts = {};
+        this.retryBudget = {
+            python: { used: 0, max: 5 },
+            data: { used: 0, max: 3 },
+            latex: { used: 0, max: 3 },
+        };
+    }
+    evaluate(event) {
+        const prevProfile = this.currentProfile;
+        const prevConfidence = this.confidence;
+        if (event.stage) {
+            this.currentStage = event.stage;
+        }
+        if (event.figuresGenerated) {
+            this.figureCount += event.figuresGenerated;
+        }
+        if (event.sectionName && event.wordCount) {
+            this.wordCounts[event.sectionName] = event.wordCount;
+        }
+        let action = { type: 'continue' };
+        let trigger = '';
+        let reasoning = '';
+        if (event.success) {
+            this.confidence = Math.min(1.0, this.confidence + 0.02);
+            if (event.stage === 'fisher' && this.currentProfile !== 'academic_figures') {
+                action = { type: 'switch_profile', profile: 'academic_figures' };
+                this.currentProfile = 'academic_figures';
+                trigger = 'Entered Fisher stage';
+                reasoning = 'Switch to academic_figures profile for publication-quality outputs';
+            }
+            else if (event.stage === 'euler_write' && this.currentProfile !== 'jf_rewrite') {
+                action = { type: 'switch_profile', profile: 'jf_rewrite' };
+                this.currentProfile = 'jf_rewrite';
+                trigger = 'Entered Euler write stage';
+                reasoning = 'Switch to Journal of Finance writing style';
+            }
+            else {
+                trigger = `${event.stage || this.currentStage} succeeded`;
+                reasoning = 'Continue normally';
+            }
+        }
+        else {
+            const output = event.output || '';
+            const matchedError = ERROR_PATTERNS.find((ep) => ep.pattern.test(output));
+            if (matchedError) {
+                this.confidence = Math.max(0, this.confidence - 0.05);
+                this.retryBudget.python.used++;
+                if (this.retryBudget.python.used >= this.retryBudget.python.max) {
+                    action = { type: 'force_finalize', reason: 'Python retry budget exhausted' };
+                    trigger = 'Python errors exceeded budget';
+                    reasoning = `${this.retryBudget.python.used} Python errors — forcing finalization`;
+                }
+                else {
+                    action = {
+                        type: 'switch_profile',
+                        profile: 'debug_recovery',
+                        directive: matchedError.advice,
+                    };
+                    this.currentProfile = 'debug_recovery';
+                    trigger = `Python error: ${matchedError.pattern.source}`;
+                    reasoning = matchedError.advice;
+                }
+            }
+            else if (/empty|no data|0 rows|NaN/i.test(output)) {
+                this.confidence = Math.max(0, this.confidence - 0.10);
+                this.retryBudget.data.used++;
+                action = {
+                    type: 'switch_profile',
+                    profile: 'conservative',
+                    directive: 'Widen date range, try alternate data source, or sample smaller subset',
+                };
+                this.currentProfile = 'conservative';
+                trigger = 'Data quality issue detected';
+                reasoning = 'Switching to conservative data retrieval approach';
+            }
+            else if (/LaTeX|pdflatex|Undefined control|Missing/i.test(output)) {
+                this.confidence = Math.max(0, this.confidence - 0.05);
+                this.retryBudget.latex.used++;
+                action = {
+                    type: 'switch_profile',
+                    profile: 'latex_fallback',
+                    directive: 'Simplify LaTeX: remove complex environments, use basic tables',
+                };
+                this.currentProfile = 'latex_fallback';
+                trigger = 'LaTeX compilation error';
+                reasoning = 'Simplifying LaTeX constructs for reliable compilation';
+            }
+            else {
+                this.confidence = Math.max(0, this.confidence - 0.03);
+                trigger = `Unknown error in ${this.currentStage}`;
+                reasoning = 'Generic failure — will retry with caution';
+            }
+        }
+        const decision = {
+            stage: this.currentStage,
+            trigger,
+            action,
+            reasoning,
+            profileBefore: prevProfile,
+            profileAfter: this.currentProfile,
+            confidenceBefore: prevConfidence,
+            confidenceAfter: this.confidence,
+        };
+        this.decisions.push(decision);
+        return decision;
+    }
+    evaluatePopper(review) {
+        const prevProfile = this.currentProfile;
+        const prevConfidence = this.confidence;
+        this.currentStage = 'popper';
+        let action;
+        let trigger;
+        let reasoning;
+        if (review.overall >= 70 && review.verdict !== 'reject') {
+            action = { type: 'continue' };
+            trigger = `Popper review: ${review.overall}/100 — ${review.verdict}`;
+            reasoning = 'Paper passes quality threshold';
+            this.confidence = Math.min(1.0, this.confidence + 0.05);
+        }
+        else if (review.overall >= 50) {
+            action = {
+                type: 'force_revision',
+                severity: 'minor',
+                issues: [`Overall score: ${review.overall}/100`, `Verdict: ${review.verdict}`],
+            };
+            trigger = `Popper review: ${review.overall}/100 — needs revision`;
+            reasoning = 'Paper needs minor revisions before publication';
+            this.confidence = Math.max(0, this.confidence - 0.15);
+        }
+        else {
+            action = {
+                type: 'block_publication',
+                reason: `Quality too low: ${review.overall}/100`,
+            };
+            trigger = `Popper review: ${review.overall}/100 — reject`;
+            reasoning = 'Paper quality below minimum threshold';
+            this.confidence = Math.max(0, this.confidence - 0.25);
+        }
+        const decision = {
+            stage: 'popper',
+            trigger,
+            action,
+            reasoning,
+            profileBefore: prevProfile,
+            profileAfter: this.currentProfile,
+            confidenceBefore: prevConfidence,
+            confidenceAfter: this.confidence,
+        };
+        this.decisions.push(decision);
+        return decision;
+    }
+    getState() {
+        return {
+            stage: this.currentStage,
+            profile: this.currentProfile,
+            confidence: this.confidence,
+            figureCount: this.figureCount,
+            wordCounts: { ...this.wordCounts },
+            retryBudget: JSON.parse(JSON.stringify(this.retryBudget)),
+        };
+    }
+    getDecisionLog() {
+        return [...this.decisions];
+    }
+    resetProfile() {
+        this.currentProfile = 'default';
+    }
+}
+exports.GovernorEngine = GovernorEngine;

package/dist/agents/newton.js

@@ -0,0 +1,207 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runNewton = runNewton;
+const ora_1 = __importDefault(require("ora"));
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const llm_1 = require("../utils/llm");
+const banner_1 = require("../cli/banner");
+const project_1 = require("../utils/project");
+const runner_1 = require("../python/runner");
+async function runNewton(topic, projectId, config, logger, dataInfo) {
+    const spinner = (0, ora_1.default)({ text: 'Newton: Designing dataset construction...', indent: 2 }).start();
+    logger.info(`[NEWTON] Starting dataset construction with ${dataInfo.datasets.length} source files`);
+    const projectDir = (0, project_1.getProjectDir)(projectId);
+    const dataDir = path.join(projectDir, 'data');
+    const dataDescription = dataInfo.datasets.map((d) => `File: ${d.filename} (${d.rows} rows)\n  Columns: ${d.columns.join(', ')}\n  Path: ${d.filePath}`).join('\n\n');
+    spinner.text = 'Newton: Planning data transformations...';
+    const plan = await (0, llm_1.callLLM)(config, `You are Newton, the dataset construction agent of AutoScholar.
+Your job is to write a Python script that:
+1. Loads all source CSV files
+2. Cleans missing values (forward fill, then drop remaining NaN)
+3. Merges data sources on date/time columns
+4. Creates derived variables (returns, log returns, rolling stats, lags)
+5. Handles outliers (winsorize at 1st/99th percentile)
+6. Constructs the final analysis-ready dataset
+7. Saves the result as final_dataset.csv
+
+IMPORTANT RULES:
+- Use only pandas, numpy, scipy (always available)
+- Read CSVs from the exact paths given
+- Save final dataset to: ${path.join(dataDir, 'final_dataset.csv')}
+- Print a JSON summary at the end with keys: rows, columns, transformations, dataDictionary
+
+Write ONLY the Python code, nothing else. Start with imports.`, `Research topic: ${topic}\n\nAvailable datasets:\n${dataDescription}`, { maxTokens: 8192 });
+    let pythonCode = plan;
+    const codeMatch = plan.match(/```python\s*\n?([\s\S]*?)```/);
+    if (codeMatch) {
+        pythonCode = codeMatch[1];
+    }
+    else if (plan.startsWith('import') || plan.startsWith('#')) {
+        pythonCode = plan;
+    }
+    (0, project_1.saveProjectFile)(projectId, 'code/newton_build_dataset.py', pythonCode);
+    spinner.text = 'Newton: Building dataset...';
+    logger.info(`[NEWTON] Executing dataset construction script`);
+    const pyResult = await (0, runner_1.runPython)(pythonCode, projectDir);
+    if (!pyResult.success) {
+        spinner.stop();
+        logger.error(`[NEWTON] Python error: ${pyResult.error}`);
+        spinner.start();
+        spinner.text = 'Newton: Retrying with simplified approach...';
+        const simplifiedCode = generateSimplifiedMerge(dataInfo.datasets, dataDir);
+        (0, project_1.saveProjectFile)(projectId, 'code/newton_build_dataset_v2.py', simplifiedCode);
+        const retryResult = await (0, runner_1.runPython)(simplifiedCode, projectDir);
+        if (!retryResult.success) {
+            spinner.stop();
+            logger.error(`[NEWTON] Retry failed: ${retryResult.error}`);
+            return {
+                success: false,
+                finalDatasetPath: '',
+                rows: 0,
+                columns: [],
+                description: 'Dataset construction failed',
+                transformations: [],
+                dataDictionary: {},
+            };
+        }
+        return parseNewtonOutput(retryResult.output, projectId, dataDir, logger, spinner);
+    }
+    return parseNewtonOutput(pyResult.output, projectId, dataDir, logger, spinner);
+}
+function parseNewtonOutput(output, projectId, dataDir, logger, spinner) {
+    spinner.stop();
+    const jsonMatch = output.match(/\{[\s\S]*"rows"[\s\S]*\}/);
+    if (jsonMatch) {
+        try {
+            const summary = JSON.parse(jsonMatch[0]);
+            const finalPath = path.join(dataDir, 'final_dataset.csv');
+            (0, banner_1.printSuccess)(`Dataset built: ${summary.rows} rows, ${(summary.columns || []).length} columns`);
+            if (summary.transformations) {
+                for (const t of summary.transformations) {
+                    (0, banner_1.printInfo)(`  Transform: ${t}`);
+                }
+            }
+            logger.info(`[NEWTON] Complete — ${summary.rows} rows, ${(summary.columns || []).length} columns`);
+            return {
+                success: true,
+                finalDatasetPath: finalPath,
+                rows: summary.rows || 0,
+                columns: summary.columns || [],
+                description: summary.description || 'Merged analysis dataset',
+                transformations: summary.transformations || [],
+                dataDictionary: summary.dataDictionary || {},
+            };
+        }
+        catch { }
+    }
+    const finalPath = path.join(dataDir, 'final_dataset.csv');
+    if (fs.existsSync(finalPath)) {
+        const content = fs.readFileSync(finalPath, 'utf-8');
+        const lines = content.split('\n').filter((l) => l.trim());
+        const headers = lines[0]?.split(',') || [];
+        const rows = Math.max(0, lines.length - 1);
+        (0, banner_1.printSuccess)(`Dataset built: ${rows} rows, ${headers.length} columns`);
+        logger.info(`[NEWTON] Complete (fallback) — ${rows} rows`);
+        return {
+            success: true,
+            finalDatasetPath: finalPath,
+            rows,
+            columns: headers,
+            description: 'Merged analysis dataset',
+            transformations: ['merge', 'clean'],
+            dataDictionary: {},
+        };
+    }
+    return {
+        success: false,
+        finalDatasetPath: '',
+        rows: 0,
+        columns: [],
+        description: 'Dataset construction failed',
+        transformations: [],
+        dataDictionary: {},
+    };
+}
+function generateSimplifiedMerge(datasets, dataDir) {
+    const loads = datasets.map((d, i) => {
+        return `df${i} = pd.read_csv("${d.filePath}")
+print(f"Loaded ${d.filename}: {df${i}.shape}")`;
+    }).join('\n');
+    return `import pandas as pd
+import numpy as np
+import json
+
+${loads}
+
+# Find common date column
+dfs = [${datasets.map((_, i) => `df${i}`).join(', ')}]
+merged = dfs[0]
+for df in dfs[1:]:
+    # Try to find common column for merge
+    common_cols = set(merged.columns) & set(df.columns)
+    date_cols = [c for c in common_cols if 'date' in c.lower() or 'time' in c.lower()]
+    if date_cols:
+        merged = pd.merge(merged, df, on=date_cols[0], how='outer', suffixes=('', '_dup'))
+    else:
+        # Concatenate if no common date
+        merged = pd.concat([merged, df], axis=1)
+
+# Drop duplicate columns
+merged = merged.loc[:, ~merged.columns.duplicated()]
+
+# Clean
+merged = merged.dropna(how='all')
+for col in merged.select_dtypes(include=[np.number]).columns:
+    merged[col] = merged[col].ffill().bfill()
+
+# Save
+output_path = "${path.join(dataDir, 'final_dataset.csv')}"
+merged.to_csv(output_path, index=False)
+
+summary = {
+    "rows": len(merged),
+    "columns": list(merged.columns),
+    "transformations": ["merge", "ffill", "bfill", "dropna"],
+    "dataDictionary": {col: str(merged[col].dtype) for col in merged.columns},
+    "description": "Merged dataset from ${datasets.length} sources"
+}
+print(json.dumps(summary))
+`;
+}