@lark-apaas/coding-miaoda-sandbox-skills 0.1.0-dev.28c4f05 → 0.1.0-dev.4e64c13

package/miaoda/data-analysis/references/json-output-specification.md

@@ -0,0 +1,277 @@
+# JSON Output Specification
+
+## Complete Schema
+
+```json
+{
+  "analysis_id": "string (UUID v4)",
+  "timestamp": "string (ISO 8601 format)",
+  "metadata": {
+    "data_source": "string (file path or data description)",
+    "row_count": "integer",
+    "column_count": "integer",
+    "analysis_type": "exploratory | confirmatory | diagnostic | descriptive",
+    "business_question": "string (user's core question, optional)"
+  },
+  "data_quality": {
+    "completeness": "number (0-1, non-missing ratio)",
+    "missing_values": {
+      "column_name": "integer (missing count)"
+    },
+    "duplicates": "integer (duplicate row count)",
+    "outliers": {
+      "column_name": "integer (outlier count)"
+    },
+    "issues": ["string (issue description)"]
+  },
+  "statistics": {
+    "numeric_summary": {
+      "column_name": {
+        "count": "integer",
+        "mean": "number",
+        "std": "number",
+        "min": "number",
+        "max": "number",
+        "quartiles": ["number (Q1)", "number (Q2/median)", "number (Q3)"],
+        "skew": "number (optional)",
+        "kurtosis": "number (optional)"
+      }
+    },
+    "categorical_summary": {
+      "column_name": {
+        "unique_count": "integer",
+        "top_values": [
+          {"value": "string", "count": "integer", "percentage": "number"}
+        ],
+        "mode": "string"
+      }
+    },
+    "correlations": [
+      {
+        "column1": "string",
+        "column2": "string",
+        "coefficient": "number",
+        "method": "pearson | spearman"
+      }
+    ]
+  },
+  "insights": [
+    {
+      "id": "string (insight_1, insight_2, ...)",
+      "type": "correlation | trend | anomaly | distribution | comparison",
+      "title": "string (short title, within 10 characters)",
+      "description": "string (detailed description)",
+      "evidence": {
+        "metric": "string (metric name)",
+        "value": "number | string",
+        "p_value": "number (optional, for statistical tests)"
+      },
+      "significance": "high | medium | low",
+      "affected_columns": ["string"]
+    }
+  ],
+  "recommendations": [
+    {
+      "id": "string (rec_1, rec_2, ...)",
+      "type": "visualization | further_analysis | data_collection | action",
+      "title": "string (recommendation title)",
+      "description": "string (detailed recommendation)",
+      "priority": "high | medium | low"
+    }
+  ],
+  "visualizations": [
+    {
+      "id": "string (viz_1, viz_2, ...)",
+      "type": "histogram | scatter | line | bar | heatmap | box | pie",
+      "title": "string (chart title)",
+      "config": {
+        "x": "string (X-axis field name)",
+        "y": "string (Y-axis field name, optional)",
+        "group_by": "string (grouping field, optional)",
+        "aggregation": "string (aggregation method: sum/mean/count, optional)"
+      },
+      "insight_refs": ["string (associated insight id)"]
+    }
+  ]
+}
+```
+
+## CRITICAL: JSON Validity Rules
+
+Python `json.dumps()` defaults to `allow_nan=True`, which outputs `NaN` and `Infinity` as bare tokens — **these are NOT valid JSON** and will cause TypeScript build failures (TS1328).
+
+| Python value | json.dumps output | Valid JSON? | Fix |
+|-------------|-------------------|-------------|-----|
+| `float('nan')` | `NaN` | **NO** | Replace with `null` |
+| `float('inf')` | `Infinity` | **NO** | Replace with `null` or max value |
+| `float('-inf')` | `-Infinity` | **NO** | Replace with `null` or min value |
+| `None` | `null` | Yes | OK |
+
+**MUST** do one of:
+1. Call `json.dump(..., allow_nan=False)` (raises error if NaN exists, forcing explicit handling)
+2. Sanitize all float values before dumping: replace `NaN`/`Infinity` with `None`
+
+Common sources of NaN in pandas: `df.mean()` on empty column, `df.std()` on single value, `df.skew()` on constant data, division by zero.
+
+## Field Description
+
+| Field Path | Required | Description |
+| ---------- | -------- | ----------- |
+| `analysis_id` | Yes | Unique identifier for analysis task, use UUID v4 |
+| `timestamp` | Yes | Analysis completion time, ISO 8601 format |
+| `metadata.*` | Yes | Data source metadata |
+| `data_quality.*` | Yes | Data quality assessment results |
+| `statistics.numeric_summary` | Conditional | Required when numeric columns exist |
+| `statistics.categorical_summary` | Conditional | Required when categorical columns exist |
+| `statistics.correlations` | No | Only populate when significant correlations found |
+| `insights` | Yes | At least 1 insight required |
+| `recommendations` | Yes | At least 1 recommendation required |
+| `visualizations` | Yes | At least 1 visualization config required |
+
+## Example Output
+
+```json
+{
+  "analysis_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890",
+  "timestamp": "2024-01-15T10:30:00Z",
+  "metadata": {
+    "data_source": "/data/sales_2023.csv",
+    "row_count": 10000,
+    "column_count": 12,
+    "analysis_type": "exploratory",
+    "business_question": "了解2023年销售数据的整体情况"
+  },
+  "data_quality": {
+    "completeness": 0.95,
+    "missing_values": {
+      "customer_age": 320,
+      "region": 180
+    },
+    "duplicates": 45,
+    "outliers": {
+      "order_amount": 23
+    },
+    "issues": [
+      "customer_age 列有 3.2% 缺失值",
+      "order_amount 存在 23 个异常高值（超出 IQR 上界）"
+    ]
+  },
+  "statistics": {
+    "numeric_summary": {
+      "order_amount": {
+        "count": 10000,
+        "mean": 156.78,
+        "std": 89.34,
+        "min": 5.00,
+        "max": 2500.00,
+        "quartiles": [85.00, 135.00, 210.00],
+        "skew": 2.34,
+        "kurtosis": 8.91
+      },
+      "customer_age": {
+        "count": 9680,
+        "mean": 35.6,
+        "std": 12.3,
+        "min": 18,
+        "max": 75,
+        "quartiles": [26, 34, 44]
+      }
+    },
+    "categorical_summary": {
+      "product_category": {
+        "unique_count": 5,
+        "top_values": [
+          {"value": "Electronics", "count": 3500, "percentage": 35.0},
+          {"value": "Clothing", "count": 2800, "percentage": 28.0},
+          {"value": "Home", "count": 2000, "percentage": 20.0}
+        ],
+        "mode": "Electronics"
+      }
+    },
+    "correlations": [
+      {
+        "column1": "order_amount",
+        "column2": "customer_age",
+        "coefficient": 0.72,
+        "method": "pearson"
+      }
+    ]
+  },
+  "insights": [
+    {
+      "id": "insight_1",
+      "type": "correlation",
+      "title": "年龄与消费正相关",
+      "description": "客户年龄与订单金额呈较强正相关（r=0.72），年长客户倾向于更高消费。",
+      "evidence": {
+        "metric": "pearson_correlation",
+        "value": 0.72,
+        "p_value": 0.001
+      },
+      "significance": "high",
+      "affected_columns": ["customer_age", "order_amount"]
+    },
+    {
+      "id": "insight_2",
+      "type": "distribution",
+      "title": "订单金额右偏分布",
+      "description": "订单金额呈明显右偏（skew=2.34），大部分订单集中在低价区间，少量高价订单拉高均值。",
+      "evidence": {
+        "metric": "skewness",
+        "value": 2.34
+      },
+      "significance": "medium",
+      "affected_columns": ["order_amount"]
+    }
+  ],
+  "recommendations": [
+    {
+      "id": "rec_1",
+      "type": "further_analysis",
+      "title": "深入分析高价值客户",
+      "description": "建议对年龄>40且订单金额>500的客户群体进行细分分析，挖掘其消费特征。",
+      "priority": "high"
+    },
+    {
+      "id": "rec_2",
+      "type": "visualization",
+      "title": "绘制年龄-消费散点图",
+      "description": "使用散点图可视化年龄与订单金额的关系，并按产品类别着色。",
+      "priority": "medium"
+    }
+  ],
+  "visualizations": [
+    {
+      "id": "viz_1",
+      "type": "scatter",
+      "title": "客户年龄 vs 订单金额",
+      "config": {
+        "x": "customer_age",
+        "y": "order_amount",
+        "group_by": "product_category"
+      },
+      "insight_refs": ["insight_1"]
+    },
+    {
+      "id": "viz_2",
+      "type": "histogram",
+      "title": "订单金额分布",
+      "config": {
+        "x": "order_amount"
+      },
+      "insight_refs": ["insight_2"]
+    },
+    {
+      "id": "viz_3",
+      "type": "bar",
+      "title": "各产品类别销售额",
+      "config": {
+        "x": "product_category",
+        "y": "order_amount",
+        "aggregation": "sum"
+      },
+      "insight_refs": []
+    }
+  ]
+}
+```

package/miaoda/data-analysis/references/post-analysis-guide.md

@@ -0,0 +1,77 @@
+# Post-Analysis Guide
+
+Steps to complete after analysis is done: update AGENTS.md and report output locations to user.
+
+## 1. Update AGENTS.md
+
+**CRITICAL**: After completing analysis, you **MUST** update AGENTS.md. This is NOT optional.
+
+### Steps
+
+1. Check if `AGENTS.md` exists in the project root; if not, create it
+2. Append the analysis record to `AGENTS.md` following the format below exactly
+
+> **NOTE**: Before writing code that consumes analysis data, always read the corresponding type definition in `shared/static/types.ts` first to ensure correct field access.
+
+### AGENTS.md Format (MUST follow exactly)
+
+> ### [Date] Analysis of [filename]
+> - **Source file**: `path/to/source/data.csv`
+> - **Analysis type**: exploratory | confirmatory | diagnostic | descriptive
+> - **Key findings**: Brief summary of main insights (1-2 sentences)
+> - **Output location**: `tmp/output/filename_analysis.json`
+> - **Copied to**: `shared/static/filename_analysis.json` (if applicable)
+> - **Type definition**: `shared/static/types.ts` → `FilenameAnalysis`
+>
+> #### Consumption
+>
+> ```typescript
+> import type { FilenameAnalysis } from '@shared/static/types';
+> import rawData from '@shared/static/filename_analysis.json';
+>
+> const data = rawData as FilenameAnalysis;
+> // access fields: data.insights, data.statistics, data.visualizations, etc.
+> ```
+
+### Example Entry
+
+> ### 2024-01-15 Analysis of sales_2023.csv
+> - **Source file**: `data/sales_2023.csv`
+> - **Analysis type**: exploratory
+> - **Key findings**: Strong correlation between customer age and order amount (r=0.72). Order amounts show right-skewed distribution.
+> - **Output location**: `tmp/output/sales_2023_exploratory.json`
+> - **Copied to**: `shared/static/sales_2023_exploratory.json`
+> - **Type definition**: `shared/static/types.ts` → `Sales2023Exploratory`
+>
+> #### Consumption
+>
+> ```typescript
+> import type { Sales2023Exploratory } from '@shared/static/types';
+> import rawData from '@shared/static/sales_2023_exploratory.json';
+>
+> const data = rawData as Sales2023Exploratory;
+> const highInsights = data.insights.filter(i => i.significance === 'high');
+> ```
+
+## 2. Report Output Locations
+
+After completing analysis, report to user:
+
+```text
+Analysis complete! Artifacts saved to:
+
+📁 tmp/
+├── output/
+│   ├── sales_2023_exploratory.json      # Structured analysis result
+│   └── visualizations/
+│       ├── sales_2023_age_vs_amount.png # Scatter plot
+│       └── sales_2023_amount_dist.png   # Histogram
+└── intermediate/
+    └── sales_2023_cleaned.csv           # Cleaned data (if applicable)
+
+To use in your code, copy the JSON to shared/static/:
+  mkdir -p shared/static/ && cp tmp/output/sales_2023_exploratory.json shared/static/
+
+Then import directly in your code:
+  import analysisData from '@shared/static/sales_2023_exploratory.json'
+```

package/miaoda/data-analysis/references/python-analysis-reference.md

@@ -0,0 +1,272 @@
+# Python Code Standards & Templates
+
+## Recommended Libraries
+
+```python
+import pandas as pd        # Data processing
+import numpy as np         # Numerical computation
+from scipy import stats    # Statistical analysis
+from datetime import datetime
+import json
+import uuid
+```
+
+## Phase-Specific Code Templates
+
+### Phase 2: Data Understanding
+
+```python
+import pandas as pd
+
+df = pd.read_csv('data.csv')  # or pd.read_excel(), pd.read_json()
+
+print(f"Shape: {df.shape}")
+print(f"Columns: {df.columns.tolist()}")
+print(f"Dtypes:\n{df.dtypes}")
+print(f"Head:\n{df.head()}")
+```
+
+### Phase 3: Data Quality Assessment
+
+```python
+# Missing values
+missing = df.isnull().sum()
+missing_pct = (missing / len(df) * 100).round(2)
+print(f"Missing values:\n{missing[missing > 0]}")
+
+# Duplicate rows
+duplicates = df.duplicated().sum()
+print(f"Duplicate rows: {duplicates}")
+
+# Numeric outliers (IQR)
+def detect_outliers_iqr(series):
+    Q1, Q3 = series.quantile([0.25, 0.75])
+    IQR = Q3 - Q1
+    lower, upper = Q1 - 1.5 * IQR, Q3 + 1.5 * IQR
+    return ((series < lower) | (series > upper)).sum()
+
+for col in df.select_dtypes(include='number').columns:
+    outliers = detect_outliers_iqr(df[col])
+    if outliers > 0:
+        print(f"{col}: {outliers} outliers")
+```
+
+### Phase 4: Descriptive Analysis
+
+```python
+# 4.1 Numeric variables
+numeric_stats = df.describe(percentiles=[0.25, 0.5, 0.75]).T
+numeric_stats['skew'] = df.select_dtypes(include='number').skew()
+numeric_stats['kurtosis'] = df.select_dtypes(include='number').kurtosis()
+
+# 4.2 Categorical variables
+for col in df.select_dtypes(include='object').columns:
+    print(f"\n{col}:")
+    print(f"  Unique: {df[col].nunique()}")
+    print(f"  Top values:\n{df[col].value_counts().head()}")
+
+# 4.3 Correlation analysis (Pearson)
+import numpy as np
+correlation_matrix = df.select_dtypes(include='number').corr(method='pearson')
+corr_pairs = []
+for i in range(len(correlation_matrix.columns)):
+    for j in range(i+1, len(correlation_matrix.columns)):
+        r = correlation_matrix.iloc[i, j]
+        if abs(r) > 0.7:
+            corr_pairs.append({
+                'col1': correlation_matrix.columns[i],
+                'col2': correlation_matrix.columns[j],
+                'correlation': round(r, 3)
+            })
+
+# 4.4 Group aggregation
+grouped = df.groupby('category_column').agg({
+    'numeric_col1': ['mean', 'std', 'count'],
+    'numeric_col2': ['mean', 'std', 'count']
+})
+```
+
+## Complete Analysis Template
+
+```python
+import pandas as pd
+import numpy as np
+from scipy import stats
+import json
+import uuid
+from datetime import datetime
+from pathlib import Path
+
+
+def setup_tmp_dirs(project_root: Path = None) -> dict:
+    """Initialize tmp directory structure for analysis artifacts."""
+    if project_root is None:
+        project_root = Path.cwd()
+
+    tmp_dir = project_root / "tmp"
+    paths = {
+        "root": tmp_dir,
+        "scripts": tmp_dir / "analysis_scripts",
+        "intermediate": tmp_dir / "intermediate",
+        "output": tmp_dir / "output",
+        "visualizations": tmp_dir / "output" / "visualizations",
+    }
+
+    for path in paths.values():
+        path.mkdir(parents=True, exist_ok=True)
+
+    return paths
+
+
+def analyze_data(file_path: str, analysis_type: str = "exploratory",
+                 project_root: Path = None) -> dict:
+    """
+    Execute complete data analysis workflow.
+
+    Args:
+        file_path: Data file path
+        analysis_type: exploratory/confirmatory/diagnostic/descriptive
+        project_root: Project root for tmp directory. Defaults to cwd.
+
+    Returns:
+        Analysis result JSON conforming to specification
+    """
+    # 0. Setup tmp directories
+    tmp_paths = setup_tmp_dirs(project_root)
+
+    # 1. Load data
+    if file_path.endswith('.csv'):
+        df = pd.read_csv(file_path)
+    elif file_path.endswith(('.xls', '.xlsx')):
+        df = pd.read_excel(file_path)
+    elif file_path.endswith('.json'):
+        df = pd.read_json(file_path)
+    else:
+        raise ValueError(f"Unsupported file format: {file_path}")
+
+    # 2. Initialize result structure
+    result = {
+        "analysis_id": str(uuid.uuid4()),
+        "timestamp": datetime.utcnow().isoformat() + "Z",
+        "metadata": {
+            "data_source": file_path,
+            "row_count": len(df),
+            "column_count": len(df.columns),
+            "analysis_type": analysis_type
+        },
+        "data_quality": {},
+        "statistics": {},
+        "insights": [],
+        "recommendations": [],
+        "visualizations": []
+    }
+
+    # 3. Data quality assessment
+    missing = df.isnull().sum()
+    result["data_quality"] = {
+        "completeness": round(1 - df.isnull().sum().sum() / df.size, 4),
+        "missing_values": {k: int(v) for k, v in missing[missing > 0].items()},
+        "duplicates": int(df.duplicated().sum()),
+        "outliers": {},
+        "issues": []
+    }
+
+    for col in df.select_dtypes(include='number').columns:
+        Q1, Q3 = df[col].quantile([0.25, 0.75])
+        IQR = Q3 - Q1
+        outlier_count = int(((df[col] < Q1 - 1.5 * IQR) | (df[col] > Q3 + 1.5 * IQR)).sum())
+        if outlier_count > 0:
+            result["data_quality"]["outliers"][col] = outlier_count
+
+    # 4. Descriptive statistics - Numeric
+    numeric_cols = df.select_dtypes(include='number').columns
+    if len(numeric_cols) > 0:
+        result["statistics"]["numeric_summary"] = {}
+        for col in numeric_cols:
+            series = df[col].dropna()
+            result["statistics"]["numeric_summary"][col] = {
+                "count": int(series.count()),
+                "mean": round(float(series.mean()), 4),
+                "std": round(float(series.std()), 4),
+                "min": round(float(series.min()), 4),
+                "max": round(float(series.max()), 4),
+                "quartiles": [
+                    round(float(series.quantile(0.25)), 4),
+                    round(float(series.quantile(0.50)), 4),
+                    round(float(series.quantile(0.75)), 4)
+                ],
+                "skew": round(float(series.skew()), 4),
+                "kurtosis": round(float(series.kurtosis()), 4)
+            }
+
+    # Categorical
+    categorical_cols = df.select_dtypes(include=['object', 'category']).columns
+    if len(categorical_cols) > 0:
+        result["statistics"]["categorical_summary"] = {}
+        for col in categorical_cols:
+            value_counts = df[col].value_counts()
+            total = len(df[col].dropna())
+            result["statistics"]["categorical_summary"][col] = {
+                "unique_count": int(df[col].nunique()),
+                "top_values": [
+                    {
+                        "value": str(idx),
+                        "count": int(cnt),
+                        "percentage": round(cnt / total * 100, 2)
+                    }
+                    for idx, cnt in value_counts.head(5).items()
+                ],
+                "mode": str(value_counts.index[0]) if len(value_counts) > 0 else None
+            }
+
+    # 5. Correlation analysis
+    if len(numeric_cols) > 1:
+        corr_matrix = df[numeric_cols].corr()
+        correlations = []
+        for i in range(len(numeric_cols)):
+            for j in range(i + 1, len(numeric_cols)):
+                r = corr_matrix.iloc[i, j]
+                if abs(r) > 0.5:
+                    correlations.append({
+                        "column1": numeric_cols[i],
+                        "column2": numeric_cols[j],
+                        "coefficient": round(float(r), 4),
+                        "method": "pearson"
+                    })
+        if correlations:
+            result["statistics"]["correlations"] = correlations
+
+    # 6. Save output (MUST sanitize NaN/Infinity before dumping)
+    base_name = Path(file_path).stem
+    output_file = tmp_paths["output"] / f"{base_name}_{analysis_type}.json"
+
+    def sanitize_for_json(obj):
+        """Replace NaN/Infinity with None to produce valid JSON."""
+        if isinstance(obj, float) and (np.isnan(obj) or np.isinf(obj)):
+            return None
+        if isinstance(obj, dict):
+            return {k: sanitize_for_json(v) for k, v in obj.items()}
+        if isinstance(obj, list):
+            return [sanitize_for_json(v) for v in obj]
+        return obj
+
+    result = sanitize_for_json(result)
+    with open(output_file, 'w', encoding='utf-8') as f:
+        json.dump(result, f, indent=2, ensure_ascii=False, allow_nan=False)
+    print(f"Analysis result saved to: {output_file}")
+
+    return result
+
+
+def save_visualization(fig, viz_id: str, tmp_paths: dict, format: str = "png"):
+    """Save matplotlib/seaborn figure to tmp/output/visualizations."""
+    output_path = tmp_paths["visualizations"] / f"{viz_id}.{format}"
+    fig.savefig(output_path, dpi=150, bbox_inches='tight')
+    print(f"Visualization saved to: {output_path}")
+    return output_path
+
+
+# Usage:
+# result = analyze_data("sales_2023.csv", "exploratory")
+# print(json.dumps(result, indent=2, ensure_ascii=False))
+```