@rosetears/aili-pi 0.1.0

package/skills/minimax-xlsx/scripts/xlsx_pack.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+"""
+xlsx_pack.py — Pack a working directory back into a valid xlsx file.
+
+Usage:
+    python3 xlsx_pack.py <source_dir> <output.xlsx>
+
+Requirements:
+    - source_dir must contain [Content_Types].xml at its root
+    - All XML files are re-validated for well-formedness before packing
+
+The resulting xlsx is a valid ZIP archive with correct OOXML structure.
+"""
+
+import sys
+import os
+import zipfile
+import xml.etree.ElementTree as ET
+
+
+def validate_xml_files(source_dir: str) -> list[str]:
+    """Return list of XML files that fail to parse."""
+    bad = []
+    for dirpath, _, filenames in os.walk(source_dir):
+        for fname in filenames:
+            if fname.endswith(".xml") or fname.endswith(".rels"):
+                fpath = os.path.join(dirpath, fname)
+                try:
+                    ET.parse(fpath)
+                except ET.ParseError as e:
+                    rel = os.path.relpath(fpath, source_dir)
+                    bad.append(f"{rel}: {e}")
+    return bad
+
+
+def pack(source_dir: str, xlsx_path: str) -> None:
+    if not os.path.isdir(source_dir):
+        print(f"ERROR: Directory not found: {source_dir}", file=sys.stderr)
+        sys.exit(1)
+
+    content_types = os.path.join(source_dir, "[Content_Types].xml")
+    if not os.path.isfile(content_types):
+        print(
+            f"ERROR: Missing [Content_Types].xml in {source_dir}\n"
+            "  This file is required at the root of every valid xlsx package.",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    # Validate XML well-formedness before packing
+    print("Validating XML files...")
+    bad_files = validate_xml_files(source_dir)
+    if bad_files:
+        print("ERROR: The following files have XML parse errors:", file=sys.stderr)
+        for b in bad_files:
+            print(f"  {b}", file=sys.stderr)
+        print(
+            "\nFix all XML errors before packing. "
+            "A malformed xlsx cannot be opened by Excel or LibreOffice.",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    print("✓ All XML files are well-formed")
+
+    # Count files to pack
+    file_count = sum(len(files) for _, _, files in os.walk(source_dir))
+
+    with zipfile.ZipFile(xlsx_path, "w", compression=zipfile.ZIP_DEFLATED) as z:
+        for dirpath, _, filenames in os.walk(source_dir):
+            for fname in filenames:
+                fpath = os.path.join(dirpath, fname)
+                arcname = os.path.relpath(fpath, source_dir)
+                z.write(fpath, arcname)
+
+    size = os.path.getsize(xlsx_path)
+    print(f"Packed {file_count} files → '{xlsx_path}' ({size:,} bytes)")
+    print("\nNext step: run formula_check.py to validate formulas:")
+    print(f"  python3 formula_check.py {xlsx_path}")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 3:
+        print("Usage: xlsx_pack.py <source_dir> <output.xlsx>")
+        sys.exit(1)
+    pack(sys.argv[1], sys.argv[2])

package/skills/minimax-xlsx/scripts/xlsx_reader.py

@@ -0,0 +1,362 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+"""
+xlsx_reader.py — Structure discovery and data analysis tool for Excel/CSV files.
+
+Usage:
+    python3 xlsx_reader.py <file>                   # full structure report
+    python3 xlsx_reader.py <file> --sheet Sales     # analyze one sheet
+    python3 xlsx_reader.py <file> --json            # machine-readable output
+    python3 xlsx_reader.py <file> --quality         # data quality audit only
+
+Supports: .xlsx, .xlsm, .csv, .tsv
+Does NOT modify the source file in any way.
+
+Exit codes:
+    0 — success
+    1 — file not found / unsupported format / encoding failure
+"""
+
+import sys
+import json
+import argparse
+from pathlib import Path
+
+
+# ---------------------------------------------------------------------------
+# Format detection and loading
+# ---------------------------------------------------------------------------
+
+def detect_and_load(file_path: str, sheet_name_filter: str | None = None) -> dict:
+    """
+    Load file into {sheet_name: DataFrame} dict.
+    CSV/TSV files are mapped to a single-key dict using the file stem as key.
+
+    Raises ValueError for unsupported formats or encoding failures.
+    """
+    try:
+        import pandas as pd
+    except ImportError:
+        raise RuntimeError(
+            "pandas is not installed. Run: pip install pandas openpyxl"
+        )
+
+    path = Path(file_path)
+    if not path.exists():
+        raise FileNotFoundError(f"File not found: {file_path}")
+
+    suffix = path.suffix.lower()
+
+    if suffix in (".xlsx", ".xlsm"):
+        target = sheet_name_filter if sheet_name_filter else None
+        result = pd.read_excel(file_path, sheet_name=target)
+        # pd.read_excel with sheet_name=None returns dict; with a name, returns DataFrame
+        if isinstance(result, dict):
+            return result
+        else:
+            return {sheet_name_filter: result}
+
+    elif suffix in (".csv", ".tsv"):
+        sep = "\t" if suffix == ".tsv" else ","
+        encodings = ["utf-8-sig", "gbk", "utf-8", "latin-1"]
+        last_error = None
+        for enc in encodings:
+            try:
+                import pandas as pd
+                df = pd.read_csv(file_path, sep=sep, encoding=enc)
+                df._reader_encoding = enc  # attach metadata (non-standard, for reporting)
+                return {path.stem: df}
+            except (UnicodeDecodeError, Exception) as e:
+                last_error = e
+                continue
+        raise ValueError(
+            f"Cannot decode {file_path}. Tried encodings: {encodings}. "
+            f"Last error: {last_error}"
+        )
+
+    elif suffix == ".xls":
+        raise ValueError(
+            ".xls is a legacy binary format not supported by this tool. "
+            "Please open the file in Excel and save as .xlsx, then retry."
+        )
+
+    else:
+        raise ValueError(
+            f"Unsupported file format: {suffix}. "
+            "Supported formats: .xlsx, .xlsm, .csv, .tsv"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Structure discovery
+# ---------------------------------------------------------------------------
+
+def explore_structure(sheets: dict) -> dict:
+    """
+    Return a structured dict describing each sheet.
+    Keys: sheet_name -> {shape, columns, dtypes, null_counts, preview}
+    """
+    result = {}
+    for sheet_name, df in sheets.items():
+        null_counts = df.isnull().sum()
+        null_info = {
+            col: {"count": int(cnt), "pct": round(cnt / max(len(df), 1) * 100, 1)}
+            for col, cnt in null_counts.items()
+            if cnt > 0
+        }
+        result[sheet_name] = {
+            "shape": {"rows": df.shape[0], "cols": df.shape[1]},
+            "columns": list(df.columns),
+            "dtypes": {col: str(dtype) for col, dtype in df.dtypes.items()},
+            "null_columns": null_info,
+            "preview": df.head(5).to_dict(orient="records"),
+        }
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Data quality audit
+# ---------------------------------------------------------------------------
+
+def audit_quality(sheets: dict) -> dict:
+    """
+    Return data quality findings per sheet.
+    Checks: nulls, duplicates, mixed-type columns, potential year formatting issues.
+    """
+    import pandas as pd
+
+    findings = {}
+    for sheet_name, df in sheets.items():
+        sheet_findings = []
+
+        # Null values
+        null_counts = df.isnull().sum()
+        for col, cnt in null_counts.items():
+            if cnt > 0:
+                pct = round(cnt / max(len(df), 1) * 100, 1)
+                sheet_findings.append({
+                    "type": "null_values",
+                    "column": col,
+                    "count": int(cnt),
+                    "pct": pct,
+                    "note": f"Column '{col}' has {cnt} null values ({pct}%). "
+                            "If this column contains Excel formulas, null values may "
+                            "indicate that the formula cache has not been populated "
+                            "(file was never opened in Excel after the formulas were written)."
+                })
+
+        # Duplicate rows
+        dup_count = int(df.duplicated().sum())
+        if dup_count > 0:
+            sheet_findings.append({
+                "type": "duplicate_rows",
+                "count": dup_count,
+                "note": f"{dup_count} fully duplicate rows found."
+            })
+
+        # Mixed-type object columns (numeric data stored as text)
+        for col in df.select_dtypes(include="object").columns:
+            numeric_converted = pd.to_numeric(df[col], errors="coerce")
+            convertible = int(numeric_converted.notna().sum())
+            non_null_total = int(df[col].notna().sum())
+            if 0 < convertible < non_null_total:
+                sheet_findings.append({
+                    "type": "mixed_type",
+                    "column": col,
+                    "convertible_to_numeric": convertible,
+                    "non_convertible": non_null_total - convertible,
+                    "note": f"Column '{col}' appears to contain mixed types: "
+                            f"{convertible} values can be parsed as numbers, "
+                            f"{non_null_total - convertible} cannot. "
+                            "Use pd.to_numeric(df[col], errors='coerce') to unify."
+                })
+
+        # Year column formatting (e.g., 2024.0 stored as float)
+        for col in df.select_dtypes(include="number").columns:
+            col_lower = str(col).lower()
+            # "年" is the Chinese character for "year" — detect year columns in CJK spreadsheets
+            if "year" in col_lower or "yr" in col_lower or "年" in col_lower:
+                if df[col].dropna().between(1900, 2200).all():
+                    if df[col].dtype == float:
+                        sheet_findings.append({
+                            "type": "year_as_float",
+                            "column": col,
+                            "note": f"Column '{col}' appears to be a year column stored as float "
+                                    "(e.g., 2024.0). Convert with df[col].astype(int).astype(str) "
+                                    "to get clean year strings like '2024'."
+                        })
+
+        # Outliers via IQR on numeric columns
+        for col in df.select_dtypes(include="number").columns:
+            series = df[col].dropna()
+            if len(series) < 4:
+                continue
+            Q1, Q3 = series.quantile(0.25), series.quantile(0.75)
+            IQR = Q3 - Q1
+            if IQR == 0:
+                continue
+            outlier_mask = (df[col] < Q1 - 1.5 * IQR) | (df[col] > Q3 + 1.5 * IQR)
+            outlier_count = int(outlier_mask.sum())
+            if outlier_count > 0:
+                sheet_findings.append({
+                    "type": "outliers_iqr",
+                    "column": col,
+                    "count": outlier_count,
+                    "note": f"Column '{col}' has {outlier_count} potential outlier(s) "
+                            f"(outside 1.5×IQR bounds: [{Q1 - 1.5*IQR:.2f}, {Q3 + 1.5*IQR:.2f}])."
+                })
+
+        findings[sheet_name] = sheet_findings
+
+    return findings
+
+
+# ---------------------------------------------------------------------------
+# Summary statistics
+# ---------------------------------------------------------------------------
+
+def compute_stats(sheets: dict) -> dict:
+    """Compute descriptive statistics for numeric columns per sheet."""
+    stats = {}
+    for sheet_name, df in sheets.items():
+        numeric_df = df.select_dtypes(include="number")
+        if numeric_df.empty:
+            stats[sheet_name] = {}
+            continue
+        desc = numeric_df.describe().round(4)
+        stats[sheet_name] = desc.to_dict()
+    return stats
+
+
+# ---------------------------------------------------------------------------
+# Human-readable report rendering
+# ---------------------------------------------------------------------------
+
+def render_report(
+    file_path: str,
+    structure: dict,
+    quality: dict,
+    stats: dict,
+) -> str:
+    lines = []
+    p = lines.append
+
+    p("=" * 60)
+    p(f"ANALYSIS REPORT: {Path(file_path).name}")
+    p("=" * 60)
+
+    # File overview
+    sheet_list = list(structure.keys())
+    total_rows = sum(s["shape"]["rows"] for s in structure.values())
+    p(f"\nSheets ({len(sheet_list)}): {', '.join(sheet_list)}")
+    p(f"Total rows across all sheets: {total_rows:,}")
+
+    for sheet_name, info in structure.items():
+        p(f"\n{'─' * 50}")
+        p(f"Sheet: {sheet_name}")
+        p(f"{'─' * 50}")
+        p(f"  Size: {info['shape']['rows']:,} rows × {info['shape']['cols']} cols")
+        p(f"  Columns: {info['columns']}")
+
+        # Data types
+        p("\n  Column types:")
+        for col, dtype in info["dtypes"].items():
+            p(f"    {col}: {dtype}")
+
+        # Nulls
+        if info["null_columns"]:
+            p("\n  Null values (columns with nulls only):")
+            for col, null_info in info["null_columns"].items():
+                p(f"    {col}: {null_info['count']} nulls ({null_info['pct']}%)")
+        else:
+            p("\n  Null values: none")
+
+        # Stats
+        sheet_stats = stats.get(sheet_name, {})
+        if sheet_stats:
+            p("\n  Numeric column statistics:")
+            numeric_cols = list(sheet_stats.keys())
+            # Show only first 6 to keep report readable
+            for col in numeric_cols[:6]:
+                col_stats = sheet_stats[col]
+                p(f"    {col}:")
+                p(f"      count={col_stats.get('count', 'N/A')}  "
+                  f"mean={col_stats.get('mean', 'N/A')}  "
+                  f"min={col_stats.get('min', 'N/A')}  "
+                  f"max={col_stats.get('max', 'N/A')}")
+            if len(numeric_cols) > 6:
+                p(f"    ... and {len(numeric_cols) - 6} more numeric columns")
+
+        # Quality findings for this sheet
+        sheet_quality = quality.get(sheet_name, [])
+        if sheet_quality:
+            p(f"\n  Data quality issues ({len(sheet_quality)} found):")
+            for finding in sheet_quality:
+                p(f"    [{finding['type'].upper()}] {finding['note']}")
+        else:
+            p("\n  Data quality: no issues found")
+
+        # Preview
+        if info["preview"]:
+            p("\n  Preview (first 3 rows):")
+            import pandas as pd
+            preview_df = pd.DataFrame(info["preview"][:3])
+            for line in preview_df.to_string(index=False).splitlines():
+                p(f"    {line}")
+
+    p("\n" + "=" * 60)
+    quality_issue_count = sum(len(v) for v in quality.values())
+    if quality_issue_count == 0:
+        p("RESULT: No data quality issues detected.")
+    else:
+        p(f"RESULT: {quality_issue_count} data quality issue(s) found. See details above.")
+    p("=" * 60)
+
+    return "\n".join(lines)
+
+
+# ---------------------------------------------------------------------------
+# CLI entry point
+# ---------------------------------------------------------------------------
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Read and analyze Excel/CSV files without modifying them."
+    )
+    parser.add_argument("file", help="Path to .xlsx, .xlsm, .csv, or .tsv file")
+    parser.add_argument("--sheet", help="Analyze a specific sheet only", default=None)
+    parser.add_argument(
+        "--json", action="store_true", help="Output machine-readable JSON"
+    )
+    parser.add_argument(
+        "--quality", action="store_true",
+        help="Run data quality audit only (skip stats)"
+    )
+    args = parser.parse_args()
+
+    try:
+        sheets = detect_and_load(args.file, sheet_name_filter=args.sheet)
+    except (FileNotFoundError, ValueError, RuntimeError) as e:
+        print(f"ERROR: {e}", file=sys.stderr)
+        sys.exit(1)
+
+    structure = explore_structure(sheets)
+    quality = audit_quality(sheets)
+    stats = {} if args.quality else compute_stats(sheets)
+
+    if args.json:
+        output = {
+            "file": args.file,
+            "structure": structure,
+            "quality": quality,
+            "stats": stats,
+        }
+        # Convert preview records to serializable form (handle non-JSON types)
+        print(json.dumps(output, indent=2, ensure_ascii=False, default=str))
+    else:
+        report = render_report(args.file, structure, quality, stats)
+        print(report)
+
+
+if __name__ == "__main__":
+    main()