jupytermind 0.3.0

package/src/ai_data_scientist/data_definition.py

@@ -0,0 +1,100 @@
+"""Data-definition and provenance manifest with per-field confidence status.
+
+Implements DES-AIDS-040 (REQ-AIDS-052): structurally distinguishes
+reproducible file identity (owner/slug, SHA-256, retrieval time) from
+verified semantic metadata (units, definitions, measurement pathway),
+making data-definition uncertainty visible and queryable instead of
+silently promoting an inferred value to "verified".
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+_VALID_STATUSES = frozenset({"verified", "inferred", "reported", "unknown"})
+
+
+# @id CODE-AIDS-071
+# @implements REQ-AIDS-052
+# @design DES-AIDS-040
+@dataclass(frozen=True)
+class FieldValue:
+    """A single semantic field paired with its confidence status.
+
+    Frozen so a constructed instance's status cannot be mutated in place;
+    changing a field's confidence always requires building a brand-new
+    ``FieldValue``, which is always an explicit caller action rather than an
+    automatic promotion from "inferred"/"unknown" to "verified".
+    """
+
+    value: object
+    status: str
+    source: str | None = None
+
+    def __post_init__(self) -> None:
+        if self.status not in _VALID_STATUSES:
+            raise ValueError(
+                f"status must be one of {sorted(_VALID_STATUSES)}, got {self.status!r}."
+            )
+
+
+# @id CODE-AIDS-083
+# @implements REQ-AIDS-052
+# @design DES-AIDS-040
+@dataclass(frozen=True)
+class DataDefinitionManifest:
+    """Aggregated data-definition manifest for one ingested dataset."""
+
+    source: dict
+    dataset_scope: dict
+    variables: dict[str, dict[str, FieldValue]]
+    transformations: tuple = field(default_factory=tuple)
+
+    def unresolved_fields(self) -> list[tuple[str, FieldValue]]:
+        """Return every ``FieldValue`` across the manifest whose status is "unknown".
+
+        Each entry's first element is a dotted path identifying its location
+        (e.g. ``"source.license"`` or ``"variables.value.unit"``).
+        """
+        return self._fields_with_status("unknown")
+
+    def inferred_fields(self) -> list[tuple[str, FieldValue]]:
+        """Return every ``FieldValue`` across the manifest whose status is "inferred".
+
+        GitHub #33: surfaces unconfirmed-but-assumed fields as their own
+        actionable, listed item, separate from (and never overlapping
+        with) :meth:`unresolved_fields`'s "unknown" list.
+        """
+        return self._fields_with_status("inferred")
+
+    def _fields_with_status(self, status: str) -> list[tuple[str, FieldValue]]:
+        matches: list[tuple[str, FieldValue]] = []
+        for key, field_value in self.source.items():
+            if isinstance(field_value, FieldValue) and field_value.status == status:
+                matches.append((f"source.{key}", field_value))
+        for key, field_value in self.dataset_scope.items():
+            if isinstance(field_value, FieldValue) and field_value.status == status:
+                matches.append((f"dataset_scope.{key}", field_value))
+        for variable_name, fields in self.variables.items():
+            for field_name, field_value in fields.items():
+                if isinstance(field_value, FieldValue) and field_value.status == status:
+                    matches.append((f"variables.{variable_name}.{field_name}", field_value))
+        return matches
+
+
+# @id CODE-AIDS-072
+# @implements REQ-AIDS-052
+# @design DES-AIDS-040
+def build_manifest(
+    source: dict,
+    dataset_scope: dict,
+    variables: dict[str, dict[str, FieldValue]],
+    transformations: tuple = (),
+) -> DataDefinitionManifest:
+    """Construct a ``DataDefinitionManifest`` from its component dicts."""
+    return DataDefinitionManifest(
+        source=source,
+        dataset_scope=dataset_scope,
+        variables=variables,
+        transformations=tuple(transformations),
+    )

package/src/ai_data_scientist/data_quality.py

@@ -0,0 +1,164 @@
+"""Semantic data-quality checks: schema-driven anomaly detection and
+independent-dataset overlap validation.
+
+Implements DES-AIDS-043 (REQ-AIDS-055). This module is deliberately
+separate from ``anomaly_detection`` (which performs statistical
+z-score outlier detection on a single numeric column); here, anomalies
+are semantic constraint violations defined by a declarative schema
+(ranges, allowed categories, non-null, uniqueness), and overlap
+validation cross-checks summary statistics between a primary dataset
+and an independent reference.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import pandas as pd
+
+
+# @id CODE-AIDS-076
+# @implements REQ-AIDS-055
+# @design DES-AIDS-043
+@dataclass(frozen=True)
+class AnomalyRecord:
+    """A single schema-constraint violation."""
+
+    column: str
+    rule: str
+    row_count: int
+    message: str
+    row_indices: tuple[int, ...] = ()
+
+
+# @id CODE-AIDS-077
+# @implements REQ-AIDS-055
+# @design DES-AIDS-043
+def detect_anomalies(df: pd.DataFrame, schema: dict) -> tuple[AnomalyRecord, ...]:
+    """Detect semantic constraint violations declared by ``schema``.
+
+    ``schema`` maps column name -> a dict of constraints, any of:
+      - ``"min"`` / ``"max"``: numeric range bounds (inclusive).
+      - ``"allowed"``: iterable of permitted categorical values.
+      - ``"not_null"``: bool, require no missing values.
+      - ``"unique"``: bool, require no duplicate values.
+    Unknown columns in ``schema`` that are absent from ``df`` are skipped.
+    """
+    records: list[AnomalyRecord] = []
+    for column, rules in schema.items():
+        if column not in df.columns:
+            continue
+        series = df[column]
+
+        if rules.get("not_null"):
+            missing = series.isna()
+            if missing.any():
+                records.append(
+                    AnomalyRecord(
+                        column=column,
+                        rule="not_null",
+                        row_count=int(missing.sum()),
+                        message=f"Column {column!r} has {int(missing.sum())} null value(s).",
+                        row_indices=tuple(series.index[missing]),
+                    )
+                )
+
+        if "min" in rules or "max" in rules:
+            numeric = pd.to_numeric(series, errors="coerce")
+            below = (
+                numeric < rules["min"] if "min" in rules else pd.Series(False, index=series.index)
+            )
+            above = (
+                numeric > rules["max"] if "max" in rules else pd.Series(False, index=series.index)
+            )
+            out_of_range = (below | above) & numeric.notna()
+            if out_of_range.any():
+                records.append(
+                    AnomalyRecord(
+                        column=column,
+                        rule="range",
+                        row_count=int(out_of_range.sum()),
+                        message=(
+                            f"Column {column!r} has {int(out_of_range.sum())} value(s) "
+                            f"outside [{rules.get('min')}, {rules.get('max')}]."
+                        ),
+                        row_indices=tuple(series.index[out_of_range]),
+                    )
+                )
+
+        if "allowed" in rules:
+            allowed = set(rules["allowed"])
+            disallowed = ~series.isin(allowed) & series.notna()
+            if disallowed.any():
+                records.append(
+                    AnomalyRecord(
+                        column=column,
+                        rule="allowed_values",
+                        row_count=int(disallowed.sum()),
+                        message=(
+                            f"Column {column!r} has {int(disallowed.sum())} value(s) "
+                            f"not in the allowed set."
+                        ),
+                        row_indices=tuple(series.index[disallowed]),
+                    )
+                )
+
+        if rules.get("unique"):
+            duplicated = series.duplicated(keep=False) & series.notna()
+            if duplicated.any():
+                records.append(
+                    AnomalyRecord(
+                        column=column,
+                        rule="unique",
+                        row_count=int(duplicated.sum()),
+                        message=f"Column {column!r} has {int(duplicated.sum())} duplicate value(s).",
+                        row_indices=tuple(series.index[duplicated]),
+                    )
+                )
+
+    return tuple(records)
+
+
+# @id CODE-AIDS-078
+# @implements REQ-AIDS-055
+# @design DES-AIDS-043
+@dataclass(frozen=True)
+class OverlapMismatch:
+    """A single column whose summary statistic disagrees between datasets."""
+
+    column: str
+    primary_value: float
+    reference_value: float
+    relative_difference: float
+
+
+def validate_anomalies(
+    primary: pd.DataFrame,
+    reference: pd.DataFrame,
+    columns: list[str],
+    tolerance: float = 0.1,
+) -> tuple[OverlapMismatch, ...]:
+    """Compare per-column means between an independent ``reference`` dataset
+    and ``primary``, flagging columns whose relative difference exceeds
+    ``tolerance``. Used to validate that detected anomalies are not an
+    artifact of a single dataset.
+    """
+    mismatches: list[OverlapMismatch] = []
+    for column in columns:
+        if column not in primary.columns or column not in reference.columns:
+            continue
+        primary_mean = float(pd.to_numeric(primary[column], errors="coerce").mean())
+        reference_mean = float(pd.to_numeric(reference[column], errors="coerce").mean())
+        if reference_mean == 0:
+            continue
+        relative_difference = abs(primary_mean - reference_mean) / abs(reference_mean)
+        if relative_difference > tolerance:
+            mismatches.append(
+                OverlapMismatch(
+                    column=column,
+                    primary_value=primary_mean,
+                    reference_value=reference_mean,
+                    relative_difference=relative_difference,
+                )
+            )
+    return tuple(mismatches)

package/src/ai_data_scientist/dataset_validation.py

@@ -0,0 +1,135 @@
+"""Independent-dataset overlap comparison (narrowed scope).
+
+Implements DES-AIDS-045 (REQ-AIDS-057). Per the approved design, this
+module operates only on already-loaded dataframes: ``compare_datasets``
+checks key overlap and per-column value agreement between a primary
+dataset and a candidate independent dataset. Automated dataset
+*discovery* (e.g. searching Kaggle or other catalogs) is explicitly
+out of scope for this repository and is not implemented here; callers
+are expected to load the candidate dataset themselves before calling
+this function.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+import pandas as pd
+
+_VALID_RELATIONSHIPS = frozenset({"unknown", "independent", "derived", "overlapping"})
+
+
+# @id CODE-AIDS-081
+# @implements REQ-AIDS-057
+# @design DES-AIDS-045
+@dataclass(frozen=True)
+class ColumnComparison:
+    """Agreement statistics for one mapped column pair."""
+
+    primary_column: str
+    candidate_column: str
+    matched_rows: int
+    mismatched_rows: int
+    agreement_rate: float | None
+
+
+@dataclass(frozen=True)
+class DatasetComparisonReport:
+    """Outcome of comparing a primary dataset against a candidate dataset."""
+
+    candidate_relationship: str
+    matched_keys: int
+    primary_only_keys: int
+    candidate_only_keys: int
+    column_comparisons: tuple[ColumnComparison, ...] = field(default_factory=tuple)
+
+    def __post_init__(self) -> None:
+        if self.candidate_relationship not in _VALID_RELATIONSHIPS:
+            raise ValueError(
+                f"candidate_relationship must be one of {sorted(_VALID_RELATIONSHIPS)}, "
+                f"got {self.candidate_relationship!r}."
+            )
+
+
+# @id CODE-AIDS-082
+# @implements REQ-AIDS-057 REQ-AIDS-062
+# @design DES-AIDS-045 DES-AIDS-050
+def compare_datasets(
+    primary: pd.DataFrame,
+    candidate: pd.DataFrame,
+    key_mapping: dict[str, str],
+    value_mapping: dict[str, str],
+    candidate_relationship: str = "unknown",
+) -> DatasetComparisonReport:
+    """Compare ``primary`` against an independent ``candidate`` dataset.
+
+    ``key_mapping`` maps primary key column name(s) -> candidate column
+    name(s), used to join the two datasets. ``value_mapping`` maps
+    primary value column name -> candidate value column name for
+    per-column agreement checks on the joined rows.
+
+    Rows whose key is null on either side are excluded from
+    ``matched_keys``/``primary_only_keys``/``candidate_only_keys`` and from
+    the per-column agreement computation (REQ-AIDS-062): otherwise two
+    null keys (e.g. both ``None``) would incorrectly count as a matched key
+    pair under Python tuple-set equality.
+    """
+    primary_keys = list(key_mapping.keys())
+    candidate_keys = list(key_mapping.values())
+
+    eligible_primary = primary.dropna(subset=primary_keys)
+    eligible_candidate = candidate.dropna(subset=candidate_keys)
+
+    primary_key_values = set(
+        map(tuple, eligible_primary[primary_keys].itertuples(index=False, name=None))
+    )
+    candidate_key_values = set(
+        map(tuple, eligible_candidate[candidate_keys].itertuples(index=False, name=None))
+    )
+
+    matched_keys = primary_key_values & candidate_key_values
+    primary_only_keys = primary_key_values - candidate_key_values
+    candidate_only_keys = candidate_key_values - primary_key_values
+
+    merged = eligible_primary.merge(
+        eligible_candidate,
+        left_on=primary_keys,
+        right_on=candidate_keys,
+        how="inner",
+        suffixes=("_primary", "_candidate"),
+    )
+
+    column_comparisons: list[ColumnComparison] = []
+    for primary_column, candidate_column in value_mapping.items():
+        primary_col_name = (
+            f"{primary_column}_primary" if primary_column in candidate.columns else primary_column
+        )
+        candidate_col_name = (
+            f"{candidate_column}_candidate"
+            if candidate_column in primary.columns
+            else candidate_column
+        )
+        if primary_col_name not in merged.columns or candidate_col_name not in merged.columns:
+            continue
+        matches = merged[primary_col_name] == merged[candidate_col_name]
+        matched_rows = int(matches.sum())
+        mismatched_rows = int((~matches).sum())
+        total = matched_rows + mismatched_rows
+        agreement_rate = (matched_rows / total) if total else None
+        column_comparisons.append(
+            ColumnComparison(
+                primary_column=primary_column,
+                candidate_column=candidate_column,
+                matched_rows=matched_rows,
+                mismatched_rows=mismatched_rows,
+                agreement_rate=agreement_rate,
+            )
+        )
+
+    return DatasetComparisonReport(
+        candidate_relationship=candidate_relationship,
+        matched_keys=len(matched_keys),
+        primary_only_keys=len(primary_only_keys),
+        candidate_only_keys=len(candidate_only_keys),
+        column_comparisons=tuple(column_comparisons),
+    )

package/src/ai_data_scientist/dependency_pins.py

@@ -0,0 +1,60 @@
+"""Read declared dependency version specifiers from pyproject.toml.
+
+Implements DES-AIDS-038 (REQ-AIDS-050): a minimal, dependency-free reader of
+pyproject.toml's [project].dependencies array, used to assert the
+jupyter-mcp-server / mcp version pins stay explicitly bounded on both sides
+(no silent drift to an untested release whose negotiated MCP protocol
+version is incompatible with the rest of the pinned stack).
+
+Deliberately avoids a TOML-parsing library: stdlib ``tomllib`` only ships
+from Python 3.11, and this project still supports 3.10, so the
+``dependencies = [...]`` array is scanned as plain text instead.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+_PYPROJECT_PATH = Path(__file__).resolve().parents[2] / "pyproject.toml"
+_DEPENDENCY_LINE_PATTERN = re.compile(r'^\s*"([A-Za-z0-9_.-]+)([^"]*)"\s*,?\s*$')
+
+
+def _normalize(name: str) -> str:
+    """Normalize a package name per PEP 503 (case/separator-insensitive)."""
+    return re.sub(r"[-_.]+", "-", name).lower()
+
+
+# @id CODE-AIDS-059
+# @implements REQ-AIDS-050
+# @design DES-AIDS-038
+def get_dependency_specifier(name: str, pyproject_path: Path = _PYPROJECT_PATH) -> str:
+    """Return the raw dependency requirement string declared for ``name``.
+
+    Scans ``pyproject_path``'s ``[project].dependencies`` array for the
+    single entry whose package name matches ``name`` (case/separator
+    insensitive), returning it verbatim including its version specifier
+    (e.g. ``"jupyter-mcp-server>=2.2,<2.3"``).
+
+    Raises ``KeyError`` if no dependency named ``name`` is declared.
+    """
+    target = _normalize(name)
+    in_dependencies = False
+    for line in pyproject_path.read_text(encoding="utf-8").splitlines():
+        stripped = line.strip()
+        if not in_dependencies:
+            if stripped.startswith("dependencies"):
+                in_dependencies = True
+            continue
+        if stripped.startswith("]"):
+            break
+        match = _DEPENDENCY_LINE_PATTERN.match(line)
+        if not match:
+            continue
+        raw_name = match.group(1)
+        if _normalize(raw_name) == target:
+            return stripped.rstrip(",").strip('"')
+
+    raise KeyError(
+        f"No dependency named {name!r} declared in {pyproject_path}'s [project.dependencies]."
+    )

package/src/ai_data_scientist/eda.py

@@ -0,0 +1,82 @@
+"""Exploratory data analysis.
+
+Implements DES-AIDS-007 (REQ-AIDS-005): summary statistics, dtypes, and
+missing-value counts matching pandas describe()/info() reference values.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import pandas as pd
+
+
+@dataclass(frozen=True)
+class EDAReport:
+    describe: dict
+    dtypes: dict
+    non_null_counts: dict
+    missing_summary: dict
+    categorical_summary: dict
+
+
+DEFAULT_TOP_N = 10
+
+
+# @id CODE-AIDS-005
+# @implements REQ-AIDS-005
+# @design DES-AIDS-007
+# @id CODE-AIDS-051
+# @implements REQ-AIDS-043
+# @design DES-AIDS-031
+def explore(df: pd.DataFrame, top_n: int = DEFAULT_TOP_N) -> EDAReport:
+    """Compute summary statistics, dtypes and non-null counts for ``df``.
+
+    Also reports, per column, a missing-value count/ratio, and, for
+    categorical (object/category/bool) columns, a unique-value count and
+    the ``top_n`` most frequent values with their counts and ratios
+    (``truncated`` is set when more unique values exist than ``top_n``).
+    The existing ``describe``/``dtypes``/``non_null_counts`` fields are
+    unchanged by this extension.
+    """
+    describe = df.describe().to_dict() if len(df.columns) > 0 else {}
+    dtypes = {column: str(dtype) for column, dtype in df.dtypes.items()}
+    non_null_counts = df.count().to_dict()
+
+    row_count = len(df)
+    missing_summary = {}
+    for column in df.columns:
+        missing_count = int(df[column].isna().sum())
+        missing_ratio = (missing_count / row_count) if row_count else 0.0
+        missing_summary[column] = {
+            "missing_count": missing_count,
+            "missing_ratio": missing_ratio,
+        }
+
+    categorical_summary = {}
+    categorical_columns = df.select_dtypes(include=["object", "str", "category", "bool"]).columns
+    for column in categorical_columns:
+        value_counts = df[column].value_counts(dropna=True)
+        unique_count = int(value_counts.shape[0])
+        non_null_total = int(value_counts.sum())
+        top_values = [
+            {
+                "value": value,
+                "count": int(count),
+                "ratio": (count / non_null_total) if non_null_total else 0.0,
+            }
+            for value, count in value_counts.head(top_n).items()
+        ]
+        categorical_summary[column] = {
+            "unique_count": unique_count,
+            "top_values": top_values,
+            "truncated": unique_count > top_n,
+        }
+
+    return EDAReport(
+        describe=describe,
+        dtypes=dtypes,
+        non_null_counts=non_null_counts,
+        missing_summary=missing_summary,
+        categorical_summary=categorical_summary,
+    )