catence 0.0.0 → 0.1.0

package/python/catence/providers/garmin/cli.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import argparse
+import getpass
+import json
+import os
+from datetime import date
+from pathlib import Path
+
+from .staging import StagingWriter
+from .worker import GarminStagingWorker
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Stage strictly read-only Garmin data for Catence.")
+    parser.add_argument("--from", dest="from_date", required=True, type=date.fromisoformat)
+    parser.add_argument("--daily-from", dest="daily_from_date", type=date.fromisoformat)
+    parser.add_argument("--daily-to", dest="daily_to_date", type=date.fromisoformat)
+    parser.add_argument("--activity-from", dest="activity_from_date", type=date.fromisoformat)
+    parser.add_argument("--activity-to", dest="activity_to_date", type=date.fromisoformat)
+    parser.add_argument("--data-dir", required=True, type=Path)
+    parser.add_argument("--output", required=True, type=Path)
+    parser.add_argument("--run-id", required=True)
+    parser.add_argument("--to", dest="to_date", type=date.fromisoformat)
+    parser.add_argument("--known-activities", type=Path)
+    parser.add_argument("--historical-only", action="store_true", help="Skip current account and collection endpoints.")
+    parser.add_argument("--skip-daily", action="store_true", help="Do not fetch date-scoped daily data.")
+    parser.add_argument("--skip-activities", action="store_true", help="Do not fetch date-scoped activities.")
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    from garminconnect import Garmin
+
+    email = os.environ.get("GARMIN_EMAIL")
+    if not email:
+        raise SystemExit("GARMIN_EMAIL is required.")
+    password = os.environ.get("GARMIN_PASSWORD") or getpass.getpass("Garmin password: ")
+    writer = StagingWriter(args.data_dir, args.output)
+    writer.manifest(args.run_id, args.from_date.isoformat())
+    token_store = args.data_dir / "auth" / "garmin_tokens.json"
+    token_store.parent.mkdir(parents=True, exist_ok=True)
+    client = Garmin(email, password, prompt_mfa=lambda: input("Garmin MFA code: "))
+    client.login(str(token_store))
+    known_hashes: dict[str, str] = {}
+    if args.known_activities and args.known_activities.exists():
+        loaded = json.loads(args.known_activities.read_text(encoding="utf-8"))
+        if isinstance(loaded, dict):
+            known_hashes = {str(key): str(value) for key, value in loaded.items()}
+    GarminStagingWorker(client, writer, args.data_dir, known_hashes).sync(
+        daily_from_date=None if args.skip_daily else args.daily_from_date or args.from_date,
+        activity_from_date=None if args.skip_activities else args.activity_from_date or args.from_date,
+        to_date=args.to_date,
+        daily_to_date=args.daily_to_date,
+        activity_to_date=args.activity_to_date,
+        include_non_historical=not args.historical_only,
+    )
+
+
+if __name__ == "__main__":
+    main()

package/python/catence/providers/garmin/contracts.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from . import STAGING_SCHEMA_VERSION
+
+
+class StagingRecord(BaseModel):
+    """Shared JSONL envelope; Node performs the final discriminated validation."""
+
+    model_config = ConfigDict(extra="allow")
+    kind: Literal["run_manifest", "raw_object", "source_entity", "stream_manifest", "activity_sync_state", "extraction_error"]
+    schemaVersion: Literal[STAGING_SCHEMA_VERSION] = STAGING_SCHEMA_VERSION
+    provider: Literal["garmin"] = "garmin"
+    payload: dict[str, Any] | None = None
+
+
+def validate_record(record: dict[str, Any]) -> dict[str, Any]:
+    return StagingRecord.model_validate(record).model_dump(exclude_none=False) | {
+        key: value for key, value in record.items() if key not in {"kind", "schemaVersion", "provider", "payload"}
+    }

package/python/catence/providers/garmin/registry.py

@@ -0,0 +1,48 @@
+"""Explicit allowlist for Garmin reads; mutation methods are deliberately absent."""
+
+GARMIN_READ_METHODS = {
+    "profile": ("get_user_profile", "get_userprofile_settings"),
+    "devices": ("get_devices", "get_device_last_used", "get_device_alarms"),
+    "health_daily": (
+        "get_stats", "get_user_summary", "get_steps_data", "get_heart_rates",
+        "get_sleep_data", "get_stress_data", "get_all_day_stress", "get_body_battery_events",
+        "get_respiration_data", "get_spo2_data", "get_hrv_data", "get_training_readiness",
+        "get_morning_training_readiness", "get_training_status", "get_fitnessage_data",
+        "get_floors", "get_intensity_minutes_data", "get_all_day_events", "get_endurance_score",
+        "get_max_metrics", "get_hydration_data", "get_menstrual_data_for_date",
+        "get_nutrition_daily_food_log", "get_nutrition_daily_meals", "get_nutrition_daily_settings",
+    ),
+    "health_range": (
+        "get_daily_steps", "get_body_battery", "get_blood_pressure", "get_weigh_ins",
+        "get_progress_summary_between_dates", "get_menstrual_calendar_data",
+    ),
+    "training_singletons": (
+        "get_lactate_threshold", "get_heart_rate_zones", "get_power_zones", "get_cycling_ftp",
+        "get_personal_record", "get_earned_badges", "get_available_badges", "get_in_progress_badges",
+        "get_training_plans", "get_pregnancy_summary", "get_golf_user_stats", "get_golf_club_stats",
+    ),
+    "training_range": (
+        "get_functional_threshold_power_range", "get_max_metrics_range", "get_hrv_data_range",
+        "get_hill_score", "get_running_tolerance", "get_race_predictions",
+    ),
+    "activities": (
+        "get_activities_by_date", "get_activity", "get_activity_details", "get_activity_splits",
+        "get_activity_typed_splits", "get_activity_split_summaries", "get_activity_weather",
+        "get_activity_hr_in_timezones", "get_activity_power_in_timezones", "get_activity_exercise_sets",
+        "get_activity_gear", "download_activity",
+    ),
+    "workouts": ("get_workouts", "get_scheduled_workouts"),
+    "goals": ("get_goals",),
+    "gear": ("get_gear",),
+    "golf": ("get_golf_summary",),
+}
+
+MUTATION_PREFIXES = ("add_", "create_", "delete_", "import_", "set_", "update_", "upload_", "push_", "request_")
+
+
+def assert_read_only_registry() -> None:
+    """Fail closed if a future registry edit accidentally exposes a mutation."""
+    for methods in GARMIN_READ_METHODS.values():
+        for method in methods:
+            if method.startswith(MUTATION_PREFIXES):
+                raise RuntimeError(f"Unsafe Garmin method in read registry: {method}")

package/python/catence/providers/garmin/staging.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from . import STAGING_SCHEMA_VERSION
+from .contracts import validate_record
+
+
+def utc_now() -> str:
+    return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
+
+
+def safe_segment(value: str) -> str:
+    return "".join(char if char.isalnum() or char in "._-" else "_" for char in value)[:160] or "unknown"
+
+
+@dataclass
+class StagingWriter:
+    data_dir: Path
+    output: Path
+
+    def __post_init__(self) -> None:
+        self.output.parent.mkdir(parents=True, exist_ok=True)
+        self.output.touch(exist_ok=True)
+
+    def emit(self, record: dict[str, Any]) -> None:
+        record["schemaVersion"] = STAGING_SCHEMA_VERSION
+        with self.output.open("a", encoding="utf-8") as file:
+            file.write(json.dumps(validate_record(record), separators=(",", ":"), default=str))
+            file.write("\n")
+
+    def manifest(self, run_id: str, from_date: str) -> None:
+        self.emit({
+            "kind": "run_manifest", "provider": "garmin", "runId": run_id,
+            "fromDate": from_date, "createdAt": utc_now(),
+        })
+
+    def archive_json(self, endpoint: str, remote_id: str | None, payload: Any, scope: dict[str, Any] | None = None) -> str:
+        contents = (json.dumps(payload, indent=2, sort_keys=True, default=str) + "\n").encode()
+        return self._archive(endpoint, remote_id, contents, "json", "application/json", scope)
+
+    def archive_bytes(self, endpoint: str, remote_id: str | None, contents: bytes, extension: str = "bin", scope: dict[str, Any] | None = None) -> str:
+        return self._archive(endpoint, remote_id, contents, extension, "application/octet-stream", scope)
+
+    def _archive(self, endpoint: str, remote_id: str | None, contents: bytes, extension: str, content_type: str, scope: dict[str, Any] | None) -> str:
+        digest = hashlib.sha256(contents).hexdigest()
+        relative = Path("raw") / "garmin" / safe_segment(endpoint) / safe_segment(remote_id or "collection") / f"{digest}.{extension}"
+        destination = self.data_dir / relative
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        if not destination.exists():
+            temporary = destination.with_suffix(f".{extension}.tmp")
+            temporary.write_bytes(contents)
+            os.replace(temporary, destination)
+        self.emit({
+            "kind": "raw_object", "provider": "garmin", "endpoint": endpoint, "remoteId": remote_id,
+            "fetchedAt": utc_now(), "contentHash": digest, "contentType": content_type,
+            "relativePath": str(relative), "scope": scope or {},
+        })
+        return digest
+
+    def source_entity(
+        self, entity_type: str, remote_id: str, payload: dict[str, Any], raw_hash: str | None,
+        parent_remote_id: str | None = None, occurred_on: str | None = None,
+    ) -> None:
+        self.emit({
+            "kind": "source_entity", "provider": "garmin", "entityType": entity_type,
+            "remoteId": str(remote_id), "parentRemoteId": parent_remote_id, "occurredOn": occurred_on,
+            "sourceUpdatedAt": None, "rawObjectHash": raw_hash, "payload": payload, "extension": {},
+        })
+
+    def activity_sync_state(self, activity_id: str, summary_hash: str, details_fetched: bool) -> None:
+        self.emit({
+            "kind": "activity_sync_state", "provider": "garmin", "activityRemoteId": activity_id,
+            "summaryHash": summary_hash, "detailsFetched": details_fetched,
+        })
+
+    def error(self, endpoint: str, message: str, remote_id: str | None = None, retryable: bool = True) -> None:
+        self.emit({
+            "kind": "extraction_error", "provider": "garmin", "endpoint": endpoint,
+            "remoteId": remote_id, "message": message, "retryable": retryable,
+        })

package/python/catence/providers/garmin/streams.py

@@ -0,0 +1,195 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import zipfile
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Iterable
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+
+SAMPLE_COLUMNS = [
+    "activity_source_id", "timestamp_utc", "elapsed_s", "distance_m", "latitude", "longitude",
+    "altitude_m", "heart_rate_bpm", "power_w", "cadence_rpm", "speed_mps", "temperature_c", "grade_pct",
+    "left_right_balance_pct", "left_torque_effectiveness_pct", "right_torque_effectiveness_pct",
+    "left_pedal_smoothness_pct", "right_pedal_smoothness_pct",
+    "left_power_phase_start_deg", "left_power_phase_end_deg", "right_power_phase_start_deg", "right_power_phase_end_deg",
+    "left_power_phase_peak_start_deg", "left_power_phase_peak_end_deg", "right_power_phase_peak_start_deg", "right_power_phase_peak_end_deg",
+    "left_platform_center_offset_mm", "right_platform_center_offset_mm", "extras_json",
+]
+
+POWER_BEST_DURATIONS = (1, 5, 15, 30, 60, 120, 300, 600, 1200, 1800, 3600, 5400, 7200)
+
+FIELD_MAP = {
+    "directTimestamp": "timestamp_utc", "timestamp": "timestamp_utc", "sumElapsedDuration": "elapsed_s", "elapsedDuration": "elapsed_s",
+    "sumDistance": "distance_m", "distance": "distance_m", "latitude": "latitude", "directLatitude": "latitude", "longitude": "longitude", "directLongitude": "longitude",
+    "altitude": "altitude_m", "directAltitude": "altitude_m", "directUncorrectedElevation": "altitude_m", "directElevation": "altitude_m",
+    "directHeartRate": "heart_rate_bpm", "heartRate": "heart_rate_bpm",
+    "directPower": "power_w", "power": "power_w", "directCadence": "cadence_rpm", "directBikeCadence": "cadence_rpm", "cadence": "cadence_rpm",
+    "directSpeed": "speed_mps", "speed": "speed_mps", "directTemperature": "temperature_c", "directAirTemperature": "temperature_c", "temperature": "temperature_c",
+    "grade": "grade_pct",
+    "leftRightBalance": "left_right_balance_pct", "left_right_balance": "left_right_balance_pct",
+    "leftTorqueEffectiveness": "left_torque_effectiveness_pct", "left_torque_effectiveness": "left_torque_effectiveness_pct",
+    "rightTorqueEffectiveness": "right_torque_effectiveness_pct", "right_torque_effectiveness": "right_torque_effectiveness_pct",
+    "leftPedalSmoothness": "left_pedal_smoothness_pct", "left_pedal_smoothness": "left_pedal_smoothness_pct",
+    "rightPedalSmoothness": "right_pedal_smoothness_pct", "right_pedal_smoothness": "right_pedal_smoothness_pct",
+    "leftPlatformCenterOffset": "left_platform_center_offset_mm", "left_pco": "left_platform_center_offset_mm",
+    "rightPlatformCenterOffset": "right_platform_center_offset_mm", "right_pco": "right_platform_center_offset_mm",
+}
+
+PAIR_FIELD_MAP = {
+    "leftPowerPhase": ("left_power_phase_start_deg", "left_power_phase_end_deg"),
+    "left_power_phase": ("left_power_phase_start_deg", "left_power_phase_end_deg"),
+    "rightPowerPhase": ("right_power_phase_start_deg", "right_power_phase_end_deg"),
+    "right_power_phase": ("right_power_phase_start_deg", "right_power_phase_end_deg"),
+    "leftPowerPhasePeak": ("left_power_phase_peak_start_deg", "left_power_phase_peak_end_deg"),
+    "left_power_phase_peak": ("left_power_phase_peak_start_deg", "left_power_phase_peak_end_deg"),
+    "rightPowerPhasePeak": ("right_power_phase_peak_start_deg", "right_power_phase_peak_end_deg"),
+    "right_power_phase_peak": ("right_power_phase_peak_start_deg", "right_power_phase_peak_end_deg"),
+}
+
+
+def _number(value: Any) -> float | None:
+    return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else None
+
+
+def _timestamp(value: Any) -> str | None:
+    if not isinstance(value, (int, float)):
+        return None
+    seconds = float(value)
+    if seconds > 10_000_000_000:
+        seconds /= 1000
+    return datetime.fromtimestamp(seconds, timezone.utc).isoformat().replace("+00:00", "Z")
+
+
+def activity_power_bests(samples: Iterable[dict[str, Any]]) -> dict[int, float]:
+    """Return rolling power bests without bridging invalid or long sample gaps."""
+    points: list[tuple[float, float]] = []
+    for sample in samples:
+        raw_time, raw_power = sample.get("timestamp_utc"), _number(sample.get("power_w"))
+        if not isinstance(raw_time, str) or raw_power is None:
+            continue
+        try:
+            points.append((datetime.fromisoformat(raw_time.replace("Z", "+00:00")).timestamp(), raw_power))
+        except ValueError:
+            continue
+    points.sort()
+    bests: dict[int, float] = {}
+    segment: list[tuple[float, float]] = []
+
+    def evaluate(rows: list[tuple[float, float]]) -> None:
+        for duration in POWER_BEST_DURATIONS:
+            start = 0
+            total = 0.0
+            for end, (current_time, current_power) in enumerate(rows):
+                total += current_power
+                while start < end and current_time - rows[start + 1][0] >= duration:
+                    total -= rows[start][1]
+                    start += 1
+                if current_time - rows[start][0] >= duration and end >= start:
+                    mean = total / (end - start + 1)
+                    bests[duration] = max(bests.get(duration, float("-inf")), mean)
+
+    for point in points:
+        if segment and point[0] - segment[-1][0] > 5:
+            evaluate(segment)
+            segment = []
+        segment.append(point)
+    evaluate(segment)
+    return bests
+
+
+def _sample_value(sample: dict[str, Any], extras: dict[str, Any], source_name: str, value: Any) -> None:
+    """Map known cycling-dynamics fields without discarding unfamiliar sensors."""
+    target = FIELD_MAP.get(source_name)
+    if target:
+        sample[target] = _number(value)
+        extras[source_name] = value
+        return
+    targets = PAIR_FIELD_MAP.get(source_name)
+    if targets and isinstance(value, (list, tuple)) and len(value) >= 2:
+        sample[targets[0]], sample[targets[1]] = _number(value[0]), _number(value[1])
+        extras[source_name] = value
+        return
+    extras[source_name] = value
+
+
+def activity_details_to_samples(activity_source_id: str, details: dict[str, Any]) -> list[dict[str, Any]]:
+    """Convert Garmin positional detail metrics into a stable, nullable sample schema."""
+    descriptors = details.get("metricDescriptors") or []
+    descriptor_names = {
+        descriptor.get("metricsIndex"): descriptor.get("key")
+        for descriptor in descriptors if isinstance(descriptor, dict) and isinstance(descriptor.get("metricsIndex"), int)
+    }
+    rows = details.get("activityDetailMetrics") or details.get("metrics") or []
+    samples: list[dict[str, Any]] = []
+    for row in rows:
+        values = row.get("metrics") if isinstance(row, dict) else row
+        if not isinstance(values, list):
+            continue
+        sample: dict[str, Any] = {column: None for column in SAMPLE_COLUMNS}
+        sample["activity_source_id"] = activity_source_id
+        extras: dict[str, Any] = {}
+        for index, value in enumerate(values):
+            source_name = descriptor_names.get(index, str(index))
+            target = FIELD_MAP.get(source_name)
+            if target == "timestamp_utc":
+                sample[target] = _timestamp(value)
+            else:
+                _sample_value(sample, extras, source_name, value)
+        sample["extras_json"] = json.dumps(extras, separators=(",", ":"), default=str)
+        samples.append(sample)
+    return samples
+
+
+def fit_archive_to_samples(activity_source_id: str, archive: Path) -> list[dict[str, Any]]:
+    """Best-effort FIT parsing; malformed/unavailable archives simply yield no samples."""
+    try:
+        import fitdecode
+        with zipfile.ZipFile(archive) as zip_file:
+            fit_name = next((name for name in zip_file.namelist() if name.lower().endswith(".fit")), None)
+            if not fit_name:
+                return []
+            with zip_file.open(fit_name) as stream, fitdecode.FitReader(stream) as reader:
+                rows: list[dict[str, Any]] = []
+                for frame in reader:
+                    if not isinstance(frame, fitdecode.records.FitDataMessage) or frame.name != "record":
+                        continue
+                    fields = {field.name: field.value for field in frame.fields}
+                    sample = {column: None for column in SAMPLE_COLUMNS}
+                    sample["activity_source_id"] = activity_source_id
+                    sample["timestamp_utc"] = fields.get("timestamp").isoformat().replace("+00:00", "Z") if fields.get("timestamp") else None
+                    sample["distance_m"] = _number(fields.get("distance"))
+                    sample["altitude_m"] = _number(fields.get("enhanced_altitude") or fields.get("altitude"))
+                    sample["heart_rate_bpm"] = _number(fields.get("heart_rate"))
+                    sample["power_w"] = _number(fields.get("power"))
+                    sample["cadence_rpm"] = _number(fields.get("cadence"))
+                    sample["speed_mps"] = _number(fields.get("enhanced_speed") or fields.get("speed"))
+                    sample["temperature_c"] = _number(fields.get("temperature"))
+                    extras: dict[str, Any] = {}
+                    for name, value in fields.items():
+                        if name in {"timestamp", "distance", "enhanced_altitude", "altitude", "heart_rate", "power", "cadence", "enhanced_speed", "speed", "temperature"}:
+                            continue
+                        _sample_value(sample, extras, name, value)
+                    for name, value in extras.items():
+                        fields[name] = value
+                    sample["extras_json"] = json.dumps(fields, separators=(",", ":"), default=str)
+                    rows.append(sample)
+                return rows
+    except (ImportError, OSError, ValueError, zipfile.BadZipFile):
+        return []
+
+
+def write_parquet(data_dir: Path, activity_id: str, start_date: str, samples: Iterable[dict[str, Any]]) -> tuple[Path, str, int]:
+    records = list(samples)
+    year, month = (start_date[:4] or "unknown"), (start_date[5:7] or "unknown")
+    destination = data_dir / "lake" / "activity_samples" / "provider=garmin" / f"year={year}" / f"month={month}" / f"activity={activity_id}.parquet"
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    table = pa.Table.from_pylist(records, schema=pa.schema([(column, pa.string() if column in {"activity_source_id", "timestamp_utc", "extras_json"} else pa.float64()) for column in SAMPLE_COLUMNS]))
+    temporary = destination.with_suffix(".tmp")
+    pq.write_table(table, temporary, compression="zstd")
+    temporary.replace(destination)
+    return destination, hashlib.sha256(destination.read_bytes()).hexdigest(), len(records)