@camstack/addon-waste-collections 0.0.0-stage → 0.1.2

package/python/vendor/waste_collection_schedule/service/OpenCities.py

@@ -0,0 +1,431 @@
+"""Shared client for councils running the "OpenCities"/"MyArea" widget.
+
+Many AU/NZ council websites expose the same two endpoints on their own
+domain (the widget behind e.g. https://www.logan.qld.gov.au/MyLogan):
+
+- ``GET <domain>/api/v1/myarea/search[fuzzy]`` — address search, returning
+  JSON ``{"Items": [{"Id": "<geolocation-guid>", "AddressSingleLine": ...}]}``
+  (a handful of legacy deployments return XML instead).
+- ``GET <domain>/ocapi/Public/myarea/wasteservices`` — given a
+  ``geolocationid``, returns ``{"success": bool, "responseContent": "<html>"}``
+  where the HTML is a series of ``<article>``/``div.waste-services-result``
+  blocks, each with an ``<h3>`` (waste type), a ``.next-service`` element
+  (next collection date) and usually a ``.note`` element (the service's
+  cadence and kerbside instructions in prose).
+
+This module centralises that flow behind :class:`OpenCitiesClient` so
+per-council source files stay a thin ``OpenCitiesConfig`` + ``Source`` shim.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import date, datetime, timedelta
+from typing import Any, Literal
+
+import requests
+from bs4 import BeautifulSoup
+from curl_cffi import requests as curl_cffi_requests
+
+from waste_collection_schedule import Collection
+from waste_collection_schedule.exceptions import (
+    SourceArgAmbiguousWithSuggestions,
+    SourceArgumentNotFound,
+)
+
+DEFAULT_DATE_FORMAT = "%a %d/%m/%Y"
+
+_WEEKDAY_NAMES = [
+    "Monday",
+    "Tuesday",
+    "Wednesday",
+    "Thursday",
+    "Friday",
+    "Saturday",
+    "Sunday",
+]
+
+
+@dataclass(frozen=True)
+class OpenCitiesConfig:
+    """Per-council configuration for :class:`OpenCitiesClient`."""
+
+    domain: str
+    """Council base URL, e.g. ``"https://www.ballina.nsw.gov.au"`` (no trailing slash)."""
+
+    argument_name: str = "address"
+    """Exact ``Source.__init__`` kwarg name, used in raised exceptions."""
+
+    direct_argument_name: str | None = None
+    """
+    ``Source.__init__`` kwarg holding a geolocation id given instead of an
+    address, for the sources that accept one. Exceptions raised on that path
+    name this argument, so the config flow marks the field the visitor
+    actually filled in rather than the empty address one.
+    """
+
+    search_fuzzy: bool = False
+    """Use ``/api/v1/myarea/searchfuzzy`` instead of ``/api/v1/myarea/search``."""
+
+    max_results: int | None = None
+    """Optional ``maxresults`` query param on the search request."""
+
+    search_response_format: Literal["json", "xml", "json_then_xml"] = "json"
+    """Search response format. ``"json_then_xml"`` tries JSON, falls back to XML."""
+
+    page_link: str | None = None
+    """Optional ``pageLink`` query param required by some council instances."""
+
+    ocsvclang: str = "en-AU"
+
+    use_curl_cffi: bool = False
+    """Use curl_cffi (Chrome impersonation) to bypass Akamai/Incapsula bot protection."""
+
+    impersonate: str = "chrome"
+
+    warm_up_url: str | None = None
+    """A page fetched once, lazily, before the first API request (session/cookie priming)."""
+
+    warm_up_before: Literal["search", "wasteservices"] = "search"
+    """
+    When the (one-off) warm-up GET fires: before the address search
+    (default), or after it — right before the wasteservices call, which
+    also covers the direct ``geolocation_id`` bypass path where no search
+    ever happens.
+    """
+
+    date_format: str = DEFAULT_DATE_FORMAT
+
+    icon_keywords: dict[str, Any] | None = None
+    """Lower-cased waste-type substring -> icon, checked in insertion order."""
+
+    strip_type_suffixes: tuple[str, ...] = ()
+    """Suffixes (case-insensitive) stripped from parsed waste-type labels, e.g. ``("collection",)``."""
+
+    headers: dict[str, str] | None = None
+
+    timeout: int = 30
+
+    strict_address_matching: bool = False
+    """
+    If False (default), the first (highest-ranked) search result is always
+    used, trusting the API's own relevance ranking -- this matches how the
+    vast majority of council deployments behave today. If True, multiple
+    search results are instead disambiguated by exact (case/space-
+    insensitive) match against ``AddressSingleLine``, raising
+    ``SourceArgAmbiguousWithSuggestions`` when that's not conclusive. Only
+    enable this for councils whose original implementation already did its
+    own such disambiguation (e.g. Hobart, Moorabool) -- for most councils
+    it does more harm than good, since a user's address rarely matches the
+    server's canonical ``AddressSingleLine`` formatting byte-for-byte (e.g.
+    missing a state abbreviation or punctuation the server adds), which
+    would otherwise turn a normal, correctly-resolved top hit into a hard
+    "ambiguous" failure.
+    """
+
+    strict_single_result: bool = False
+    """
+    Only meaningful with ``strict_address_matching``. By default a lone search
+    result is trusted even under strict matching, on the assumption that one
+    hit means the API was confident. A few deployments' search is loose enough
+    to answer an unrelated query with exactly one wrong hit -- lakemac returns
+    the single result "2 Lake Ridge Lane, MURRAYS BEACH" for the query
+    "2 Wallarah Rd" -- and there strict matching needs to apply to a single
+    result too, so the caller is offered it as a suggestion to confirm rather
+    than silently handed another property's bins. Left off by default because
+    for most councils the lone hit is right and merely formatted differently
+    from what the user typed (a missing state abbreviation or postcode), which
+    would otherwise turn a correct resolution into a hard failure.
+    """
+
+    require_date_precise: bool = False
+    """
+    If True, only wasteservices blocks additionally carrying a
+    ``date-precise`` CSS class are parsed, skipping vague/recurring entries
+    with no concrete next date (some council deployments, e.g. Shoalhaven,
+    mix both kinds of block in the same response).
+    """
+
+    exclude_types: tuple[str, ...] = ()
+    """
+    Waste-type labels (exact match) to always drop, e.g. a council mixing
+    an unrelated "Burning off" permit article into the same wasteservices
+    response.
+    """
+
+    exclude_type_prefixes: tuple[str, ...] = ()
+    """
+    Waste-type label prefixes to always drop, e.g. "Calendar" entries
+    (calendar-download links, not actual collections) some deployments mix
+    into the same wasteservices response.
+    """
+
+
+class OpenCitiesClient:
+    def __init__(self, config: OpenCitiesConfig) -> None:
+        self._cfg = config
+        self._session = self._build_session()
+        self._geolocation_id: str | None = None
+        self._warmed_up = False
+
+    # ---- public API ---------------------------------------------------
+
+    def fetch(
+        self, address: str | None = None, geolocation_id: str | None = None
+    ) -> list[Collection]:
+        """Resolve collections for an address, or directly for a geolocation id.
+
+        When resolving from ``address``, the geolocation id is cached across
+        calls (HA polls ``fetch()`` repeatedly) so only the first call pays
+        for the address search. If a cached id later fails, it is re-resolved
+        once and the wasteservices call retried.
+        """
+        if geolocation_id is not None:
+            return self.fetch_by_geolocation_id(geolocation_id)
+
+        if address is None:
+            raise ValueError("Either address or geolocation_id must be provided")
+
+        used_cache = self._geolocation_id is not None
+        if not used_cache:
+            self._geolocation_id = self.resolve_geolocation_id(address)
+
+        try:
+            return self.fetch_by_geolocation_id(self._geolocation_id, address)  # type: ignore[arg-type]
+        except SourceArgumentNotFound:
+            if not used_cache:
+                raise
+            self._geolocation_id = self.resolve_geolocation_id(address)
+            return self.fetch_by_geolocation_id(self._geolocation_id, address)
+
+    def resolve_geolocation_id(self, address: str) -> str:
+        if self._cfg.warm_up_before == "search":
+            self._ensure_warmed_up()
+        items = self._search(address)
+        return self._select_address(address, items)
+
+    def fetch_by_geolocation_id(
+        self, geolocation_id: str, address: str | None = None
+    ) -> list[Collection]:
+        html = self.get_waste_services_html(geolocation_id, address)
+        return self._parse_wasteservices_html(html)
+
+    def get_waste_services_html(
+        self, geolocation_id: str, address: str | None = None
+    ) -> str:
+        """Return the raw wasteservices HTML fragment for a geolocation id.
+
+        Exposed alongside :meth:`fetch_by_geolocation_id` for the rare
+        source that needs to parse extra per-block content (e.g. a "note"
+        div with a collection-frequency hint) beyond the waste type and
+        next-service date the shared parser extracts.
+        """
+        if self._cfg.warm_up_before == "wasteservices":
+            self._ensure_warmed_up()
+        params: dict[str, str] = {
+            "geolocationid": geolocation_id,
+            "ocsvclang": self._cfg.ocsvclang,
+        }
+        if self._cfg.page_link:
+            params["pageLink"] = self._cfg.page_link
+        response = self._get(
+            f"{self._cfg.domain}/ocapi/Public/myarea/wasteservices", params
+        )
+        data = response.json()
+        if not data.get("success", True) or not data.get("responseContent"):
+            # The address resolved, but the council holds no waste service for
+            # that property (vacant land, a commercial lot, a newly titled block
+            # not yet on a run). Report the address the visitor gave rather than
+            # the internal geolocation GUID, which reads as nonsense to them --
+            # unless the GUID is what they gave, in which case name that
+            # argument so the config flow flags the field they filled in.
+            if address is not None:
+                argument_name, value = self._cfg.argument_name, address
+            else:
+                argument_name = (
+                    self._cfg.direct_argument_name or self._cfg.argument_name
+                )
+                value = geolocation_id
+            raise SourceArgumentNotFound(
+                argument_name,
+                value,
+                "The council lists no waste collection service for this "
+                "property. Check the address, or contact the council if it "
+                "should have a collection.",
+            )
+        return data["responseContent"]
+
+    # ---- internals ------------------------------------------------------
+
+    def _build_session(self):
+        if self._cfg.use_curl_cffi:
+            session = curl_cffi_requests.Session(impersonate=self._cfg.impersonate)
+        else:
+            session = requests.Session()
+        if self._cfg.headers:
+            session.headers.update(self._cfg.headers)
+        return session
+
+    def _get(self, url: str, params: dict[str, Any] | None = None):
+        response = self._session.get(url, params=params, timeout=self._cfg.timeout)
+        response.raise_for_status()
+        return response
+
+    def _ensure_warmed_up(self) -> None:
+        if self._warmed_up or not self._cfg.warm_up_url:
+            return
+        self._get(self._cfg.warm_up_url)
+        self._warmed_up = True
+
+    def _search(self, address: str) -> list[dict[str, Any]]:
+        path = "searchfuzzy" if self._cfg.search_fuzzy else "search"
+        params: dict[str, Any] = {"keywords": address}
+        if self._cfg.max_results is not None:
+            params["maxresults"] = self._cfg.max_results
+        response = self._get(f"{self._cfg.domain}/api/v1/myarea/{path}", params)
+
+        fmt = self._cfg.search_response_format
+        if fmt == "xml":
+            return self._parse_search_xml(response.text)
+        if fmt == "json_then_xml":
+            try:
+                return response.json().get("Items") or []
+            except ValueError:
+                return self._parse_search_xml(response.text)
+        return response.json().get("Items") or []
+
+    @staticmethod
+    def _parse_search_xml(text: str) -> list[dict[str, Any]]:
+        soup = BeautifulSoup(text, "xml")
+        items: list[dict[str, Any]] = []
+        for result in soup.find_all("PhysicalAddressSearchResult"):
+            id_el = result.find("Id")
+            if id_el and id_el.text.strip():
+                item: dict[str, Any] = {"Id": id_el.text.strip()}
+                address_el = result.find("AddressSingleLine")
+                if address_el and address_el.text.strip():
+                    item["AddressSingleLine"] = address_el.text.strip()
+                items.append(item)
+        return items
+
+    def _select_address(self, address: str, items: list[dict[str, Any]]) -> str:
+        if not items:
+            raise SourceArgumentNotFound(self._cfg.argument_name, address)
+        if not self._cfg.strict_address_matching:
+            return items[0]["Id"]
+        if len(items) == 1 and not self._cfg.strict_single_result:
+            return items[0]["Id"]
+
+        normalized = address.lower().replace(" ", "")
+        exact = [
+            item
+            for item in items
+            if item.get("AddressSingleLine", "").lower().replace(" ", "") == normalized
+        ]
+        if len(exact) == 1:
+            return exact[0]["Id"]
+
+        if not any(item.get("AddressSingleLine") for item in items):
+            # No address text to disambiguate with (e.g. legacy XML results) —
+            # fall back to the first match rather than crash.
+            return items[0]["Id"]
+
+        suggestions = [item.get("AddressSingleLine", item["Id"]) for item in items]
+        raise SourceArgAmbiguousWithSuggestions(
+            self._cfg.argument_name, address, suggestions
+        )
+
+    @staticmethod
+    def _select_top_level(soup: BeautifulSoup, selector: str) -> list[Any]:
+        """soup.select(), but drop matches nested inside another match.
+
+        Some council deployments wrap a matching ``<article>`` inside a
+        matching ``div.waste-services-result`` (or vice versa); selecting
+        both tags would otherwise double-count every entry.
+        """
+        candidates = soup.select(selector)
+        candidate_ids = {id(candidate) for candidate in candidates}
+        return [
+            candidate
+            for candidate in candidates
+            if not any(id(parent) in candidate_ids for parent in candidate.parents)
+        ]
+
+    def _parse_wasteservices_html(self, html: str) -> list[Collection]:
+        soup = BeautifulSoup(html, "html.parser")
+        entries: list[Collection] = []
+        selector = (
+            "div.waste-services-result.date-precise"
+            if self._cfg.require_date_precise
+            else "article, div.waste-services-result"
+        )
+        for block in self._select_top_level(soup, selector):
+            title = block.select_one("h3")
+            next_service = block.select_one(".next-service")
+            if title is None or next_service is None:
+                continue
+
+            waste_type = title.get_text(" ", strip=True)
+            if waste_type in self._cfg.exclude_types:
+                continue
+            if waste_type.startswith(self._cfg.exclude_type_prefixes):
+                continue
+            for suffix in self._cfg.strip_type_suffixes:
+                if waste_type.lower().endswith(suffix.lower()):
+                    waste_type = waste_type[: -len(suffix)].strip()
+
+            date_text = next_service.get_text(" ", strip=True)
+            if not date_text:
+                continue
+            collection_date = self._parse_date(date_text)
+            if collection_date is None:
+                continue
+
+            # Most deployments state the service's cadence and kerbside
+            # instructions in a "note" div next to the date ("Collected
+            # fortnightly. Place bin on the kerb before the morning of
+            # collection."). The API returns only the ONE next date per
+            # service, so this sentence is the only place the cadence appears
+            # at all -- carry it through as the collection's description
+            # instead of dropping it.
+            note = block.select_one(".note")
+            description = note.get_text(" ", strip=True) if note else None
+
+            entries.append(
+                Collection(
+                    date=collection_date,
+                    t=waste_type,
+                    icon=self._resolve_icon(waste_type),
+                    description=description,
+                )
+            )
+        return entries
+
+    def _parse_date(self, text: str) -> date | None:
+        try:
+            return datetime.strptime(text, self._cfg.date_format).date()
+        except ValueError:
+            pass
+        # Some deployments describe a recurring service as "Every <Weekday>"
+        # instead of a concrete next date (no explicit next occurrence is
+        # given by the API at all) -- resolve it to the next occurrence of
+        # that weekday from today.
+        parts = text.split()
+        if len(parts) >= 2 and parts[0].lower() == "every":
+            try:
+                target_weekday = _WEEKDAY_NAMES.index(parts[1].capitalize())
+            except ValueError:
+                return None
+            today = datetime.now().date()
+            days_ahead = (target_weekday - today.weekday()) % 7
+            return today + timedelta(days=days_ahead)
+        return None
+
+    def _resolve_icon(self, label: str) -> Any | None:
+        if not self._cfg.icon_keywords:
+            return None
+        lowered = label.lower()
+        for keyword, icon in self._cfg.icon_keywords.items():
+            if keyword.lower() in lowered:
+                return icon
+        return None

package/python/vendor/waste_collection_schedule/service/Pozi.py

@@ -0,0 +1,194 @@
+"""
+Pozi GIS Service Library.
+
+Provides reusable helpers for interacting with Pozi-based council GIS services:
+  - Querying GeoJSON zone layers hosted on connect.pozi.com
+  - Querying QGIS WFS endpoints via Pozi server-side spatial filters
+  - Client-side point-in-polygon matching for GeoJSON zone lookups
+
+Typical workflows:
+
+  GeoJSON zones (e.g. Frankston, Bendigo):
+      1. Call query_geojson_zones() with the connect.pozi.com URL and coordinates.
+      2. Receive the properties dict of the matching zone feature.
+
+  WFS spatial query (e.g. Vincent):
+      1. Call query_wfs_layer() with the WFS endpoint URL, layer name and
+         coordinates. Pass map_path as well for QGIS servers that expect a MAP
+         parameter; omit it for dataset proxies that already point at a project.
+      2. Receive the properties dict of the matching feature.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import requests
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class PoziError(Exception):
+    """Base exception for Pozi-related failures."""
+
+
+class PoziGeoJsonError(PoziError):
+    """Raised when a GeoJSON zone query fails or returns no match."""
+
+
+class PoziWfsError(PoziError):
+    """Raised when a WFS query fails or returns no features."""
+
+
+def query_geojson_zones(
+    url: str,
+    lat: float,
+    lng: float,
+    *,
+    timeout: int = 20,
+) -> dict[str, Any]:
+    """Fetch GeoJSON zones and find the zone containing the given point.
+
+    Args:
+        url: Full URL to a GeoJSON file (typically on connect.pozi.com).
+        lat: Latitude of the point.
+        lng: Longitude of the point.
+        timeout: Request timeout in seconds.
+
+    Returns:
+        Properties dict of the matching feature.
+
+    Raises:
+        PoziGeoJsonError: If no zone contains the point.
+    """
+    r = requests.get(url, timeout=timeout)
+    r.raise_for_status()
+
+    data = r.json()
+    features = data.get("features", [])
+    if not features:
+        raise PoziGeoJsonError(f"No features found in GeoJSON from {url}")
+
+    for feature in features:
+        geometry = feature.get("geometry", {})
+        if _point_in_geometry(lat, lng, geometry):
+            _LOGGER.debug(
+                "Point (%s, %s) matched zone: %s",
+                lat,
+                lng,
+                feature.get("properties", {}),
+            )
+            return feature.get("properties", {})
+
+    raise PoziGeoJsonError(f"Point ({lat}, {lng}) not found in any zone from {url}")
+
+
+def query_wfs_layer(
+    base_url: str,
+    typename: str,
+    lat: float,
+    lng: float,
+    *,
+    map_path: str | None = None,
+    timeout: int = 20,
+) -> dict[str, Any]:
+    """Query a Pozi QGIS WFS endpoint with a spatial intersects filter.
+
+    Args:
+        base_url: Base URL of the QGIS WFS server, or of a Pozi dataset proxy
+            that already points at a project (e.g.
+            'https://vincent.pozi.com/proxy/v1/maps/waste/datasets/<id>').
+        typename: WFS typename / layer name (e.g. 'Waste_Collection').
+        lat: Latitude of the point (EPSG:4326).
+        lng: Longitude of the point (EPSG:4326).
+        map_path: Server-side path to the QGIS project file, for endpoints that
+            take a MAP parameter (e.g. 'C:/Pozi/Waste.qgs').
+        timeout: Request timeout in seconds.
+
+    Returns:
+        Properties dict of the first matching feature.
+
+    Raises:
+        PoziWfsError: If no features are found.
+    """
+    wfs_filter = (
+        "<Filter>"
+        "<Intersects>"
+        "<PropertyName>geom</PropertyName>"
+        '<Point xmlns="http://www.opengis.net/gml" srsName="EPSG:4326">'
+        f'<pos srsDimension="2">{lng} {lat}</pos>'
+        "</Point>"
+        "</Intersects>"
+        "</Filter>"
+    )
+
+    params: dict[str, str] = {
+        "TYPENAME": typename,
+        "LAYERS": typename.replace("_", " "),
+        "STYLES": "default",
+        "SERVICE": "WFS",
+        "REQUEST": "GetFeature",
+        "VERSION": "1.1.0",
+        "SRSNAME": "EPSG:4326",
+        "OUTPUTFORMAT": "application/json",
+        "FILTER": wfs_filter,
+    }
+    if map_path is not None:
+        params["MAP"] = map_path
+
+    r = requests.get(base_url, params=params, timeout=timeout)
+    r.raise_for_status()
+
+    data = r.json()
+    features = data.get("features", [])
+    if not features:
+        raise PoziWfsError(
+            f"No features found for point ({lat}, {lng}) in layer {typename}"
+        )
+
+    props = features[0].get("properties", {})
+    _LOGGER.debug("WFS query matched: %s", props)
+    return props
+
+
+def _point_in_geometry(lat: float, lng: float, geometry: dict) -> bool:
+    """Check if a point lies within a GeoJSON geometry (Polygon or MultiPolygon)."""
+    geom_type = geometry.get("type", "")
+    coords = geometry.get("coordinates", [])
+
+    if geom_type == "Polygon":
+        return _point_in_polygon(lng, lat, coords[0])
+    if geom_type == "MultiPolygon":
+        return any(_point_in_polygon(lng, lat, ring[0]) for ring in coords)
+
+    return False
+
+
+def _point_in_polygon(x: float, y: float, polygon: list[list[float]]) -> bool:
+    """Ray-casting point-in-polygon test.
+
+    Args:
+        x: X coordinate (longitude) of the point.
+        y: Y coordinate (latitude) of the point.
+        polygon: List of [x, y] coordinate pairs forming the polygon ring.
+
+    Returns:
+        True if the point is inside the polygon.
+    """
+    n = len(polygon)
+    inside = False
+
+    p1x, p1y = polygon[0]
+    for i in range(1, n + 1):
+        p2x, p2y = polygon[i % n]
+        if y > min(p1y, p2y):
+            if y <= max(p1y, p2y):
+                if x <= max(p1x, p2x):
+                    if p1y != p2y:
+                        xinters = (y - p1y) * (p2x - p1x) / (p2y - p1y) + p1x
+                    if p1x == p2x or x <= xinters:
+                        inside = not inside
+        p1x, p1y = p2x, p2y
+
+    return inside