@camstack/addon-waste-collections 0.0.0-stage → 0.1.2

package/python/vendor/waste_collection_schedule/service/Sepan.py

@@ -0,0 +1,334 @@
+import datetime
+import re
+
+import requests
+from bs4 import BeautifulSoup
+
+from waste_collection_schedule.exceptions import (
+    SourceArgumentNotFoundWithSuggestions,
+    SourceArgumentRequiredWithSuggestions,
+)
+
+# Shared client for the "SEPAN" (aka "ICHI System") waste-schedule platform,
+# used (via different deployments/base URLs) by sepan_remondis_pl.py,
+# zys_harmonogram_pl.py, alba_com_pl.py and ichisystem_eu.py. All four expose
+# the identical address-resolution API; see issue #6749 for the
+# consolidation rationale and issue #6763 for the /years+token report flow.
+
+POLISH_MONTHS = {
+    "STYCZEŃ": 1,
+    "LUTY": 2,
+    "MARZEC": 3,
+    "KWIECIEŃ": 4,
+    "MAJ": 5,
+    "CZERWIEC": 6,
+    "LIPIEC": 7,
+    "SIERPIEŃ": 8,
+    "WRZESIEŃ": 9,
+    "PAŹDZIERNIK": 10,
+    "LISTOPAD": 11,
+    "GRUDZIEŃ": 12,
+}
+
+_PL_DIACRITICS = str.maketrans(
+    {
+        "ą": "a",
+        "Ą": "a",
+        "ć": "c",
+        "Ć": "c",
+        "ę": "e",
+        "Ę": "e",
+        "ł": "l",
+        "Ł": "l",
+        "ń": "n",
+        "Ń": "n",
+        "ó": "o",
+        "Ó": "o",
+        "ś": "s",
+        "Ś": "s",
+        "ź": "z",
+        "Ź": "z",
+        "ż": "z",
+        "Ż": "z",
+    }
+)
+
+
+def normalize_pl(text: str) -> str:
+    """Normalise Polish text for matching: uppercase, drop diacritics,
+    collapse whitespace. This lets e.g. "Swiety Marcin" (diacritics
+    dropped, as many users type) still match a platform's "ŚWIĘTY MARCIN"
+    listing. See issue #6763."""
+    return re.sub(r"\s+", " ", text.translate(_PL_DIACRITICS).upper()).strip()
+
+
+class SepanClient:
+    """Client for the SEPAN waste-schedule platform.
+
+    `base_urls` is a list of candidate base URLs tried in order (first
+    success wins); this preserves each source's existing year-suffix /
+    fallback-URL retry behaviour without duplicating it three times.
+    """
+
+    def __init__(self, base_urls: list[str]):
+        self._base_urls = base_urls
+        self._resolved_base_url: str | None = None
+        self._resolved_number: dict | None = None
+
+    def resolve_address(self, city: str, street: str, number: str) -> str:
+        """Resolve city/street/number to an address id, trying each base URL
+        in turn. Returns the address id used by fetch_schedule()."""
+        last_error: Exception | None = None
+        for base_url in self._base_urls:
+            try:
+                return self._resolve_address_at(base_url, city, street, number)
+            except (
+                SourceArgumentNotFoundWithSuggestions,
+                SourceArgumentRequiredWithSuggestions,
+            ):
+                # A well-formed response that just didn't match: no point
+                # retrying against another base URL, the argument is wrong.
+                raise
+            except Exception as e:
+                last_error = e
+                continue
+        raise last_error or Exception("Could not resolve address: no base URL worked")
+
+    def _resolve_address_at(
+        self, base_url: str, city: str, street: str, number: str
+    ) -> str:
+        city = normalize_pl(city)
+        street = normalize_pl(street)
+        number = str(number).strip()
+
+        cities = requests.get(f"{base_url}/addresses/cities", timeout=30).json()
+        city_suggestions = [c["value"] for c in cities]
+        if not city:
+            raise SourceArgumentRequiredWithSuggestions(
+                "city", "Select your city.", city_suggestions
+            )
+        city_match = next((c for c in cities if normalize_pl(c["value"]) == city), None)
+        if not city_match:
+            raise SourceArgumentNotFoundWithSuggestions("city", city, city_suggestions)
+        city_id = city_match["id"]
+
+        streets = requests.get(
+            f"{base_url}/addresses/streets/{city_id}", timeout=30
+        ).json()
+        street_suggestions = [s["value"] for s in streets]
+        if not street:
+            raise SourceArgumentRequiredWithSuggestions(
+                "street", "Select your street.", street_suggestions
+            )
+        street_match = next(
+            (s for s in streets if normalize_pl(s["value"]) == street), None
+        )
+        if not street_match:
+            raise SourceArgumentNotFoundWithSuggestions(
+                "street", street, street_suggestions
+            )
+        street_id = street_match["id"]
+
+        numbers = requests.get(
+            f"{base_url}/addresses/numbers/{city_id}/{street_id}", timeout=30
+        ).json()
+        number_suggestions = [str(n["value"]) for n in numbers]
+        if not number:
+            raise SourceArgumentRequiredWithSuggestions(
+                "number", "Select your house number.", number_suggestions
+            )
+        number_match = next(
+            (n for n in numbers if str(n["value"]).strip() == number), None
+        )
+        if not number_match:
+            raise SourceArgumentNotFoundWithSuggestions(
+                "number", number, number_suggestions
+            )
+
+        self._resolved_base_url = base_url
+        self._resolved_number = number_match
+        return number_match["id"]
+
+    def _fetch_years(self) -> list[dict] | None:
+        """Return this deployment's report years (list of {"id", "value"}),
+        or None if it doesn't expose a `/years` endpoint (or the endpoint
+        errors/returns something unexpected). Older/simpler SEPAN
+        deployments only expose the current report via `/reports` without a
+        `yearId`; newer ones (see issue #6763) expose `/years` plus a
+        per-address `token` (already returned alongside the address id by
+        `/addresses/numbers/...`) to fetch any year's report on demand."""
+        if self._resolved_base_url is None:
+            return None
+        try:
+            resp = requests.get(f"{self._resolved_base_url}/years", timeout=30)
+            resp.raise_for_status()
+            years = resp.json()
+        except Exception:
+            return None
+        if not isinstance(years, list) or not years:
+            return None
+        return years
+
+    def fetch_report_html(
+        self,
+        address_id: str,
+        year_id: str | None = None,
+        token: str | None = None,
+    ) -> str:
+        """Fetch the raw HTML report for a resolved address id.
+
+        If `year_id` (and the matching `token`, as returned alongside the
+        address id by `resolve_address()`'s underlying
+        `/addresses/numbers/...` call) are given, the report for that
+        specific year is requested; otherwise the deployment's
+        default/current report is requested (previous behaviour, still used
+        by deployments without a `/years` endpoint).
+
+        Parsing is intentionally left to the caller: the known SEPAN
+        deployments don't all render the report table the same way (see
+        `parse_month_name_table` for the format shared by zys_harmonogram_pl,
+        alba_com_pl and ichisystem_eu; sepan_remondis_pl uses its own
+        positional parser).
+        """
+        base_url = self._resolved_base_url
+        if base_url is None:
+            raise Exception("fetch_report_html() called before resolve_address()")
+
+        params: dict[str, str | None] = {"type": "html", "id": address_id}
+        if year_id is not None:
+            params["responseType"] = "json"
+            params["token"] = token
+            params["yearId"] = year_id
+
+        report = requests.get(
+            f"{base_url}/reports",
+            params=params,
+            timeout=30,
+        ).json()
+        if report.get("status") != "success":
+            raise Exception("fetch report failed")
+
+        return requests.get(report["filePath"], timeout=30).content.decode("utf-8")
+
+    def fetch_schedule(self, address_id: str) -> list[tuple[datetime.date, str]]:
+        """Fetch and parse the HTML report(s) using the shared month-name
+        table format (see `parse_month_name_table`).
+
+        If the deployment exposes a `/years` endpoint, every available
+        year's report is fetched (using the `token` returned alongside the
+        address id by `resolve_address()`) and merged, deduplicating
+        entries. This is more robust across a calendar-year rollover than
+        guessing a year-suffixed base URL (see issue #6763). Deployments
+        without a `/years` endpoint fall back to the single "current"
+        report returned by `/reports` (previous behaviour).
+        """
+        years = self._fetch_years()
+        if not years:
+            return parse_month_name_table(self.fetch_report_html(address_id))
+
+        token = (self._resolved_number or {}).get("token")
+        entries: list[tuple[datetime.date, str]] = []
+        seen: set[tuple[datetime.date, str]] = set()
+        for year_entry in years:
+            try:
+                year_value = int(year_entry["value"])
+            except (KeyError, TypeError, ValueError):
+                continue
+            try:
+                html = self.fetch_report_html(
+                    address_id, year_id=year_entry.get("id"), token=token
+                )
+            except Exception:
+                continue
+            for entry in parse_month_name_table(html, year=year_value):
+                if entry in seen:
+                    continue
+                seen.add(entry)
+                entries.append(entry)
+        return entries
+
+
+def parse_month_name_table(
+    html: str, year: int | None = None
+) -> list[tuple[datetime.date, str]]:
+    """Parse a SEPAN report table whose rows are one comma-separated-days
+    cell per waste-type column, keyed by a leading "<Month>" or
+    "<Month> <Year>" cell (the format used by zys_harmonogram_pl,
+    alba_com_pl and ichisystem_eu's deployments).
+
+    Column names are read from the header row itself (rather than assumed
+    by position) because some deployments have a two-row <thead> (an outer
+    grouping row plus the real per-column names) — only the row that
+    actually contains "Miesiąc" is used for header names, so the extra
+    grouping row's cells don't get misread as data columns.
+
+    Some deployments (e.g. ichisystem_eu) render just the month name
+    without a year in each row, relying on the report having been requested
+    for a specific year (see `SepanClient.fetch_schedule`'s /years+token
+    flow); pass that year explicitly via `year` for those. If a row's first
+    cell does include a year (as zys_harmonogram_pl / alba_com_pl do), that
+    embedded year always takes precedence over `year`.
+    """
+    soup = BeautifulSoup(html, "html.parser")
+
+    table = None
+    headers: list[str] = []
+    for t in soup.find_all("table"):
+        for header_row in t.find_all("tr"):
+            row_th_texts = [th.get_text(strip=True) for th in header_row.find_all("th")]
+            if "Miesiąc" in row_th_texts:
+                table = t
+                headers = row_th_texts[1:]
+                break
+        if table is not None:
+            break
+
+    if table is None:
+        raise Exception("Schedule table not found in HTML response")
+
+    entries: list[tuple[datetime.date, str]] = []
+    tbody = table.find("tbody")
+    rows = tbody.find_all("tr") if tbody else table.find_all("tr")
+    for row in rows:
+        cells = row.find_all("td")
+        if not cells:
+            continue
+
+        parts = cells[0].get_text(strip=True).split()
+        if not parts:
+            continue
+
+        month = POLISH_MONTHS.get(parts[0].upper())
+        if not month:
+            continue
+
+        if len(parts) >= 2 and parts[-1].isdigit():
+            row_year = int(parts[-1])
+        elif year is not None:
+            row_year = year
+        else:
+            continue
+
+        for i, cell in enumerate(cells[1:]):
+            if i >= len(headers):
+                break
+            days_text = cell.get_text(strip=True)
+            if not days_text:
+                continue
+
+            waste_type = headers[i]
+            for day_str in days_text.split(","):
+                day_str = day_str.strip()
+                if not day_str.isdigit():
+                    continue
+                try:
+                    entries.append(
+                        (
+                            datetime.date(row_year, month, int(day_str)),
+                            waste_type,
+                        )
+                    )
+                except ValueError:
+                    pass
+
+    return entries

package/python/vendor/waste_collection_schedule/service/SiteparkIES.py

@@ -0,0 +1,231 @@
+import datetime
+
+from bs4 import BeautifulSoup
+from curl_cffi import requests
+
+from waste_collection_schedule.exceptions import (
+    SourceArgAmbiguousWithSuggestions,
+    SourceArgumentNotFoundWithSuggestions,
+)
+from waste_collection_schedule.service.ICS import ICS
+
+
+def _mojibake(term: str) -> str:
+    """Re-encode a term to work around a server-side double UTF-8 decode.
+
+    Some Sitepark installations (e.g. hilchenbach.de) mis-decode the
+    ``term`` query parameter, so a correctly UTF-8-encoded term containing
+    umlauts or "ß" never matches (e.g. "Am Bühl" or "Dammstraße" return no
+    results at all). Re-encoding the term as UTF-8 bytes and decoding those
+    bytes as Latin-1 reproduces the same mis-decoding the server applies,
+    which cancels it out and lets the match succeed. Other installations
+    (e.g. ab-peine.de) decode correctly already and this would break them,
+    so it is only tried as a fallback, not applied unconditionally.
+    """
+    return term.encode("utf-8").decode("latin-1")
+
+
+def match_icon(waste_type: str, icon_map: dict):
+    """Return the first icon whose key is contained in the waste type.
+
+    The Sitepark feeds often append a rhythm suffix to the waste type
+    (e.g. "Restmülltonne 14-täglichen Rhythmus"), so a substring match is
+    used instead of an exact lookup.
+    """
+    lowered = waste_type.lower()
+    for key, icon in icon_map.items():
+        if key.lower() in lowered:
+            return icon
+    return None
+
+
+class SiteparkIES:
+    """Shared client for the Sitepark IES / abto waste-calendar platform.
+
+    A number of German municipalities run the same Sitepark "iKISS" CMS with
+    the abto waste module. They all share two endpoints:
+
+    * ``/output/autocomplete.php?out=json&type=abto&select=2&term=<street>``
+      returns ``[[pois, "Street (Place)"], ...]`` for a street search. Some
+      installations require a ``refid`` (district/municipality id); others
+      return all streets across the district when none is given and
+      disambiguate by the "(Place)" suffix.
+    * ``/output/options.php?ModID=48&call=ical&pois=<pois>`` returns an ICS
+      calendar for the resolved point of interest.
+
+    Every request must send a ``Referer`` header pointing at the base domain,
+    otherwise the server answers ``403``.
+    """
+
+    def __init__(
+        self,
+        base_url: str,
+        *,
+        refid: str | None = None,
+        download_params: dict | None = None,
+        ics: ICS | None = None,
+    ):
+        self._base_url = base_url.rstrip("/")
+        self._refid = refid
+        self._download_params = download_params or {}
+        self._ics = ics or ICS()
+        self._session = requests.Session(impersonate="chrome")
+
+    @property
+    def _headers(self) -> dict:
+        return {
+            "Referer": self._base_url + "/",
+            "X-Requested-With": "XMLHttpRequest",
+        }
+
+    def autocomplete(self, term: str | None, refid: str | None = None) -> list[list]:
+        """Query the abto autocomplete endpoint and return ``[[pois, label], ...]``."""
+        params = {
+            "out": "json",
+            "type": "abto",
+            "select": "2",
+            "term": term or "",
+        }
+        rid = refid if refid is not None else self._refid
+        if rid is not None:
+            params["refid"] = rid
+
+        r = self._session.get(
+            f"{self._base_url}/output/autocomplete.php",
+            params=params,
+            headers=self._headers,
+            timeout=30,
+        )
+        r.raise_for_status()
+        # The endpoint answers JSON ``null`` (instead of ``[]``) for some
+        # searches, e.g. terms with parentheses or when a refid is required.
+        data = r.json()
+        return data if isinstance(data, list) else []
+
+    def resolve_refid(
+        self,
+        ort: str,
+        page_url: str,
+        value_prefix: str | None = None,
+    ) -> str:
+        """Resolve an "Ort" to its refid via the ``sf_locid`` dropdown on a page."""
+        r = self._session.get(
+            page_url, headers={"Referer": self._base_url + "/"}, timeout=30
+        )
+        r.raise_for_status()
+
+        soup = BeautifulSoup(r.text, "html.parser")
+        select = soup.find("select", id="sf_locid")
+        options: dict = {}
+        if select is not None:
+            for option in select.find_all("option"):
+                value = str(option.get("value") or "").strip()
+                name = option.get_text(strip=True)
+                if not value or not name:
+                    continue
+                if value_prefix is not None and not value.startswith(value_prefix):
+                    continue
+                options[name] = value
+
+        # exact match first, then case-insensitive containment
+        if ort in options:
+            return options[ort]
+        for name, value in options.items():
+            if ort.lower() in name.lower():
+                return value
+
+        raise SourceArgumentNotFoundWithSuggestions("ort", ort, list(options.keys()))
+
+    def get_pois(
+        self,
+        strasse: str | None = None,
+        ort: str | None = None,
+        refid: str | None = None,
+    ) -> str:
+        """Resolve a street (and optional place) to a single pois id."""
+        results = self._query(strasse, refid=refid)
+
+        candidates = results
+        if strasse:
+            filtered = [item for item in results if strasse.lower() in item[1].lower()]
+            if filtered:
+                candidates = filtered
+        if ort:
+            by_ort = [item for item in candidates if ort.lower() in item[1].lower()]
+            if by_ort:
+                candidates = by_ort
+
+        if not candidates:
+            # Re-query without a term to build a helpful suggestion list.
+            suggestions = [item[1] for item in self.autocomplete("", refid=refid)]
+            raise SourceArgumentNotFoundWithSuggestions(
+                "strasse", strasse or "", suggestions or [item[1] for item in results]
+            )
+
+        if len(candidates) > 1:
+            # Prefer an exact match on the street part (the bit before " (Place)").
+            wanted = (strasse or "").strip().lower()
+            exact = [
+                item
+                for item in candidates
+                if item[1].split(" (")[0].strip().lower() == wanted
+            ]
+            if len(exact) == 1:
+                return exact[0][0]
+            raise SourceArgAmbiguousWithSuggestions(
+                "strasse", strasse or "", [item[1] for item in candidates]
+            )
+
+        return candidates[0][0]
+
+    def _query(self, term: str | None, refid: str | None) -> list[list]:
+        """Autocomplete with a fallback for terms the endpoint cannot match.
+
+        The abto endpoint returns an empty result for some inputs: umlauts
+        or "ß" on installations affected by the double-decode bug (see
+        ``_mojibake``), or a trailing space, parentheses or a comma on any
+        installation. When that happens, retry with a more tolerant form of
+        the term and let the caller filter the (broader) result set locally.
+        """
+        results = self.autocomplete(term, refid=refid)
+        if results or not term:
+            return results
+
+        if not term.isascii():
+            results = self.autocomplete(_mojibake(term), refid=refid)
+            if results:
+                return results
+
+        short = term
+        for sep in ("(", ",", "  "):
+            short = short.split(sep)[0]
+        short = short.strip()
+        if short and short != term:
+            results = self.autocomplete(short, refid=refid)
+        return results
+
+    def fetch_ics(self, pois: str) -> list[tuple[datetime.date, str]]:
+        """Download and parse the ICS calendar for a pois id."""
+        params = {"ModID": "48", "call": "ical", "pois": pois}
+        params.update(self._download_params)
+
+        r = self._session.get(
+            f"{self._base_url}/output/options.php",
+            params=params,
+            headers={"Referer": self._base_url + "/"},
+            timeout=30,
+        )
+        r.raise_for_status()
+        return self._ics.convert(r.text)
+
+    def fetch(
+        self,
+        strasse: str | None = None,
+        ort: str | None = None,
+        pois: str | None = None,
+        refid: str | None = None,
+    ) -> list[tuple[datetime.date, str]]:
+        """Resolve the pois (unless given directly) and return parsed dates."""
+        if not pois:
+            pois = self.get_pois(strasse=strasse, ort=ort, refid=refid)
+        return self.fetch_ics(pois)

package/python/vendor/waste_collection_schedule/service/WasteInfo.py

@@ -0,0 +1,83 @@
+"""Register matching for councils on the waste-info.com.au platform.
+
+Those councils expose the same v1 API: ``localities.json``, then
+``streets.json?locality=`` and ``properties.json?street=``, each answering with
+``{"name": ..., "id": ...}`` rows that a source has to match by name. The names
+are the council's own register wording, and matching them literally is what
+makes an ordinary address look unserviced:
+
+- case and spacing vary ("berala" and "Merrylands  Road" are what people type,
+  "Berala" and "Merrylands Road" is what the register holds);
+- a property row may carry a building name between the number and the street
+  ("4-12 five dock library Garfield Street Five Dock"), so the number, street
+  and suburb are all correct and the row still does not compare equal.
+
+Sources with their own richer matching (brisbane, impactapps, redland) are
+untouched; this is for the ones that compared raw strings.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+
+
+def norm(value: object) -> str:
+    """Register-comparable form: single-spaced and case-folded.
+
+    Only ``None`` is empty. Street numbers arrive as ``int`` as well as ``str``,
+    so ``0`` has to keep its own value rather than fall back to "".
+    """
+    if value is None:
+        return ""
+    return " ".join(str(value).split()).casefold()
+
+
+def same(a: object, b: object) -> bool:
+    """True when two register names differ only by case or spacing."""
+    return norm(a) == norm(b)
+
+
+def property_matches(
+    register_name: object, street_number: object, street_name: object, suburb: object
+) -> bool:
+    """True when a ``properties.json`` row is the address that was asked for.
+
+    Accepts the register's canonical ``"<number> <street> <suburb>"`` and the
+    same row with a building name inserted after the number. The number must
+    still match in full, so "4-12" does not answer for "1m/4-12".
+    """
+    name = norm(register_name)
+    if name == norm(f"{street_number} {street_name} {suburb}"):
+        return True
+    return name.startswith(norm(street_number) + " ") and name.endswith(
+        norm(f"{street_name} {suburb}")
+    )
+
+
+def street_number_suggestions(
+    register_names: Iterable[object], street_name: object
+) -> list[str]:
+    """The number (and building name) of every register row on that street.
+
+    Suggesting a street number only helps if the rows are found the same way
+    ``property_matches`` finds them, so the street is located on the case- and
+    spacing-insensitive form while the register's own wording is what gets
+    suggested. The last occurrence wins, because a building name can repeat the
+    street ("1 Garfield Street Cafe Garfield Street Five Dock").
+    """
+    wanted = norm(street_name).split()
+    if not wanted:
+        return []
+
+    suggestions = []
+    for register_name in register_names:
+        words = str(register_name if register_name is not None else "").split()
+        folded = [word.casefold() for word in words]
+        for start in range(len(folded) - len(wanted), -1, -1):
+            if folded[start : start + len(wanted)] != wanted:
+                continue
+            number = " ".join(words[:start])
+            if number:
+                suggestions.append(number)
+            break
+    return suggestions