@camstack/addon-waste-collections 0.0.0-stage → 0.1.2

package/python/vendor/waste_collection_schedule/source/feuerthalen_ch.py

@@ -0,0 +1,304 @@
+import io
+import re
+from datetime import date, timedelta
+from urllib.parse import urljoin
+
+import requests
+from bs4 import BeautifulSoup
+from pdfminer.high_level import extract_pages
+from pdfminer.layout import LTChar, LTRect, LTTextLine
+from waste_collection_schedule import Collection, Icons
+
+TITLE = "Gemeinde Feuerthalen"
+DESCRIPTION = (
+    "Source for waste collection in Feuerthalen, Canton of Zurich, Switzerland."
+)
+URL = "https://www.feuerthalen.ch/umwelt/entsorgung/abfall.html/275"
+COUNTRY = "ch"
+
+SOURCE_CODEOWNERS = ["@oh-supra"]
+
+TEST_CASES: dict[str, dict] = {
+    "Feuerthalen": {},
+}
+
+_PRODUCT_URL = (
+    "https://www.feuerthalen.ch/verwaltung/online-schalter.html/384/product/7"
+)
+
+ICON_MAP = {
+    "Hauskehricht": Icons.GENERAL_WASTE,
+    "Gruengut": Icons.ORGANIC,
+    "Werkhof": Icons.RECYCLING,
+    "AltpapierKarton": Icons.PAPER,
+    "Sonderabfall": Icons.HAZARDOUS,
+}
+
+# Pretty labels for the internal (ASCII-safe) waste-type keys used while
+# matching cell colours.
+DISPLAY_NAME = {
+    "Hauskehricht": "Hauskehricht",
+    "Gruengut": "Grüngut",
+    "Werkhof": "Direktentsorgung Werkhof",
+    "AltpapierKarton": "Altpapier/Karton",
+    "Sonderabfall": "Sonderabfälle/Giftsammlung",
+}
+
+MONTHS_DE = [
+    "Januar", "Februar", "März", "April", "Mai", "Juni",
+    "Juli", "August", "September", "Oktober", "November", "Dezember",
+]  # fmt: skip
+
+WEEKDAYS_DE = {
+    "Montag": 0,
+    "Dienstag": 1,
+    "Mittwoch": 2,
+    "Donnerstag": 3,
+    "Freitag": 4,
+    "Samstag": 5,
+    "Sonntag": 6,
+}
+_WEEKDAY_RE = "|".join(WEEKDAYS_DE)
+
+# The calendar grid encodes each waste type as a small coloured square behind
+# the day-of-month number; every municipality-wide (non-Hauskehricht) date is
+# read off these colours. Hauskehricht itself has no colour (it is simply
+# "every Monday", called out separately in the page-2 legend text together
+# with its holiday-shift exceptions), so it is expanded programmatically.
+_WASTE_COLORS: dict[tuple[float, float, float], str] = {
+    (0.0, 0.483, 0.735): "Hauskehricht",
+    (0.259, 0.606, 0.24): "Gruengut",
+    (0.717, 0.65, 0.509): "Werkhof",
+    (1.0, 0.795, 0.0): "AltpapierKarton",
+    (0.896, 0.199, 0.532): "Sonderabfall",
+}
+_COLOR_TOL = 0.015
+
+
+def _colors_close(c1, c2: tuple[float, float, float]) -> bool:
+    # pdfminer reports non_stroking_color as None, a plain float (DeviceGray)
+    # or a tuple/list depending on the colour space, so guard the type.
+    if not isinstance(c1, (tuple, list)) or len(c1) != len(c2):
+        return False
+    return all(abs(a - b) < _COLOR_TOL for a, b in zip(c1, c2, strict=True))
+
+
+def _walk(obj):
+    yield obj
+    if hasattr(obj, "__iter__"):
+        for child in obj:
+            yield from _walk(child)
+
+
+def _extract_words(page) -> list[dict]:
+    """Extract words with bounding boxes from a pdfminer page layout."""
+    words: list[dict] = []
+    for obj in _walk(page):
+        if not isinstance(obj, LTTextLine):
+            continue
+        current: list[LTChar] = []
+        for c in obj:
+            if not isinstance(c, LTChar):
+                continue
+            if current and (
+                c.x0 - current[-1].x1 > max(1.2, 0.35 * c.size)
+                or not c.get_text().strip()
+            ):
+                _flush_word(words, current)
+                current = []
+            if c.get_text().strip():
+                current.append(c)
+        _flush_word(words, current)
+    return words
+
+
+def _flush_word(words: list[dict], chars: list) -> None:
+    if not chars:
+        return
+    text = "".join(c.get_text() for c in chars).strip()
+    if not text:
+        return
+    words.append(
+        {
+            "text": text,
+            "x0": min(c.x0 for c in chars),
+            "x1": max(c.x1 for c in chars),
+            "y0": min(c.y0 for c in chars),
+            "y1": max(c.y1 for c in chars),
+        }
+    )
+
+
+def _find_year(words: list[dict]) -> int | None:
+    text = " ".join(w["text"] for w in sorted(words, key=lambda w: -w["y0"]))
+    m = re.search(r"ABFALLKALENDER\s+(\d{4})", text)
+    return int(m.group(1)) if m else None
+
+
+def _find_month_headers(words: list[dict]) -> list[dict]:
+    return [w for w in words if w["text"] in MONTHS_DE]
+
+
+def _nearest_month(rect: LTRect, month_headers: list[dict]) -> str | None:
+    cx, cy = (rect.x0 + rect.x1) / 2, (rect.y0 + rect.y1) / 2
+    best, best_dist = None, None
+    for h in month_headers:
+        if h["y0"] < cy:  # a month header only ever sits above its own grid
+            continue
+        dist = (cx - h["x0"]) ** 2 + (cy - h["y0"]) ** 2
+        if best_dist is None or dist < best_dist:
+            best_dist, best = dist, h["text"]
+    return best
+
+
+def _day_number(rect: LTRect, words: list[dict]) -> int | None:
+    cx, cy = (rect.x0 + rect.x1) / 2, (rect.y0 + rect.y1) / 2
+    for w in words:
+        if (
+            w["x0"] <= cx <= w["x1"]
+            and w["y0"] <= cy <= w["y1"]
+            and w["text"].isdigit()
+        ):
+            return int(w["text"])
+    return None
+
+
+def _parse_colored_grid(page, words: list[dict], year: int) -> set[tuple[str, date]]:
+    month_headers = _find_month_headers(words)
+    events: set[tuple[str, date]] = set()
+    for rect in _walk(page):
+        if not isinstance(rect, LTRect):
+            continue
+        waste_type = next(
+            (
+                name
+                for color, name in _WASTE_COLORS.items()
+                if name != "Hauskehricht"
+                and _colors_close(rect.non_stroking_color, color)
+            ),
+            None,
+        )
+        if waste_type is None:
+            continue
+        month_name = _nearest_month(rect, month_headers)
+        day = _day_number(rect, words)
+        if month_name is None or day is None:
+            continue
+        try:
+            events.add((waste_type, date(year, MONTHS_DE.index(month_name) + 1, day)))
+        except ValueError:
+            continue
+    return events
+
+
+def _expand_weekly_monday(year: int) -> set[date]:
+    d = date(year, 1, 1)
+    d += timedelta(days=(0 - d.weekday()) % 7)
+    dates = set()
+    while d.year == year:
+        dates.add(d)
+        d += timedelta(days=7)
+    return dates
+
+
+def _parse_hauskehricht_shifts(words: list[dict]) -> list[date]:
+    """Parse holiday replacement dates, e.g. 'Mittwoch, 08.04.2026 statt Ostermontag'.
+
+    Returns the collection dates that replace their week's Monday collection.
+    """
+    lines: dict[int, list[dict]] = {}
+    for w in words:
+        lines.setdefault(round(w["y0"]), []).append(w)
+    text_lines = []
+    for _, ws in sorted(lines.items(), key=lambda kv: -kv[0]):
+        ws.sort(key=lambda w: w["x0"])
+        text_lines.append(" ".join(w["text"] for w in ws))
+    text = "\n".join(text_lines)
+
+    shifts = []
+    for m in re.finditer(
+        rf"({_WEEKDAY_RE}),\s*(\d{{1,2}})\.(\d{{1,2}})\.(\d{{4}})\s+statt", text
+    ):
+        _weekday_name, day, month, year_str = m.groups()
+        try:
+            shifts.append(date(int(year_str), int(month), int(day)))
+        except ValueError:
+            continue
+    return shifts
+
+
+def _parse_hauskehricht(pages_words: list[list[dict]], year: int) -> set[date]:
+    mondays = _expand_weekly_monday(year)
+    for page_words in pages_words:
+        for shift_date in _parse_hauskehricht_shifts(page_words):
+            offset = shift_date.weekday()  # days after that week's Monday
+            original_monday = shift_date - timedelta(days=offset)
+            mondays.discard(original_monday)
+            mondays.add(shift_date)
+    return mondays
+
+
+class Source:
+    def __init__(self) -> None:
+        # No parameters needed - Feuerthalen has common dates for the entire
+        # municipality.
+        pass
+
+    def _find_pdf_url(self, session: requests.Session) -> str:
+        r = session.get(_PRODUCT_URL, timeout=30)
+        r.raise_for_status()
+        soup = BeautifulSoup(r.text, "html.parser")
+
+        link = soup.select_one('a[href*=".pdf"]')
+        if link is None or not link.get("href"):
+            raise ValueError(
+                "No waste calendar PDF link found on the Feuerthalen online "
+                "counter page. The website structure may have changed."
+            )
+        return urljoin(_PRODUCT_URL, str(link["href"]))
+
+    def fetch(self) -> list[Collection]:
+        session = requests.Session()
+        session.headers.update({"User-Agent": "Mozilla/5.0"})
+
+        pdf_url = self._find_pdf_url(session)
+        r = session.get(pdf_url, timeout=30)
+        r.raise_for_status()
+
+        pages = list(extract_pages(io.BytesIO(r.content)))
+        if not pages:
+            raise ValueError("The waste calendar PDF has no pages.")
+
+        page1_words = _extract_words(pages[0])
+        year = _find_year(page1_words)
+        if year is None:
+            raise ValueError(
+                "Could not determine the calendar year from the waste "
+                "calendar PDF. The website structure may have changed."
+            )
+
+        grid_events = _parse_colored_grid(pages[0], page1_words, year)
+        if not grid_events:
+            raise ValueError(
+                "No waste collection events found in the waste calendar PDF. "
+                "The website structure may have changed."
+            )
+
+        # The exception notes live on page 2 today, but scan every page after
+        # the calendar grid so a future re-layout does not silently drop them.
+        note_pages = [_extract_words(page) for page in pages[1:]]
+        hauskehricht_dates = _parse_hauskehricht(note_pages, year)
+
+        events: set[tuple[str, date]] = set(grid_events)
+        events.update(("Hauskehricht", d) for d in hauskehricht_dates)
+
+        return [
+            Collection(
+                date=collection_date,
+                t=DISPLAY_NAME[waste_type],
+                icon=ICON_MAP.get(waste_type),
+            )
+            for waste_type, collection_date in sorted(
+                events, key=lambda e: (e[1], e[0])
+            )
+        ]

package/python/vendor/waste_collection_schedule/source/fev_se.py

@@ -0,0 +1,156 @@
+import json
+import re
+from datetime import date, datetime, timedelta
+from zoneinfo import ZoneInfo
+
+import requests
+from waste_collection_schedule import Collection, Icons  # type: ignore[attr-defined]
+from waste_collection_schedule.exceptions import (
+    SourceArgumentNotFound,
+    SourceArgumentNotFoundWithSuggestions,
+)
+
+TITLE = "Falu Energi & Vatten (FEV)"
+DESCRIPTION = (
+    "Source for Falu Energi & Vatten waste collection schedule, Falun, Sweden."
+)
+URL = "https://fev.se"
+COUNTRY = "se"
+TEST_CASES = {
+    "Rådmansvägen 3": {"address": "Rådmansvägen 3"},
+    "Rådmansvägen 5": {"address": "Rådmansvägen 5"},
+}
+
+ICON_MAP = {
+    "Restavfall": Icons.GENERAL_WASTE,
+    "Matavfall": Icons.BIO_KITCHEN,
+    "Trädgårdsavfall": Icons.GARDEN,
+}
+
+PAGE_URL = "https://fev.se/atervinning/sophamtning.html"
+
+TIMEZONE = ZoneInfo("Europe/Stockholm")
+
+# Matches every `AppRegistry.registerInitialState('<portletId>', {...});`
+# call embedded in the server-rendered page. The waste search widget's
+# portlet id is treated as an implementation detail and not hardcoded here;
+# instead the correct payload is identified by the presence of the
+# "formAddressText" key, which is unique to the fetch-planner widget and is
+# more resilient to a future re-deploy of the widget under a different
+# portlet id.
+STATE_RE = re.compile(
+    r"registerInitialState\('[^']+',(\{.*?\})\);",
+    re.DOTALL,
+)
+
+# FEV's public fetch planner only publishes the next two occurrences per
+# container ("pickupDate" and "nextPickupDate"). Rather than hardcode a fixed
+# cadence, the interval between those two dates is measured per container
+# and used to extrapolate further future collections.
+NUMBER_OF_COLLECTIONS_TO_GENERATE = 12
+
+HOW_TO_GET_ARGUMENTS_DESCRIPTION = {
+    "en": "Go to https://fev.se/atervinning/sophamtning.html, search for your "
+    "address and copy the street name and house number exactly as shown in "
+    "the results, e.g. 'Rådmansvägen 3'.",
+}
+
+PARAM_TRANSLATIONS = {
+    "en": {
+        "address": "Address",
+    },
+}
+
+PARAM_DESCRIPTIONS = {
+    "en": {
+        "address": "Street address including house number, exactly as it "
+        "appears on fev.se, e.g. 'Rådmansvägen 3'.",
+    },
+}
+
+
+def _to_local_date(iso_timestamp: str) -> date:
+    dt_utc = datetime.fromisoformat(iso_timestamp.replace("Z", "+00:00"))
+    return dt_utc.astimezone(TIMEZONE).date()
+
+
+class Source:
+    def __init__(self, address: str):
+        self._address = address.strip()
+
+    def fetch(self) -> list[Collection]:
+        r = requests.get(
+            PAGE_URL,
+            params={"q": self._address},
+            headers={"User-Agent": "Mozilla/5.0"},
+            timeout=30,
+        )
+        r.raise_for_status()
+
+        state = None
+        for match in STATE_RE.finditer(r.text):
+            try:
+                payload = json.loads(match.group(1))
+            except json.JSONDecodeError:
+                continue
+            if "formAddressText" in payload:
+                state = payload
+                break
+
+        if state is None:
+            raise SourceArgumentNotFound(
+                "address",
+                self._address,
+                message_addition="the schedule page could not be parsed, it may have changed format.",
+            )
+
+        containers = state.get("containers") or []
+
+        if not containers:
+            hits = state.get("hits") or []
+            if hits:
+                suggestions = sorted(
+                    {
+                        f"{hit['PickupAddress']}, {hit.get('PickupCity', '').title()}"
+                        for hit in hits
+                        if hit.get("PickupAddress")
+                    }
+                )
+                raise SourceArgumentNotFoundWithSuggestions(
+                    "address", self._address, suggestions
+                )
+            raise SourceArgumentNotFound(
+                "address",
+                self._address,
+            )
+
+        entries: list[Collection] = []
+        for container in containers:
+            waste_type = container.get("typeText", "Unknown")
+            icon = ICON_MAP.get(waste_type)
+
+            pickup_iso = container.get("pickupDateIso")
+            if not pickup_iso:
+                continue
+            pickup_date = _to_local_date(pickup_iso)
+
+            next_pickup_iso = container.get("nextPickupDateIso")
+            interval_days = None
+            if container.get("hasNextPickup") and next_pickup_iso:
+                next_pickup_date = _to_local_date(next_pickup_iso)
+                interval_days = (next_pickup_date - pickup_date).days
+
+            if not interval_days:
+                entries.append(Collection(date=pickup_date, t=waste_type, icon=icon))
+                continue
+
+            for i in range(NUMBER_OF_COLLECTIONS_TO_GENERATE):
+                entries.append(
+                    Collection(
+                        date=pickup_date + timedelta(days=interval_days * i),
+                        t=waste_type,
+                        icon=icon,
+                    )
+                )
+
+        return entries

package/python/vendor/waste_collection_schedule/source/fife_gov_uk.py

@@ -0,0 +1,96 @@
+from datetime import datetime
+
+import requests
+from waste_collection_schedule import Collection, Icons  # type: ignore[attr-defined]
+
+TITLE = "Fife Council"
+DESCRIPTION = "Source for Fife Council."
+URL = "https://www.fife.gov.uk"
+TEST_CASES = {
+    "SANDYHILL ROAD, ST ANDREWS": {"uprn": 320069186},
+    "CERES ROAD, PITSCOTTIE": {"uprn": 320063641},
+    "SHORE ROAD, BALMALCOLM": {"uprn": "320083539"},
+    "CANMORE STREET, DUNFERMLINE": {"uprn": "320101510"},
+}
+ICON_MAP = {
+    "Blue": Icons.GENERAL_WASTE,
+    "Glass": Icons.GLASS,
+    "Brown": Icons.ORGANIC,
+    "Grey": Icons.PAPER,
+    "Green": Icons.RECYCLING,
+}
+
+API_BASE_URL = "https://fife.form.uk.empro.verintcloudservices.com/api"
+API_URL = f"{API_BASE_URL}/custom"
+AUTH_URL = f"{API_BASE_URL}/citizen"
+
+DATE_FORMAT = "%A, %B %d, %Y"
+REQUEST_TIMEOUT = 10  # seconds
+
+
+class Source:
+    def __init__(self, uprn: str | int):
+        self._uprn: str | int = str(uprn)
+
+    def fetch(self):
+        """Fetch bin calendar collections for the given UPRN from Fife Council."""
+        session = requests.Session()
+
+        # Get authentication token
+        auth_response = session.get(
+            AUTH_URL,
+            params={"preview": "false", "locale": "en"},
+            timeout=REQUEST_TIMEOUT,
+        )
+        auth_response.raise_for_status()
+
+        auth_token = auth_response.headers.get("Authorization")
+        if not auth_token:
+            raise ValueError("No authorization token received from API")
+
+        session.headers.update({"Authorization": auth_token})
+
+        # Get collection data
+        api_response = session.post(
+            API_URL,
+            params={
+                "action": "powersuite_bin_calendar_collections",
+                "actionedby": "bin_calendar",
+                "loadform": True,
+                "access": "citizen",
+                "locale": "en",
+            },
+            json={"name": "bin_calendar", "data": {"uprn": self._uprn}},
+            timeout=REQUEST_TIMEOUT,
+        )
+        api_response.raise_for_status()
+
+        # Parse response data
+        response_data = api_response.json()
+        data = response_data.get("data", {})
+
+        if data.get("results_returned") == "false":
+            raise ValueError(f"No results returned for UPRN: {self._uprn}")
+
+        collections = data.get("tab_collections", [])
+        if not collections:
+            raise ValueError(f"No collection data found for UPRN: {self._uprn}")
+
+        entries = []
+        for collection in collections:
+            try:
+                date_str = collection.get("date")
+                if not date_str:
+                    continue
+
+                date = datetime.strptime(date_str, DATE_FORMAT).date()
+                collection_type = collection.get("type", "")
+                colour = collection.get("colour", "")
+                icon = ICON_MAP.get(colour)
+
+                entries.append(Collection(date=date, t=collection_type, icon=icon))
+            except (ValueError, KeyError):
+                # Skip invalid date entries but continue processing others
+                continue
+
+        return entries

package/python/vendor/waste_collection_schedule/source/fkf_bo_hu.py

@@ -0,0 +1,94 @@
+import datetime
+import json
+
+import requests
+from bs4 import BeautifulSoup
+from waste_collection_schedule import Collection, Icons
+from waste_collection_schedule.exceptions import SourceArgumentNotFound
+
+TITLE = "FKF Budaörs"
+DESCRIPTION = "Source script for www.fkf.hu"
+URL = "https://www.fkf.hu"
+COUNTRY = "hu"
+TEST_CASES = {
+    "Test_1": {"street": "Templom tér", "ssl_verify": False},
+    "Test_2": {"street": "Völgy utca", "ssl_verify": False},
+    "Test_3": {"street": "Zombori utca", "ssl_verify": False},
+}
+
+API_URL = "https://www.mohubudapest.hu/hulladeknaptar-budaors"
+ICON_MAP = {
+    "COMMUNAL": Icons.GENERAL_WASTE,
+    "SELECTIVE": Icons.RECYCLING,
+    "GREEN": Icons.ORGANIC,
+}
+
+
+class Source:
+    def __init__(self, street, ssl_verify: bool = True):
+        self._street = street
+        self._ssl_verify = ssl_verify
+
+    def fetch(self):
+        session = requests.Session()
+
+        r = session.post(
+            API_URL,
+            data={"publicPlace": self._street},
+            headers={
+                "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
+                "X-October-Request-Handler": "onSearch",
+                "X-October-Request-Partials": "ajax/budaorsResults",
+                "X-Requested-With": "XMLHttpRequest",
+            },
+            verify=self._ssl_verify,
+        )
+        r.raise_for_status()
+        soup = BeautifulSoup(
+            json.loads(r.text)["ajax/budaorsResults"], features="html.parser"
+        )
+
+        if soup.find("div", attrs={"class": "alert"}) is not None:
+            raise SourceArgumentNotFound("street", self._street)
+
+        entries = []
+        communal_divs = soup.find_all("div", attrs={"class": "communal"})
+
+        selective = soup.find_all("div", attrs={"class": "selective"})
+        communal = list(filter(lambda div: div.text == "Kommunális", communal_divs))
+        green = list(filter(lambda div: div.text == "Zöld", communal_divs))
+
+        for element in communal:
+            entries.append(
+                Collection(
+                    date=datetime.datetime.strptime(
+                        element.parent.parent.findAll("td")[1].text, "%Y.%m.%d"
+                    ).date(),
+                    t="Communal",
+                    icon=ICON_MAP.get("COMMUNAL"),
+                )
+            )
+
+        for element in selective:
+            entries.append(
+                Collection(
+                    date=datetime.datetime.strptime(
+                        element.parent.parent.findAll("td")[1].text, "%Y.%m.%d"
+                    ).date(),
+                    t="Selective",
+                    icon=ICON_MAP.get("SELECTIVE"),
+                )
+            )
+
+        for element in green:
+            entries.append(
+                Collection(
+                    date=datetime.datetime.strptime(
+                        element.parent.parent.findAll("td")[1].text, "%Y.%m.%d"
+                    ).date(),
+                    t="Green",
+                    icon=ICON_MAP.get("GREEN"),
+                )
+            )
+
+        return entries