@camstack/addon-waste-collections 0.0.0-stage → 0.1.1

package/python/vendor/waste_collection_schedule/source/korneuburg_stadtservice_at.py

@@ -0,0 +1,199 @@
+import json
+from urllib.parse import urljoin
+
+import requests
+from bs4 import BeautifulSoup
+from waste_collection_schedule.exceptions import (
+    SourceArgumentNotFoundWithSuggestions,
+)
+from waste_collection_schedule.service.RiSKommunalAT import RiSKommunalSource
+
+TITLE = "Stadtservice Korneuburg"
+DESCRIPTION = "Source for Stadtservice Korneuburg"
+URL = "https://www.korneuburg.gv.at"
+COUNTRY = "at"
+TEST_CASES = {
+    "Rathaus": {"street_name": "Hauptplatz", "street_number": 39},  # Teilgebiet 4
+    "Rathaus using Teilgebiet": {
+        "street_name": "SomeStreet",
+        "street_number": "1A",
+        "teilgebiet": "4",
+    },  # Teilgebiet 4
+    "Werft": {"street_name": "Am Hafen", "street_number": 6},  # Teilgebiet 2
+}
+
+# Mapping of teilgebiete to calendar urls
+WASTE_TYPE_URLS = {
+    "1": ("Biomuell_3", "Restmuell_3", "Papier_2", "Gelber_Sack_4"),
+    "2": ("Biomuell_4", "Restmuell_2", "Papier_3", "Gelber_Sack_1"),
+    "3": ("Biomuell_1", "Restmuell_1", "Papier_1", "Gelber_Sack_2"),
+    "4": ("Biomuell_2", "Restmuell", "Papier", "Gelber_Sack_3"),
+}
+
+
+PARAM_TRANSLATIONS = {
+    "de": {
+        "street_name": "Straßenname",
+        "street_number": "Hausnummer",
+        "teilgebiet": "Teilgebiet",
+    },
+    "en": {
+        "street_name": "Street Name",
+        "street_number": "Street Number",
+        "teilgebiet": "Subarea",
+    },
+}
+
+
+class Source(RiSKommunalSource):
+    BASE_URL = "https://www.korneuburg.gv.at"
+
+    def __init__(self, street_name, street_number, teilgebiet=-1):
+        super().__init__()
+        self.street_name = street_name
+        self.street_number = street_number
+        self.teilgebiet = teilgebiet
+
+        self._region = None
+        self._street_name_id = -1
+        self._street_number_id = -1
+        self._headers = {"User-Agent": "Mozilla/5.0"}
+        self._cookies = {"ris_cookie_setting": "g7750"}  # Accept Cookie Consent
+
+    @staticmethod
+    def extract_street_numbers(soup):
+        scripts = soup.findAll("script", {"type": "text/javascript"})
+
+        street_number_idx = 0
+        for s in scripts:
+            if s.string and "var strassenArr" in s.string:
+                break
+            street_number_idx += 1
+
+        possible_numbers = json.loads(
+            scripts[street_number_idx]
+            .string[19:]
+            .replace("\r\n", "")
+            .replace(", ]", "]")
+            .replace("'", '"')
+        )
+
+        number_dict = {}
+
+        for idx, street_id in enumerate(possible_numbers):
+            number_dict[street_id[0]] = {
+                e[1]: (e[0], e[2]) for _idx, e in enumerate(possible_numbers[idx][1])
+            }
+
+        return number_dict
+
+    @staticmethod
+    def extract_street_names(soup):
+        street_selector = soup.find(
+            "select", {"id": "225991280_boxmuellkalenderstrassedd"}
+        ).findAll("option")
+        available_streets = {
+            street.string: int(street["value"])
+            for _idx, street in enumerate(street_selector)
+        }
+
+        return available_streets
+
+    @staticmethod
+    def extract_region(soup):
+        region = -1
+
+        for span in soup.findAll("span"):
+            if span.parent.name == "td" and "teilgebiet" in span.string.lower():
+                region = span.string.split(" ")[1]
+                break
+
+        return region
+
+    def determine_region(self):
+        """Find the target region for the street and street number."""
+        if 0 < int(self.teilgebiet) <= 4:
+            return str(self.teilgebiet)
+
+        # request address selection form
+        url = urljoin(URL, "Rathaus/Buergerservice/Muellabfuhr")
+        page = requests.get(url=url, headers=self._headers, cookies=self._cookies)
+        soup = BeautifulSoup(page.content, "html.parser")
+
+        # extract possible street and number combinations from html source
+        available_streets = self.extract_street_names(soup)
+        number_dict = self.extract_street_numbers(soup)
+
+        street_found = self.street_name in available_streets.keys()
+
+        if not street_found:
+            raise SourceArgumentNotFoundWithSuggestions(
+                "street_name", self.street_name, list(available_streets.keys())
+            )
+
+        self._street_name_id = available_streets.get(self.street_name)
+
+        self._street_number_id, street_number_link = number_dict.get(
+            available_streets.get(self.street_name)
+        ).get(str(self.street_number), (-1, "not found"))
+
+        if street_number_link == "not found":
+            raise SourceArgumentNotFoundWithSuggestions(
+                "street_number",
+                self.street_number,
+                list(number_dict.get(available_streets.get(self.street_name)).keys()),
+            )
+
+        # add selection cookie
+        self._cookies["riscms_muellkalender"] = str(
+            f"{self._street_name_id}_{self._street_number_id}"
+        )
+
+        # request overview with address selection to get the region
+        url = urljoin(URL, "system/web/kalender.aspx")
+        page = requests.get(
+            url=url,
+            headers=self._headers,
+            cookies=self._cookies,
+            params={
+                "sprache": "1",
+                "menuonr": "225991280",
+                "typids": street_number_link,
+            },
+        )
+        soup = BeautifulSoup(page.content, "html.parser")
+
+        region = self.extract_region(soup)
+
+        if region == -1:
+            raise Exception("Region could not be found")
+
+        return str(region)
+
+    def get_region_links(self):
+        """Traverse the waste-type pages and collect the iCal download links."""
+        if self._region is None:
+            self._region = self.determine_region()
+
+        # create waste type urls
+        ical_urls = []
+        urls = [urljoin(URL, u) for u in WASTE_TYPE_URLS.get(self._region)]
+
+        for u in urls:
+            r = requests.get(url=u, headers=self._headers, cookies=self._cookies)
+            soup = BeautifulSoup(r.content, "html.parser")
+            download_link = soup.findAll(
+                "a",
+                {
+                    "class": "piwik_download_tracker",
+                    "data-trackingtyp": "iCal/Kalender",
+                },
+            )
+            if len(download_link):
+                ical_urls.append(urljoin(URL, download_link[0].get("href")))
+
+        return ical_urls
+
+    def fetch(self):
+        ical_urls = self.get_region_links()
+        return self.parse_ics_urls(ical_urls, cookies=self._cookies)

package/python/vendor/waste_collection_schedule/source/kositeast_sk.py

@@ -0,0 +1,382 @@
+import io
+import logging
+import re
+from datetime import date
+from typing import ClassVar
+
+import requests
+from bs4 import BeautifulSoup
+from pdfminer.high_level import extract_pages
+from pdfminer.layout import LTPage, LTRect, LTTextContainer, LTTextLine
+from waste_collection_schedule import Collection, Icons
+from waste_collection_schedule.exceptions import SourceArgumentNotFound
+
+TITLE = "KOSIT EAST"
+DESCRIPTION = "Source for KOSIT EAST waste collection."
+URL = "https://kositeast.sk"
+COUNTRY = "sk"
+
+TEST_CASES = {
+    "Adidovce": {"town": "Adidovce"},
+    "Andrejová": {"town": "Andrejová"},
+}
+
+PARAM_TRANSLATIONS = {
+    "en": {
+        "town": "Town",
+    }
+}
+
+PARAM_DESCRIPTIONS = {
+    "en": {
+        "town": "Town name as displayed on the kositeast.sk website.",
+    }
+}
+
+HOW_TO_GET_ARGUMENTS_DESCRIPTION = {
+    "en": "Find your town on https://kositeast.sk/obyvatelia/harmonogram-zberu-odpadu-v-obciach/"
+    " and enter it exactly as it appears in the link.",
+}
+
+_LOGGER = logging.getLogger(__name__)
+
+ICON_MAP = {
+    "Komunálny odpad": Icons.GENERAL_WASTE,
+    "Plasty, VKM, Kovové obaly": Icons.RECYCLING,
+    "Sklo": Icons.GLASS,
+    "Papier": Icons.PAPER,
+    "Jedlé oleje a tuky": Icons.ORGANIC,
+    "Nebezpečný odpad": Icons.HAZARDOUS,
+}
+
+# RGB colour tuples as they appear in KOSIT EAST PDFs (rounded to 3 d.p.).
+# Verified across multiple municipalities (Adidovce, Andrejová, ...).
+COLOR_MAP = {
+    (1.0, 1.0, 0.0): "Plasty, VKM, Kovové obaly",
+    (0.451, 0.89, 0.008): "Sklo",
+    (0.584, 0.729, 1.0): "Papier",
+    (1.0, 0.702, 0.4): "Jedlé oleje a tuky",
+    (0.741, 0.494, 0.984): "Nebezpečný odpad",
+    (0.0, 0.0, 0.0): "Komunálny odpad",
+}
+
+# Maximum Euclidean RGB distance to accept as a colour match.
+# A small tolerance handles floating-point drift across PDF versions without
+# misidentifying genuinely different colours (nearest neighbours in COLOR_MAP
+# are all > 0.3 apart).
+_COLOR_TOLERANCE = 0.05
+
+# The schedule grid is always 6 month-columns per half-page:
+#   top half    -> months 1-6  (Jan-Jun)
+#   bottom half -> months 7-12 (Jul-Dec)
+_N_MONTH_COLS = 6
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _nearest_color(
+    color: tuple,
+    color_map: dict[tuple, str] = COLOR_MAP,
+    tolerance: float = _COLOR_TOLERANCE,
+) -> str | None:
+    """Return the nearest waste-type name for *color*, or None if too far away."""
+    best_name, best_dist = None, float("inf")
+    for known, name in color_map.items():
+        dist = sum((a - b) ** 2 for a, b in zip(color, known, strict=False)) ** 0.5
+        if dist < best_dist:
+            best_dist = dist
+            best_name = name
+    return best_name if best_dist <= tolerance else None
+
+
+def _cluster_values(values: list[float], n: int) -> list[float]:
+    """
+    Cluster *values* into at most *n* groups and return sorted centre positions.
+
+    Algorithm:
+      1. Round each value to 1 d.p. and deduplicate.
+      2. Greedily merge any two adjacent values that are < 5 pt apart.
+      3. While more than *n* groups remain, merge the closest adjacent pair.
+
+    This derives column boundaries entirely from the data — no hard-coded pixel
+    offsets or pitch values — so it adapts if the PDF is re-generated at a
+    different scale.
+    """
+    buckets = sorted({round(v, 1) for v in values})
+    merged: list[float] = []
+    for v in buckets:
+        if merged and v - merged[-1] < 5.0:
+            merged[-1] = (merged[-1] + v) / 2  # absorb into previous cluster
+        else:
+            merged.append(v)
+    while len(merged) > n:
+        gaps = [merged[i + 1] - merged[i] for i in range(len(merged) - 1)]
+        idx = gaps.index(min(gaps))
+        merged[idx] = (merged[idx] + merged[idx + 1]) / 2
+        del merged[idx + 1]
+    return merged
+
+
+def _nearest_cluster_idx(value: float, centres: list[float]) -> int:
+    """Return the 0-based index of the cluster centre closest to *value*."""
+    return min(range(len(centres)), key=lambda i: abs(centres[i] - value))
+
+
+def _extract_year(page: LTPage) -> int | None:
+    """Scan *page* for a 'ROK YYYY' header and return the year, or None."""
+    for element in page:
+        if isinstance(element, LTTextContainer):
+            for text_line in element:
+                if isinstance(text_line, LTTextLine):
+                    m = re.search(r"ROK\s+(\d{4})", text_line.get_text(), re.IGNORECASE)
+                    if m:
+                        return int(m.group(1))
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Source
+# ---------------------------------------------------------------------------
+
+
+class Source:
+    _MAX_RECT_WIDTH = 0.026
+    _N_MONTH_COLS = 6
+    _N_MONTH_ROWS = 2
+    _MATCH_HORIZONTAL_OVERLAP = False
+    _COLOR_MAP: ClassVar[dict[tuple, str]] = COLOR_MAP
+    _COLOR_TOLERANCE = _COLOR_TOLERANCE
+    _IGNORED_COLORS: ClassVar[set[tuple]] = set()
+
+    def __init__(self, town: str):
+        self._town = town
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def fetch(self) -> list[Collection]:
+        pdf_stream = self._download_pdf()
+        pages = list(extract_pages(pdf_stream))
+
+        if not pages:
+            return []
+
+        if len(pages) > 1:
+            _LOGGER.warning(
+                "PDF for '%s' has %d pages (only 1 expected). "
+                "Parsing all pages — please open an issue if results look wrong.",
+                self._town,
+                len(pages),
+            )
+
+        # Resolve year from PDF content — required to build valid dates.
+        # A hardcoded fallback would silently produce wrong dates when the PDF
+        # is published for a new year, so we raise instead.
+        year: int | None = None
+        for page in pages:
+            year = self._extract_year(page)
+            if year is not None:
+                break
+
+        if year is None:
+            raise ValueError(
+                f"Could not determine the schedule year from the PDF for town '{self._town}'. "
+                "The PDF layout may have changed — please open an issue and include the PDF URL."
+            )
+
+        # Parse collections from every page.
+        collections: list[Collection] = []
+        for page in pages:
+            collections.extend(self._parse_page(page, year))
+
+        return collections
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _extract_year(page: LTPage) -> int | None:
+        return _extract_year(page)
+
+    def _download_pdf(self) -> io.BytesIO:
+        schedule_url = (
+            "https://kositeast.sk/obyvatelia/harmonogram-zberu-odpadu-v-obciach/"
+        )
+        r = requests.get(schedule_url, timeout=30)
+        r.raise_for_status()
+        soup = BeautifulSoup(r.text, "html.parser")
+
+        pdf_link: str | None = None
+        for a in soup.find_all("a", href=True):
+            if a.get_text(strip=True).lower() == self._town.strip().lower() and a[
+                "href"
+            ].endswith(".pdf"):
+                pdf_link = a["href"]
+                break
+
+        if pdf_link is None:
+            raise SourceArgumentNotFound("town", self._town)
+
+        r_pdf = requests.get(pdf_link, timeout=30)
+        r_pdf.raise_for_status()
+        return io.BytesIO(r_pdf.content)
+
+    def _parse_page(self, page: LTPage, year: int) -> list[Collection]:
+        """
+        Extract Collection objects from one PDF page.
+
+        The grid layout (column positions and the top/bottom half split) is
+        inferred entirely from the positions of the coloured rectangles — no
+        hard-coded pixel offsets or pitch values are used.  Rect-size thresholds
+        are expressed as fractions of the page dimensions so they remain valid
+        if the PDF is regenerated at a different scale.
+        """
+        page_h: float = page.bbox[3]
+        page_w: float = page.bbox[2]
+
+        # Rect-size thresholds as fractions of page dimensions.
+        # Equivalent to roughly 8-19 pt wide and 4-13 pt tall on a standard
+        # A4 PDF, but will scale correctly with any page size.
+        min_w = page_w * 0.011
+        max_w = page_w * self._MAX_RECT_WIDTH
+        min_h = page_h * 0.005
+        max_h = page_h * 0.016
+
+        lines: list[dict] = []
+        rects: list[dict] = []
+
+        for element in page:
+            if isinstance(element, LTTextContainer):
+                for text_line in element:
+                    if isinstance(text_line, LTTextLine):
+                        text = text_line.get_text().strip()
+                        if text:
+                            lines.append({"bbox": text_line.bbox, "text": text})
+
+            elif isinstance(element, LTRect):
+                if min_w < element.width < max_w and min_h < element.height < max_h:
+                    color = element.non_stroking_color
+                    if isinstance(color, (int, float)):
+                        color = (round(color, 3),) * 3
+                    elif color is not None:
+                        color = tuple(round(c, 3) for c in color)
+                    else:
+                        color = (0.0, 0.0, 0.0)
+                    if color not in self._IGNORED_COLORS:
+                        rects.append({"bbox": element.bbox, "color": color})
+
+        if not rects:
+            _LOGGER.warning(
+                "No schedule rectangles found on page for '%s'. "
+                "The PDF layout may have changed.",
+                self._town,
+            )
+            return []
+
+        # --- Infer grid layout from rect positions ---
+
+        x_ctrs = [(r["bbox"][0] + r["bbox"][2]) / 2 for r in rects]
+        y_ctrs = [(r["bbox"][1] + r["bbox"][3]) / 2 for r in rects]
+
+        x_clusters = _cluster_values(x_ctrs, n=self._N_MONTH_COLS)
+        y_clusters = _cluster_values(y_ctrs, n=self._N_MONTH_ROWS)
+
+        if len(x_clusters) != self._N_MONTH_COLS:
+            _LOGGER.warning(
+                "Expected %d x-clusters (month columns) for '%s', found %d. "
+                "Month assignments may be incorrect.",
+                self._N_MONTH_COLS,
+                self._town,
+                len(x_clusters),
+            )
+
+        if len(y_clusters) != self._N_MONTH_ROWS:
+            _LOGGER.warning(
+                "Expected %d y-clusters (month rows) for '%s', found %d. "
+                "Month assignments may be incorrect.",
+                self._N_MONTH_ROWS,
+                self._town,
+                len(y_clusters),
+            )
+
+        # --- Map each rect to a date + waste type ---
+        # y_clusters is sorted ascending (pdfminer: y=0 at bottom of page), so
+        # the last cluster is the topmost row of months and the first cluster
+        # the bottom row. For KOSIT EAST that means:
+        #   y_clusters[1] -> months 1-6  (Jan-Jun)
+        #   y_clusters[0] -> months 7-12 (Jul-Dec)
+
+        collections: list[Collection] = []
+
+        for r_item in rects:
+            rx0, ry0, rx1, ry1 = r_item["bbox"]
+            rcx = (rx0 + rx1) / 2
+            rcy = (ry0 + ry1) / 2
+
+            col = _nearest_cluster_idx(rcx, x_clusters)
+            row = _nearest_cluster_idx(rcy, y_clusters)
+            month = (self._N_MONTH_ROWS - 1 - row) * self._N_MONTH_COLS + col + 1
+
+            if not 1 <= month <= 12:
+                continue
+
+            # Find the day number from a text line that overlaps this rect
+            # vertically and belongs to the same column.
+            day: int | None = None
+            for line in lines:
+                lx0, ly0, lx1, ly1 = line["bbox"]
+                if max(ry0, ly0) >= min(ry1, ly1):  # no vertical overlap
+                    continue
+                if self._MATCH_HORIZONTAL_OVERLAP and max(rx0, lx0) >= min(rx1, lx1):
+                    continue
+                if not self._MATCH_HORIZONTAL_OVERLAP:
+                    lcx = (lx0 + lx1) / 2
+                    if _nearest_cluster_idx(lcx, x_clusters) != col:
+                        continue
+
+                # UPDATED REGEX: Added uppercase slovak characters and general uppercase support
+                m = re.search(
+                    r"(?:^|[A-ZŽŠČŤĎŇĽĹÁÉÍÓÚÄÔa-zžščťďňľĺáéíóúäô]+\s+)(\d+)\b",
+                    line["text"],
+                )
+                if m:
+                    day = int(m.group(1))
+                    break
+
+            if day is None:
+                continue
+
+            waste_type = _nearest_color(
+                r_item["color"], self._COLOR_MAP, self._COLOR_TOLERANCE
+            )
+            if waste_type is None:
+                _LOGGER.warning(
+                    "Unrecognised colour %s for '%s'; skipping entry.",
+                    r_item["color"],
+                    self._town,
+                )
+                continue
+
+            try:
+                collections.append(
+                    Collection(
+                        date=date(year, month, day),
+                        t=waste_type,
+                        icon=ICON_MAP.get(waste_type),
+                    )
+                )
+            except ValueError:
+                _LOGGER.warning(
+                    "Invalid date %04d-%02d-%02d for '%s'; skipping.",
+                    year,
+                    month,
+                    day,
+                    self._town,
+                )
+
+        return collections

package/python/vendor/waste_collection_schedule/source/kositwest_sk.py

@@ -0,0 +1,90 @@
+import io
+import re
+from typing import ClassVar
+from urllib.parse import urljoin
+
+import requests
+from bs4 import BeautifulSoup
+from pdfminer.layout import LTPage, LTTextContainer, LTTextLine
+from waste_collection_schedule.exceptions import SourceArgumentNotFound
+from waste_collection_schedule.source.kositeast_sk import COLOR_MAP as EAST_COLOR_MAP
+from waste_collection_schedule.source.kositeast_sk import Source as KositEastSource
+
+TITLE = "KOSIT WEST"
+DESCRIPTION = "Source for KOSIT WEST waste collection."
+URL = (
+    "https://kositwest.sk/sluzby/zber-komunalneho-odpadu-a-triedenych-zloziek-z-obci-"
+    "a-samosprav/harmonogramy-zberu-odpadu-v-obciach/"
+)
+COUNTRY = "sk"
+
+SOURCE_CODEOWNERS = ["@mynameisdominik"]
+
+TEST_CASES = {
+    "Vieska": {"town": "Vieska"},
+    "Sap": {"town": "Sap"},
+    "Michal na Ostrove": {"town": "Michal na Ostrove"},
+}
+
+PARAM_TRANSLATIONS = {
+    "en": {
+        "town": "Town",
+    }
+}
+
+PARAM_DESCRIPTIONS = {
+    "en": {
+        "town": "Town name as displayed on the kositwest.sk website.",
+    }
+}
+
+HOW_TO_GET_ARGUMENTS_DESCRIPTION = {
+    "en": "Find your town on the KOSIT WEST website and enter it exactly as it appears in the link.",
+}
+
+
+class Source(KositEastSource):
+    _MAX_RECT_WIDTH = 0.04
+    _N_MONTH_COLS = 3
+    _N_MONTH_ROWS = 4
+    _MATCH_HORIZONTAL_OVERLAP = True
+    _COLOR_MAP: ClassVar[dict[tuple, str]] = {
+        **EAST_COLOR_MAP,
+        (0.573, 0.816, 0.314): "Sklo",
+        (0.557, 0.663, 0.859): "Papier",
+    }
+    _IGNORED_COLORS: ClassVar[set[tuple]] = {(1.0, 1.0, 1.0)}
+
+    @staticmethod
+    def _extract_year(page: LTPage) -> int | None:
+        for element in page:
+            if isinstance(element, LTTextContainer):
+                for text_line in element:
+                    if isinstance(text_line, LTTextLine):
+                        match = re.search(r"\b(20\d{2})\b", text_line.get_text())
+                        if match:
+                            return int(match.group(1))
+        return None
+
+    def _download_pdf(self) -> io.BytesIO:
+        response = requests.get(URL, timeout=30)
+        response.raise_for_status()
+        soup = BeautifulSoup(response.text, "html.parser")
+
+        pdf_link: str | None = None
+        for anchor in soup.find_all("a", href=True):
+            href = anchor["href"]
+            if not isinstance(href, str):
+                continue
+            if anchor.get_text(
+                strip=True
+            ).lower() == self._town.strip().lower() and href.lower().endswith(".pdf"):
+                pdf_link = href
+                break
+
+        if pdf_link is None:
+            raise SourceArgumentNotFound("town", self._town)
+
+        pdf_response = requests.get(urljoin(URL, pdf_link), timeout=30)
+        pdf_response.raise_for_status()
+        return io.BytesIO(pdf_response.content)