@camstack/addon-waste-collections 0.0.0-stage → 0.1.1

package/python/vendor/waste_collection_schedule/source/neath_port_talbot_gov_uk.py

@@ -0,0 +1,236 @@
+from __future__ import annotations
+
+import datetime
+import re
+from dataclasses import dataclass
+from typing import ClassVar
+
+import requests
+from bs4 import BeautifulSoup
+from dateutil import parser
+from waste_collection_schedule import Collection, Icons
+
+TITLE = "Neath Port Talbot Council"
+DESCRIPTION = "Source for waste collection services for Neath Port Talbot Council"
+URL = "https://www.npt.gov.uk/"
+
+
+HOW_TO_GET_ARGUMENTS_DESCRIPTION = {
+    "en": "an easy way to discover your Unique Property Reference Number (UPRN) is by going to https://www.findmyaddress.co.uk/ and entering in your address details.",
+}
+
+
+PARAM_TRANSLATIONS = {
+    "en": {
+        "postcode": "Postcode",
+        "uprn": "Unique Property Reference Number (UPRN)",
+    }
+}
+
+
+PARAM_DESCRIPTIONS = {
+    "en": {
+        "postcode": "The postcode of the address to look up (e.g., SA11 1AB).",
+        "uprn": "Your Unique Street Reference Number (USRN) can be found by searching for your address at https://uprn.uk/ and viewing the _Data Associations_ section.",
+    }
+}
+
+
+TEST_CASES = {
+    "Test_001": {
+        "postcode": "SA11 3HW",
+        "uprn": 100100601042,
+    },
+    "Test_002": {
+        "postcode": "SA11 3HY",
+        "uprn": "100100599841",
+    },
+    "Test_003": {
+        "postcode": "SA11 3DY",
+        "uprn": "100100600279",
+    },
+}
+
+
+ICON_MAP = {
+    "Plastic / Tins / Cans": Icons.PLASTIC_PACKAGING,
+    "Cardboard, Cartons and Paper": Icons.PAPER,
+    "Glass": Icons.GLASS,
+    "Food Waste": Icons.BIO_KITCHEN,
+    "Batteries": Icons.BATTERY,
+    "General Household Rubbish": Icons.GENERAL_WASTE,
+    "Garden Waste": Icons.GARDEN,
+}
+
+
+class FailedToFindTokensError(Exception): ...
+
+
+class FailedToFindCollections(Exception): ...
+
+
+_DAY_MONTH_RE = re.compile(
+    r"(?i)^\s*(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)\w*,?\s*(\d{1,2})\s*([A-Za-z]+)\s*$"
+)
+
+
+@dataclass(frozen=True)
+class Tokens:
+    request_verification_token: str | None
+    ufprt: str | None
+
+
+class Source:
+    """
+    Neath Port Talbot Council waste collection source.
+
+    The Neath Port Talbot Council website is built in a way that makes it very difficult to scrape waste collection
+    data.
+    Collection dates are only given for the next two weeks from the current date.
+    Unfortunately, the user must submit their postcode even if they already know their UPRN.
+
+    This source implementation works by simulating the multi-step form submission process that a user would go
+    through on the website:
+        1. Load the initial page to obtain necessary tokens.
+        2. Submit the postcode to get the list of addresses.
+        3. Submit the UPRN to get the waste collection schedule.
+
+    Finally, the waste collection schedule is parsed from the resulting HTML page.
+    """
+
+    _BASE_URL = f"{URL}bins-and-recycling/equipment-and-collections/bin-day-finder/"
+    _HEADERS: ClassVar = {
+        "User-Agent": "Mozilla/5.0",
+        "Referer": _BASE_URL,
+        "Origin": URL,
+    }
+
+    def __init__(self, uprn: str | int, postcode: str) -> None:
+        self._uprn = str(uprn).zfill(12)
+        self._postcode = postcode
+
+    def fetch(self) -> list[Collection]:
+        session = requests.Session()
+        session.headers.update(self._HEADERS)
+
+        # First, make an initial GET request to obtain the necessary tokens.
+        initial_request = session.get(self._BASE_URL, timeout=20)
+        initial_request.raise_for_status()
+
+        # Step 1: Submit the postcode to get the address list / UPRNs.
+        # Note that even if we already know the UPRN, we still need to go through this step.
+        fetch_addresses_payload = {
+            **_base_post_payload(initial_request.text),
+            "PostCode": self._postcode,
+            "action": "Find address",
+        }
+        fetch_addresses_request = session.post(
+            self._BASE_URL, data=fetch_addresses_payload, timeout=20
+        )
+        fetch_addresses_request.raise_for_status()
+
+        # Step 2: Submit the UPRN to get the waste collection schedule.
+        fetch_collections_payload = {
+            **_base_post_payload(fetch_addresses_request.text),
+            "Address": self._uprn,
+            "action": "Show my bin days",
+        }
+        fetch_collections_request = session.post(
+            self._BASE_URL, data=fetch_collections_payload, timeout=20
+        )
+        fetch_collections_request.raise_for_status()
+
+        # Step 3: Parse the waste collection schedule from the response HTML.
+        return _parse_collections_from_page_source(fetch_collections_request.text)
+
+
+def _base_post_payload(raw_html: str) -> dict[str, str | None]:
+    tokens = _extract_tokens(raw_html)
+    return {
+        "__RequestVerificationToken": tokens.request_verification_token,
+        "ufprt": tokens.ufprt,
+    }
+
+
+def _extract_tokens(raw_html: str) -> Tokens:
+    soup = BeautifulSoup(raw_html, "html.parser")
+
+    try:
+        request_verification_token = soup.find(
+            "input", {"name": "__RequestVerificationToken"}
+        )[
+            "value"  # type: ignore[index]
+        ]
+    except TypeError as e:
+        raise FailedToFindTokensError(
+            "Failed to find __RequestVerificationToken in HTML."
+        ) from e
+
+    try:
+        ufprt = soup.find("input", {"name": "ufprt"})["value"]  # type: ignore[index]
+    except TypeError as e:
+        raise FailedToFindTokensError("Failed to find ufprt in HTML.") from e
+
+    return Tokens(
+        request_verification_token=(
+            str(request_verification_token) if request_verification_token else None
+        ),
+        ufprt=str(ufprt) if ufprt else None,
+    )
+
+
+def _parse_collections_from_page_source(raw_html: str) -> list[Collection]:
+    soup = BeautifulSoup(raw_html, "html.parser")
+    root = soup.find(id="contentInner")
+
+    # Find all date headers (they are <h2> containing e.g. "Thursday, 23 October").
+    headers = [
+        h
+        for h in root.find_all("h2")
+        if _DAY_MONTH_RE.match(_clean_text(h.get_text()))  # type: ignore[union-attr]
+    ]
+
+    collections: list = []
+    for h2 in headers:
+        collection_date = _collection_date_from_header(h2.get_text())
+
+        for sib in h2.next_siblings:
+            name = sib.name
+            if not name:
+                continue
+
+            if name == "h2":
+                # We've finished iterating this date's collections, so break to loop to the next date header.
+                break
+
+            cards = sib.find_all(class_="card") or []
+            for card in cards:
+                a = card.find("a")
+                if not a:
+                    continue
+
+                type_text = _clean_text(a.get_text())
+                if not type_text:
+                    continue
+
+                collections.append(
+                    Collection(
+                        date=collection_date,
+                        t=type_text,
+                        icon=ICON_MAP.get(type_text),
+                    )
+                )
+
+    if not collections:
+        raise FailedToFindCollections("No waste collection entries found on page.")
+
+    return collections
+
+
+def _collection_date_from_header(raw_html: str) -> datetime.date:
+    """Parse a datetime from a header like 'Thursday, 23 October'."""
+    return parser.parse(_clean_text(raw_html), dayfirst=True, fuzzy=True).date()
+
+
+def _clean_text(s: str) -> str:
+    return " ".join(s.replace("\xa0", " ").split())

package/python/vendor/waste_collection_schedule/source/nelincs_gov_uk.py

@@ -0,0 +1,61 @@
+import requests
+from bs4 import BeautifulSoup, Tag
+from dateutil import parser
+from waste_collection_schedule import Collection, Icons  # type: ignore[attr-defined]
+
+TITLE = "North East Lincolnshire Council"
+DESCRIPTION = "Source for North East Lincolnshire Council."
+URL = "https://www.nelincs.gov.uk/"
+TEST_CASES = {"11042949": {"uprn": 11042949}, "11043243": {"uprn": "11043243"}}
+
+
+HOW_TO_GET_ARGUMENTS_DESCRIPTION = {
+    "en": """Fill in your address details at [North East Lincolnshire Council's Find My Address](https://www.nelincs.gov.uk/find-my-address/), the Unique Property Reference Number (UPRN) will be shown in the URL field when you see your collection schedule. (e.g. `https://www.nelincs.gov.uk/?s=DN40+1JU&uprn=11043243` where `11043243` is the UPRN).
+
+Another easy way to discover your Unique Property Reference Number (UPRN) is by going to <https://www.findmyaddress.co.uk/> and entering in your address details.
+"""
+}
+
+
+ICON_MAP = {
+    "household waste": Icons.GENERAL_WASTE,
+    "garden": Icons.GARDEN,
+    "paper": Icons.PAPER,
+    "cans, plastic & glass": Icons.PLASTIC_PACKAGING,
+}
+
+
+API_URL = "https://www.nelincs.gov.uk/refuse-collection-schedule/"
+
+
+class Source:
+    def __init__(self, uprn: str | int):
+        self._uprn: str | int = uprn
+
+    def fetch(self) -> list[Collection]:
+        args = {"uprn": self._uprn}
+
+        r = requests.get(API_URL, params=args)
+        r.raise_for_status()
+
+        soup = BeautifulSoup(r.text, "html.parser")
+
+        heading_i = soup.select_one("i.fa-trash")
+        if not heading_i:
+            raise ValueError("No collection data found for the provided UPRN.")
+        collection_div = heading_i.find_parent("div")
+        if not isinstance(collection_div, Tag):
+            raise ValueError("No collection data found for the provided UPRN.")
+
+        entries = []
+        for heading, col_list in zip(
+            collection_div.select("div.h4"), collection_div.select("ul"), strict=False
+        ):
+            bin_type = heading.text.strip()
+            icon = ICON_MAP.get(bin_type.casefold())  # Collection icon
+            for li in col_list.select("li"):
+                date_str = li.text.strip()
+                date_ = parser.parse(date_str, dayfirst=True).date()
+                entries.append(Collection(date=date_, t=bin_type, icon=icon))
+
+        return entries

package/python/vendor/waste_collection_schedule/source/nemaffaldsservice_kk_dk.py

@@ -0,0 +1,147 @@
+import re
+from urllib.parse import parse_qs, urlparse
+
+import requests
+from waste_collection_schedule import Collection, Icons  # type: ignore[attr-defined]
+from waste_collection_schedule.exceptions import SourceArgumentNotFoundWithSuggestions
+from waste_collection_schedule.service.ICS import ICS
+
+TITLE = "Nem Affaldsservice (Københavns Kommune)"
+DESCRIPTION = "Source for Nem Affaldsservice, the waste collection schedule service of Københavns Kommune (City of Copenhagen), Denmark."
+URL = "https://nemaffaldsservice.kk.dk"
+COUNTRY = "dk"
+TEST_CASES = {
+    "Nørrebrogade 10": {"address": "Nørrebrogade 10"},
+    "Amagerbrogade 10": {"address": "Amagerbrogade 10"},
+    "Rådhuspladsen 1": {"address": "Rådhuspladsen 1"},
+}
+
+BASE_URL = "https://nemaffaldsservice.kk.dk"
+ADDRESS_LOOKUP_URL = f"{BASE_URL}/WasteHome/AddressByTerm/"
+CUSTOMER_LOOKUP_URL = f"{BASE_URL}/WasteHome/SearchCustomerRelation"
+CALENDAR_URL = f"{BASE_URL}/Calendar/GetICaldendar"
+
+HEADERS = {"User-Agent": "Mozilla/5.0"}
+
+TOKEN_RE = re.compile(
+    r'name="__RequestVerificationToken"\s+type="hidden"\s+value="([^"]+)"'
+)
+
+ICON_MAP = {
+    "Restaffald": Icons.GENERAL_WASTE,
+    "Madaffald": Icons.BIO_KITCHEN,
+    "Bioposer": Icons.BIO_KITCHEN,
+    "Papir": Icons.PAPER,
+    "Pap": Icons.PAPER,
+    "Glas": Icons.GLASS,
+    "Metal": Icons.METAL,
+    "Plast": Icons.PLASTIC_PACKAGING,
+    "Elektronik": Icons.ELECTRONICS,
+    "Farligt affald": Icons.HAZARDOUS,
+    "Tekstil": Icons.TEXTILE,
+    "Storskrald": Icons.BULKY,
+    "Haveaffald": Icons.GARDEN,
+}
+
+HOW_TO_GET_ARGUMENTS_DESCRIPTION = {
+    "en": (
+        "Enter your address exactly as it appears in Denmark, e.g. "
+        "'Nørrebrogade 10'. You can verify the spelling by typing your street "
+        "and house number into the search box on "
+        "https://nemaffaldsservice.kk.dk/ - if the site offers your address as "
+        "an autocomplete suggestion, that exact text is what should be used "
+        "here. If the address cannot be found, the resulting error message "
+        "will list similar addresses to help you find the correct spelling."
+    ),
+}
+
+PARAM_DESCRIPTIONS = {
+    "en": {
+        "address": (
+            "Street name and house number, e.g. 'Nørrebrogade 10'. Must match "
+            "an address served by Københavns Kommune / Nem Affaldsservice."
+        ),
+    },
+}
+
+PARAM_TRANSLATIONS = {
+    "en": {
+        "address": "Address",
+    },
+}
+
+
+class Source:
+    def __init__(self, address: str):
+        self._address: str = address.strip()
+
+    def fetch(self) -> list[Collection]:
+        session = requests.Session()
+        session.headers.update(HEADERS)
+
+        customer_id = self._get_customer_id(session, self._address)
+
+        r = session.get(CALENDAR_URL, params={"customerId": customer_id})
+        r.raise_for_status()
+
+        ics = ICS()
+        entries = []
+        for ev in ics.convert_events(r.text):
+            entries.append(
+                Collection(
+                    ev.date,
+                    ev.title,
+                    icon=ICON_MAP.get(ev.title),
+                )
+            )
+        return entries
+
+    def _get_customer_id(self, session: requests.Session, address: str) -> str:
+        # Resolve/validate the address against the site's own autocomplete
+        # endpoint first, so we can offer useful suggestions on a typo/mismatch
+        # instead of silently searching for a wrong address.
+        suggestions_r = session.get(ADDRESS_LOOKUP_URL, params={"term": address})
+        suggestions_r.raise_for_status()
+        suggestions = suggestions_r.json() or []
+
+        matched_address = None
+        labels = []
+        for suggestion in suggestions:
+            if not suggestion.get("fullAdress"):
+                continue
+            label = suggestion.get("label", "")
+            labels.append(label)
+            if label.lower() == address.lower():
+                matched_address = label
+                break
+
+        if matched_address is None:
+            raise SourceArgumentNotFoundWithSuggestions("address", address, labels)
+
+        home_r = session.get(BASE_URL)
+        home_r.raise_for_status()
+        token_match = TOKEN_RE.search(home_r.text)
+        if token_match is None:
+            raise SourceArgumentNotFoundWithSuggestions(
+                "address",
+                address,
+                [],
+            )
+        token = token_match.group(1)
+
+        search_r = session.post(
+            CUSTOMER_LOOKUP_URL,
+            data={
+                "SearchTerm": matched_address,
+                "__RequestVerificationToken": token,
+            },
+        )
+        search_r.raise_for_status()
+
+        customer_id = parse_qs(urlparse(search_r.url).query).get("customerId")
+        if not customer_id:
+            raise SourceArgumentNotFoundWithSuggestions(
+                "address", address, [matched_address]
+            )
+
+        return customer_id[0]