@camstack/addon-waste-collections 0.0.0-stage → 0.1.2

package/python/vendor/waste_collection_schedule/source/newforest_gov_uk.py

@@ -0,0 +1,294 @@
+import http.client
+from urllib.parse import urlencode
+
+from bs4 import BeautifulSoup
+from dateutil import parser as dateutil_parser
+from waste_collection_schedule import Collection, Icons  # type: ignore[attr-defined]
+from waste_collection_schedule.exceptions import (  # type: ignore[attr-defined]
+    SourceArgumentNotFoundWithSuggestions,
+)
+
+TITLE = "New Forest District Council"
+DESCRIPTION = "Source for newforest.gov.uk services for New Forest District Council."
+URL = "https://www.newforest.gov.uk"
+COUNTRY = "uk"
+FORM_HOST = "forms.newforest.gov.uk"
+FORM_PATH = "/ufs/FIND_MY_BIN_BAR.eb"
+FORM_BASE_URL = f"https://{FORM_HOST}/ufs"
+
+TEST_CASES = {
+    "1 Oleander Drive, Totton": {"postcode": "SO40 8XX", "uprn": 100060514912},
+    "49 Salisbury Road, Totton": {"postcode": "SO40 3HX", "uprn": "100060518013"},
+    "12 Hawkins Close, Ringwood": {"postcode": "BH24 1UQ", "uprn": "100060495458"},
+}
+
+ICON_MAP = {
+    "Food": Icons.BIO_KITCHEN,
+    "Garden": Icons.GARDEN,
+    "General": Icons.GENERAL_WASTE,
+    "Glass": Icons.GLASS,
+    "Recycle": Icons.RECYCLING,
+}
+
+PARAM_TRANSLATIONS = {
+    "en": {
+        "postcode": "Postcode",
+        "uprn": "Unique Property Reference Number (UPRN)",
+    }
+}
+
+PARAM_DESCRIPTIONS = {
+    "en": {
+        "postcode": "Your property's postcode, e.g. SO40 8XX",
+        "uprn": "Your Unique Property Reference Number. Find it at https://www.findmyaddress.co.uk/",
+    }
+}
+
+
+class Source:
+    def __init__(self, postcode: str, uprn: str | int):
+        self._postcode = str(postcode).strip()
+        self._uprn = str(uprn).strip()
+
+    def fetch(self) -> list[Collection]:
+        conn = http.client.HTTPSConnection(FORM_HOST, timeout=30)
+        try:
+            jsessionid, ebs, formstack = self._init_session(conn)
+            ebs2 = self._submit_postcode(conn, jsessionid, ebs, formstack)
+            html = self._submit_uprn(conn, jsessionid, ebs2, formstack)
+        finally:
+            conn.close()
+
+        return self._parse_collections(html)
+
+    def _get(
+        self,
+        conn: http.client.HTTPSConnection,
+        path: str,
+        jsessionid: str | None = None,
+    ) -> tuple[int, list[tuple[str, str]], str]:
+        headers: dict[str, str] = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        }
+        if jsessionid:
+            headers["Cookie"] = f"JSESSIONID={jsessionid}"
+        conn.request("GET", path, headers=headers)
+        resp = conn.getresponse()
+        body = resp.read().decode("utf-8")
+        # Return raw header list to preserve duplicates (server sends two Set-Cookie headers)
+        return resp.status, resp.getheaders(), body
+
+    def _post(
+        self, conn: http.client.HTTPSConnection, path: str, data: dict, jsessionid: str
+    ) -> tuple[int, list[tuple[str, str]]]:
+        encoded = urlencode(data)
+        headers = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Content-Type": "application/x-www-form-urlencoded",
+            "Content-Length": str(len(encoded.encode("utf-8"))),
+            "Origin": FORM_BASE_URL,
+            "Cookie": f"JSESSIONID={jsessionid}",
+        }
+        conn.request("POST", path, body=encoded, headers=headers)
+        resp = conn.getresponse()
+        resp.read()
+        return resp.status, resp.getheaders()
+
+    @staticmethod
+    def _header(raw_headers: list[tuple[str, str]], name: str) -> str | None:
+        """Return the first value for a header name (case-insensitive)."""
+        name_lower = name.lower()
+        for k, v in raw_headers:
+            if k.lower() == name_lower:
+                return v
+        return None
+
+    def _init_session(self, conn: http.client.HTTPSConnection) -> tuple[str, str, str]:
+        """Perform the initial GET (following two redirects) to establish a session."""
+        # First GET: server returns 302 with JSESSIONID in the first Set-Cookie header.
+        # The server sends two Set-Cookie headers; the second is a malformed "Secure; HttpOnly"
+        # that would overwrite JSESSIONID in a dict.  We use raw header iteration instead.
+        _status, raw_headers, _ = self._get(conn, FORM_PATH)
+        jsessionid = None
+        for k, v in raw_headers:
+            if k.lower() == "set-cookie" and "JSESSIONID=" in v:
+                jsessionid = v.split("JSESSIONID=")[1].split(";")[0]
+                break
+        if not jsessionid:
+            raise Exception(
+                "Could not obtain session cookie from New Forest form server."
+            )
+
+        # Second GET: follows the location from the 302
+        loc1 = self._header(raw_headers, "location") or FORM_PATH
+        _status, raw_headers, _ = self._get(conn, f"/ufs/{loc1}", jsessionid)
+
+        # Third GET: follows to the parameterised form URL (ebd=0&ebp=10&ebz=...)
+        loc2 = self._header(raw_headers, "location") or loc1
+        _status, raw_headers, body = self._get(conn, f"/ufs/{loc2}", jsessionid)
+
+        soup = BeautifulSoup(body, "html.parser")
+        form = soup.find("form")
+        if not form:
+            raise Exception("Could not find the collection schedule form.")
+        ebs = form.find("input", {"name": "ebs"})["value"]
+        formstack = form.find("input", {"name": "formstack"})["value"]
+        return jsessionid, ebs, formstack
+
+    def _submit_postcode(
+        self,
+        conn: http.client.HTTPSConnection,
+        jsessionid: str,
+        ebs: str,
+        formstack: str,
+    ) -> str:
+        """Submit the postcode and return the updated ebs for page 2."""
+        status, raw_headers = self._post(
+            conn,
+            f"{FORM_PATH}?ebz={ebs}",
+            {
+                "formid": "/Forms/FIND_MY_BIN_BAR",
+                "ebs": ebs,
+                "origrequrl": f"{FORM_BASE_URL}/FIND_MY_BIN_BAR.eb",
+                "formstack": formstack,
+                "PAGE:E.h": "",
+                "PAGE:B.h": "",
+                "PAGE:N.h": "",
+                "PAGE:S.h": "",
+                "PAGE:R.h": "",
+                "PAGE:D": "",
+                "PAGE:H": "",
+                "PAGE:X": "0",
+                "PAGE:Y": "0",
+                "PAGE:F": "CTID-EBTnjgwK-_",
+                "ufsEndUser*": "1",
+                "pageSeq": "1",
+                "pageId": "Page_1",
+                "formStateId": "1",
+                "$USERVAR1": "",
+                "$USERVAR2": "",
+                "$USERVAR3": "",
+                "CTRL:JmLqCKl2:_:A": self._postcode,
+                "CTRL:JmLqCKl2:_:B.h": "",
+                "HID:inputs": "ICTRL:JmLqCKl2:_:A,ACTRL:JmLqCKl2:_:B.h,ACTRL:EBTnjgwK:_,APAGE:E.h,APAGE:B.h,APAGE:N.h,APAGE:S.h,APAGE:R.h",
+                "CTRL:EBTnjgwK:_": "Submit",
+            },
+            jsessionid,
+        )
+        loc = self._header(raw_headers, "location")
+        if not loc:
+            raise Exception(
+                f"Postcode lookup did not redirect (status {status}). Check the postcode is valid."
+            )
+
+        # Follow redirect to page 2 (address selection)
+        _, _, body = self._get(conn, f"/ufs/{loc}", jsessionid)
+        soup = BeautifulSoup(body, "html.parser")
+
+        # Validate that our UPRN is in the address list
+        options = soup.find_all("option")
+        available_uprns = [opt.get("value") for opt in options if opt.get("value")]
+        if not available_uprns:
+            raise Exception(f"No addresses found for postcode '{self._postcode}'.")
+        if self._uprn not in available_uprns:
+            raise SourceArgumentNotFoundWithSuggestions(
+                "uprn",
+                self._uprn,
+                available_uprns,
+            )
+
+        form = soup.find("form")
+        return form.find("input", {"name": "ebs"})["value"]
+
+    def _submit_uprn(
+        self,
+        conn: http.client.HTTPSConnection,
+        jsessionid: str,
+        ebs: str,
+        formstack: str,
+    ) -> str:
+        """Submit the UPRN and return the results page HTML."""
+        status, raw_headers = self._post(
+            conn,
+            f"{FORM_PATH}?ebz={ebs}",
+            {
+                "formid": "/Forms/FIND_MY_BIN_BAR",
+                "ebs": ebs,
+                "origrequrl": f"{FORM_BASE_URL}/FIND_MY_BIN_BAR.eb",
+                "formstack": formstack,
+                "PAGE:E.h": "",
+                "PAGE:B.h": "",
+                "PAGE:N.h": "",
+                "PAGE:S.h": "",
+                "PAGE:R.h": "",
+                "PAGE:D": "",
+                "PAGE:H": "",
+                "PAGE:X": "0",
+                "PAGE:Y": "0",
+                "PAGE:F": "",
+                "ufsEndUser*": "1",
+                "pageSeq": "2",
+                "pageId": "Page_1",
+                "formStateId": "1",
+                "$USERVAR1": "",
+                "$USERVAR2": "",
+                "$USERVAR3": "",
+                "CTRL:KOeKcmrC:_:A": self._uprn,
+                "CTRL:KOeKcmrC:_:B.h": "",
+                "HID:inputs": "ICTRL:KOeKcmrC:_:A,ACTRL:KOeKcmrC:_:B.h,ACTRL:QxB4NyYs:_,ACTRL:Ggx8Z7ze:_,APAGE:E.h,APAGE:B.h,APAGE:N.h,APAGE:S.h,APAGE:R.h",
+                "CTRL:QxB4NyYs:_": "Submit",
+            },
+            jsessionid,
+        )
+        loc = self._header(raw_headers, "location")
+        if not loc:
+            raise Exception(f"UPRN submission did not redirect (status {status}).")
+
+        _, _, body = self._get(conn, f"/ufs/{loc}", jsessionid)
+        return body
+
+    def _parse_collections(self, html: str) -> list[Collection]:
+        soup = BeautifulSoup(html, "html.parser")
+
+        # Find the "Future collections" table — it has a class containing "tableContent"
+        # and exactly two column headers: "Collection date" and "Container type".
+        collection_table = None
+        for table in soup.find_all("table", class_=True):
+            classes = " ".join(table.get("class", []))
+            if "tableContent" not in classes:
+                continue
+            thead = table.find("thead")
+            if not thead:
+                continue
+            th_tags = thead.find_all("th")
+            header_texts = [th.get_text(strip=True) for th in th_tags]
+            if header_texts == ["Collection date", "Container type"]:
+                collection_table = table
+                break
+
+        if not collection_table:
+            raise Exception(
+                "Could not find the collection schedule table in the response."
+            )
+
+        entries = []
+        for row in collection_table.find_all("tr")[1:]:  # skip header row
+            cells = row.find_all("td")
+            if len(cells) < 2:
+                continue
+            date_str = cells[0].get_text(strip=True)
+            waste_type = cells[1].get_text(strip=True)
+            if not date_str or not waste_type:
+                continue
+            try:
+                date = dateutil_parser.parse(date_str).date()
+            except Exception:
+                continue
+
+            # Map icon by the first word of the waste type (e.g. "Food", "General", etc.)
+            icon = ICON_MAP.get(waste_type.split()[0] if waste_type else "")
+            entries.append(Collection(date=date, t=waste_type, icon=icon))
+
+        return entries

package/python/vendor/waste_collection_schedule/source/newham_gov_uk.py

@@ -0,0 +1,102 @@
+import re
+from datetime import datetime
+
+import requests
+import urllib3
+from bs4 import BeautifulSoup
+from waste_collection_schedule import Collection, Icons  # type: ignore[attr-defined]
+from waste_collection_schedule.exceptions import SourceArgumentNotFound
+
+TITLE = "London Borough of Newham"
+DESCRIPTION = "Source for newham.gov.uk services for London Borough of Newham, UK."
+URL = "https://www.newham.gov.uk"
+TEST_CASES = {
+    "Test_001": {"property": "000046029461"},
+    "Test_002": {"property": "000046250697"},
+    "Test_003": {"property": 46012509},
+}
+
+ICON_MAP = {
+    "DOMESTIC": Icons.GENERAL_WASTE,
+    "RECYCLING": Icons.RECYCLING,
+    "FOOD WASTE": Icons.BIO_KITCHEN,
+}
+
+API_URL = "https://bincollection.newham.gov.uk/Details/Index/{property}"
+
+# "Next<nbsp>Tuesday<nbsp>01/09/2026" - the day name is optional, the date is
+# always dd/mm/yyyy. Allow at most one word between "Next" and the date, and
+# never "Previous", so that a card with an empty "Next" cannot match the
+# "Previous" date that follows it.
+NEXT_DATE = re.compile(r"Next\s*(?:(?!Previous)[A-Za-z]+\s*)?(\d{2}/\d{2}/\d{4})")
+
+# The server presents only its leaf certificate, so the chain cannot be verified.
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+
+class Source:
+    def __init__(self, property):
+        self._property = str(property).zfill(12)
+
+    def fetch(self):
+        r = requests.get(
+            API_URL.format(property=self._property), verify=False, timeout=30
+        )
+        r.raise_for_status()
+
+        soup = BeautifulSoup(r.text, features="html.parser")
+
+        # An unknown (but well formed) property ID still returns HTTP 200 with a
+        # fully rendered page; only the address card is left empty. Without this
+        # check every collection card is skipped and fetch() returns [] silently.
+        # Key the lookup on the header text, not on card order: every collection
+        # card carries the "card" class too, so a positional lookup would read a
+        # collection card (and pass) if the page were ever reordered.
+        address = None
+        for card in soup.find_all("div", class_="card"):
+            header = card.find("div", {"class": "card-header"})
+            if header is not None and "address" in header.get_text(strip=True).lower():
+                address = card.find("p", {"class": "card-text"})
+                break
+        if address is None or not address.get_text(strip=True):
+            raise SourceArgumentNotFound(
+                "property",
+                self._property,
+                "this property ID does not exist on bincollection.newham.gov.uk.",
+            )
+
+        entries = []
+
+        # Each collection type is its own card; the type is in bold in the card
+        # header ("Your <b>Domestic</b> Collection Day"). Cards without a bold
+        # header (e.g. green and garden waste) carry no schedule.
+        for card in soup.find_all("div", class_="card"):
+            header = card.find("div", {"class": "card-header"})
+            if header is None:
+                continue
+            bin_type_element = header.find("b")
+            if bin_type_element is None:
+                continue
+            bin_type = bin_type_element.get_text(strip=True)
+            # ICON_MAP doubles as the allowlist of collection types: a type
+            # Newham adds later is skipped until it is given an icon here.
+            if bin_type.upper() not in ICON_MAP:
+                continue
+
+            body = card.find("p", {"class": "card-text"})
+            if body is None:
+                continue
+            match = NEXT_DATE.search(body.get_text())
+            if match is None:
+                # The card is rendered even when no collection is scheduled.
+                continue
+
+            entries.append(
+                Collection(
+                    date=datetime.strptime(match.group(1), "%d/%m/%Y").date(),
+                    t=bin_type,
+                    icon=ICON_MAP.get(bin_type.upper()),
+                )
+            )
+
+        return entries

package/python/vendor/waste_collection_schedule/source/nillumbik_vic_gov_au.py

@@ -0,0 +1,32 @@
+from waste_collection_schedule import Collection, Icons  # type: ignore[attr-defined]
+from waste_collection_schedule.service.OpenCities import (
+    OpenCitiesClient,
+    OpenCitiesConfig,
+)
+
+TITLE = "Nillumbik Shire Council"
+DESCRIPTION = "Source for Nillumbik Shire Council rubbish collection."
+URL = "https://www.nillumbik.vic.gov.au"
+TEST_CASES = {"Test": {"street_address": "11 Sunnyside Crescent, WATTLE GLEN, 3096"}}
+
+ICON_MAP = {
+    "Food and Green Waste": Icons.BIO_KITCHEN,
+    "Hard Waste": Icons.BULKY,
+    "Recycling": Icons.RECYCLING,
+}
+
+_CONFIG = OpenCitiesConfig(
+    domain="https://www.nillumbik.vic.gov.au",
+    argument_name="street_address",
+    warm_up_url="https://www.nillumbik.vic.gov.au/Residents/Waste-and-recycling/Bin-collection/Check-my-bin-day",
+    icon_keywords=ICON_MAP,
+)
+
+
+class Source:
+    def __init__(self, street_address: str):
+        self._street_address = street_address
+        self._client = OpenCitiesClient(_CONFIG)
+
+    def fetch(self) -> list[Collection]:
+        return self._client.fetch(address=self._street_address)

package/python/vendor/waste_collection_schedule/source/north_ayrshire_gov_uk.py

@@ -0,0 +1,67 @@
+import requests
+from dateutil import parser
+from waste_collection_schedule import Collection, Icons  # type: ignore[attr-defined]
+
+TITLE = "North Ayrshire Council"
+DESCRIPTION = "Source for north-ayrshire.gov.uk services for North Ayrshire"
+URL = "https://www.north-ayrshire.gov.uk/"
+API_URL = "https://www.maps.north-ayrshire.gov.uk/arcgis/rest/services/AGOL/YourLocationLive/MapServer/8/query?f=json&outFields=*&returnDistinctValues=true&returnGeometry=false&spatialRel=esriSpatialRelIntersects&where=UPRN%20%3D%20%27{0}%27"
+
+TEST_CASES = {
+    "Test_001": {"uprn": "126043248"},
+    "Test_002": {"uprn": 126021147},
+    "Test_003": {"uprn": 126091148},
+    "Test_004": {"uprn": "126000270"},
+}
+
+ICON_MAP = {
+    "Grey": Icons.GENERAL_WASTE,
+    "Brown": Icons.ORGANIC,
+    "Purple": Icons.GLASS,
+    "Blue": Icons.RECYCLING,
+}
+
+BIN_TEXTS = [
+    "BLUE_DATE_TEXT",
+    "GREY_DATE_TEXT",
+    "PURPLE_DATE_TEXT",
+    "BROWN_DATE_TEXT",
+]
+
+
+class Source:
+    def __init__(self, uprn):
+        self._uprn = str(uprn)
+
+    def fetch(self):
+        return self.__get_bin_collection_info_json(self._uprn)
+
+    def __get_bin_collection_info_json(self, uprn):
+        r = requests.get(API_URL.format(uprn))
+        bin_json = r.json()["features"]
+        bin_list = []
+        for item in BIN_TEXTS:
+            if item in bin_json[0]["attributes"]:
+                colour = item.split("_")[0].capitalize()
+                try:
+                    bin_list.append(
+                        [
+                            colour,
+                            "/".join(
+                                reversed(bin_json[0]["attributes"][item].split("/"))
+                            ),
+                        ]
+                    )
+                except AttributeError:  # catches error when no date is present
+                    pass
+
+        entries = []
+        for bins in bin_list:
+            entries.append(
+                Collection(
+                    date=parser.parse(bins[1]).date(),
+                    t=bins[0],
+                    icon=ICON_MAP.get(bins[0]),
+                )
+            )
+        return entries

package/python/vendor/waste_collection_schedule/source/north_kesteven_org_uk.py

@@ -0,0 +1,66 @@
+from datetime import datetime
+
+from bs4 import BeautifulSoup
+from curl_cffi import requests
+from waste_collection_schedule import Collection, Icons  # type: ignore[attr-defined]
+
+TITLE = "North Kesteven District Council"
+DESCRIPTION = (
+    "Source for n-kesteven.org.uk services for North Kesteven District Council, UK."
+)
+URL = "https://n-kesteven.org.uk"
+TEST_CASES = {
+    "Test_001": {"uprn": "100030860713"},
+    "Test_002": {"uprn": "10006514327"},
+    "Test_003": {"uprn": "100030857039"},
+    "Test_004": {"uprn": 100030864449},
+    "Test_005": {"uprn": 10006507163},
+}
+ICON_MAP = {
+    "BLACK": Icons.GENERAL_WASTE,
+    "GREEN": Icons.RECYCLING,
+    "PURPLE": Icons.NEWSPAPER,
+    "BROWN": Icons.ORGANIC,
+    "ORANGE": Icons.BIO_KITCHEN,
+}
+
+
+def _icon_for(bin_type: str):
+    # The council has changed the wording of bin type names over time
+    # (e.g. "Purple (Paper/Card)" -> "Purple Lidded", "Orange Lidded Caddy"),
+    # so match on the leading colour keyword rather than the full string.
+    for keyword, icon in ICON_MAP.items():
+        if keyword in bin_type:
+            return icon
+    return None
+
+
+class Source:
+    def __init__(self, uprn):
+        self._uprn = str(uprn)
+
+    def fetch(self):
+        # North Kesteven District Council intermittently enables Cloudflare's
+        # bot-check on this endpoint, which blocks plain "requests" sessions.
+        # Impersonating a real browser via curl_cffi avoids that in most cases.
+        s = requests.Session(impersonate="chrome")
+        r = s.get(f"https://www.n-kesteven.org.uk/bins/display?uprn={self._uprn}")
+        r.raise_for_status()
+        soup = BeautifulSoup(r.text, "html.parser")
+        bin_type = soup.find_all("span", {"class": "font-weight-bold"})
+        bin_dates = soup.find_all("strong")
+
+        entries = []
+        for idx in range(len(bin_type)):
+            bin_type_upper = bin_type[idx].text.upper()
+            entries.append(
+                Collection(
+                    date=datetime.strptime(
+                        bin_dates[idx].text.split(", ")[1], "%d %B %Y"
+                    ).date(),
+                    t=bin_type_upper,
+                    icon=_icon_for(bin_type_upper),
+                )
+            )
+
+        return entries