@camstack/addon-waste-collections 0.0.0-stage → 0.1.1

package/python/vendor/waste_collection_schedule/source/cheshire_east_gov_uk.py

@@ -0,0 +1,111 @@
+from datetime import datetime
+
+import requests
+import urllib3
+from bs4 import BeautifulSoup
+from waste_collection_schedule import Collection, Icons
+from waste_collection_schedule.exceptions import (
+    SourceArgumentException,
+    SourceArgumentExceptionMultiple,
+)
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+TITLE = "Cheshire East Council"
+DESCRIPTION = "Source for cheshireeast.gov.uk services for Cheshire East"
+URL = "https://cheshireeast.gov.uk"
+TEST_CASES = {
+    "houseUPRN-VerifyTrue": {"uprn": "100010132073", "verify": True},
+    "houseUPRN-VerifyFalse": {"uprn": "100010132073", "verify": False},
+    "houseAddress-VerifyTrue": {
+        "postcode": "WA16 0AY",
+        "name_number": "3",
+        "verify": True,
+    },
+    "houseAddress-VerifyFalse": {
+        "postcode": "WA16 0AY",
+        "name_number": "3",
+        "verify": False,
+    },
+}
+
+ICON_MAP = {
+    "General Waste": Icons.GENERAL_WASTE,
+    "Mixed Recycling": Icons.RECYCLING,
+    "Garden Waste": Icons.GARDEN,
+}
+
+
+class Source:
+    def __init__(self, uprn=None, postcode=None, name_number=None, verify=True):
+        self._uprn = uprn
+        self._postcode = postcode
+        self._name_number = name_number
+        self._verify = verify
+
+    def fetch(self):
+        session = requests.Session()
+
+        if self._postcode and self._name_number:
+            # Lookup postcode and number to get UPRN
+            params = {
+                "postcode": self._postcode,
+                "propertyname": self._name_number,
+            }
+            r = session.get(
+                "https://online.cheshireeast.gov.uk/MyCollectionDay/SearchByAjax/Search",
+                params=params,
+                verify=self._verify,
+            )
+            r.raise_for_status()
+            soup = BeautifulSoup(r.text, features="html.parser")
+            s = soup.find("a", attrs={"class": "get-job-details"})
+
+            if s is None or s["data-uprn"] is None:
+                raise SourceArgumentExceptionMultiple(
+                    ["postcode", "name_number"], "address not found"
+                )
+            self._uprn = s["data-uprn"]
+
+        if self._uprn is None:
+            raise SourceArgumentException(
+                "uprn",
+                "uprn not set but required if postcode and name_number are not set",
+            )
+
+        params = {"uprn": self._uprn}
+        r = session.get(
+            "https://online.cheshireeast.gov.uk/MyCollectionDay/SearchByAjax/GetBartecJobList",
+            params=params,
+            verify=self._verify,
+        )
+        r.raise_for_status()
+
+        soup = BeautifulSoup(r.text, features="html.parser")
+        s = soup.find_all("td", attrs={"class": "visible-cell"})
+
+        entries = []
+
+        for cell in s:
+            labels = cell.find_all("label")
+            if labels:
+                date = datetime.strptime(labels[1].text, "%d/%m/%Y").date()
+
+                if "general waste" in labels[2].text.lower():
+                    type = "General Waste"
+                elif "mixed recycling" in labels[2].text.lower():
+                    type = "Mixed Recycling"
+                elif "garden waste" in labels[2].text.lower():
+                    type = "Garden Waste"
+                else:
+                    type = "Unknown"
+
+                entries.append(
+                    Collection(
+                        date=date,
+                        t=type,
+                        icon=ICON_MAP.get(type),
+                    )
+                )
+
+        return entries

package/python/vendor/waste_collection_schedule/source/cheshire_west_and_chester_gov_uk.py

@@ -0,0 +1,159 @@
+import logging
+import time
+from datetime import datetime
+
+import requests
+from waste_collection_schedule import Collection, Icons  # type: ignore[attr-defined]
+
+TITLE = "Cheshire West and Chester Council"
+DESCRIPTION = (
+    "Source for waste collection services for Cheshire West and Chester Council"
+)
+URL = "https://www.cheshirewestandchester.gov.uk"
+
+TEST_CASES = {
+    "Chester": {"uprn": 100010030086},
+    "Chester - string": {"uprn": "100010030086"},
+    "Northwich": {"uprn": 10011715183},
+    "Hartford": {"uprn": 100010181592},
+    "Tarporley": {"uprn": 10014514851},
+    "Knutsford - no results": {
+        "uprn": 100010132172
+    },  # Knutsford is in Cheshire East and shouldn't have any collection information!
+}
+
+ICON_MAP = {
+    "Domestic": Icons.GENERAL_WASTE,
+    "Food": Icons.BIO_KITCHEN,
+    "Recycling": Icons.RECYCLING,
+    "Garden": Icons.GARDEN,
+}
+
+API_URLS = {
+    "session": "https://my.cheshirewestandchester.gov.uk/en/AchieveForms/?form_uri=sandbox-publish://AF-Process-0187a2f6-15cb-413a-8a3f-b6d14d63da57/AF-Stage-e18b38ff-be8a-45f4-ac14-f8821024f0c4/definition.json&redirectlink=%2Fen&cancelRedirectLink=%2Fen&consentMessage=yes",
+    "auth": "https://my.cheshirewestandchester.gov.uk/authapi/isauthenticated?uri=https%3A%2F%2Fmy.cheshirewestandchester.gov.uk%2Fen%2FAchieveForms%2F%3Fform_uri%3Dsandbox-publish%3A%2F%2FAF-Process-0187a2f6-15cb-413a-8a3f-b6d14d63da57%2FAF-Stage-e18b38ff-be8a-45f4-ac14-f8821024f0c4%2Fdefinition.json%26redirectlink%3D%2Fen%26cancelRedirectLink%3D%2Fen%26consentMessage%3Dyes&hostname=my.cheshirewestandchester.gov.uk&withCredentials=true",
+    "authResp": "https://my.cheshirewestandchester.gov.uk/apibroker/runLookup?id=609b918c7dd6d&repeat_against=&noRetry=false&getOnlyTokens=undefined&log_id=&app_name=AchieveForms",
+    "serviceTypes": "https://my.cheshirewestandchester.gov.uk/apibroker/runLookup?id=6101d1a29ba09&repeat_against=&noRetry=false&getOnlyTokens=undefined&log_id=&app_name=AchieveForms",
+    "schedule": "https://my.cheshirewestandchester.gov.uk/apibroker/runLookup?id=6101d23110243&repeat_against=&noRetry=false&getOnlyTokens=undefined&log_id=&app_name=AchieveForms",
+}
+
+HEADERS = {
+    "user-agent": "Mozilla/5.0",
+}
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class Source:
+    def __init__(self, uprn: str):
+        self._uprn = str(uprn)
+
+    def fetch(self):
+        s = requests.Session()
+
+        # Get session cookies
+        r0 = s.get(API_URLS["session"], headers=HEADERS)
+        r0.raise_for_status()
+
+        # Get session key from the PHPSESSID (in the cookies)
+        authRequest = s.get(API_URLS["auth"], headers=HEADERS)
+        authData = authRequest.json()
+        sessionKey = authData["auth-session"]
+
+        now = time.time_ns() // 1_000_000
+        urlNonce = str(now)
+
+        # Get AuthenticateResponse nonce
+        authRespRequest = s.post(
+            API_URLS["authResp"] + "&_" + urlNonce + "&sid=" + sessionKey,
+            headers=HEADERS,
+        )
+        authenticateResponseNonce = self.achieveFormsData(authRespRequest)["0"][
+            "AuthenticateResponse"
+        ]
+
+        # This payload is used for all subsequent requests
+        uprnPayload = {
+            "formValues": {
+                "Section 1": {
+                    "AuthenticateResponse": {
+                        "name": "AuthenticateResponse",
+                        "value": authenticateResponseNonce,
+                    },
+                    "UPRN": {"name": "UPRN", "value": self._uprn},
+                }
+            }
+        }
+
+        # Query service types with UPRN and map to generic service name
+        #
+        # Not all serviced UPRNs within the area share the same service type names!
+        # so we need to map them back to a generic service for the icon map...
+        #
+        # E.g. In Chester:
+        #  - Empty Black Sacks -> Domestic
+        #  - Empty Recycling   -> Recycling
+        #  - Empty 23L Caddy   -> Food
+        #  - Empty 240L Garden -> Garden
+        #
+        # and in a different area of Cheshire West, it could be:
+        #  - Empty 180l Domestic -> Domestic
+        #  - Empty 180L Blue     -> Recycling
+        #  - Empty 180L Red      -> Recycling
+        #  - Empty 23L Caddy     -> Food
+        #  - Empty 240L Garden   -> Garden
+        scheduleRequest = s.post(
+            API_URLS["serviceTypes"] + "&_" + urlNonce + "&sid=" + sessionKey,
+            headers=HEADERS,
+            json=uprnPayload,
+        )
+        data = self.achieveFormsData(scheduleRequest)
+
+        if len(data) < 1:
+            _LOGGER.warn("couldn't find service data for UPRN %s", self._uprn)
+            return []
+
+        # Map non-generic service type names to generic service type
+        serviceTypeToGenericService = {}
+        for service in data.values():
+            genericService = service["service"].strip()
+            if ICON_MAP.get(genericService) is not None:
+                serviceTypeToGenericService[service["serviceType"]] = genericService
+
+        # Now query the jobs (collection schedule)
+        scheduleRequest = s.post(
+            API_URLS["schedule"] + "&_" + urlNonce + "&sid=" + sessionKey,
+            headers=HEADERS,
+            json=uprnPayload,
+        )
+        data = self.achieveFormsData(scheduleRequest)
+
+        if len(data) < 1:
+            _LOGGER.warn("couldn't find collection data for UPRN %s", self._uprn)
+            return []
+
+        entries = []
+
+        for collection in data.values():
+            date = datetime.strptime(
+                collection["collectionDateTime"], "%Y-%m-%dT%H:%M:%S"
+            ).date()
+            collection_type = collection["serviceType"]
+            # Only emit the collection if it's a recognised collection type (I.e. Ignore: "BULKY BOOKINGS" and whatever else crops up)
+            if serviceTypeToGenericService.get(collection_type) is not None:
+                collection_type_generic = serviceTypeToGenericService[collection_type]
+                entries.append(
+                    Collection(
+                        date=date,
+                        t=collection_type,
+                        icon=ICON_MAP.get(collection_type_generic),
+                    )
+                )
+            else:
+                _LOGGER.debug("unknown collection_type %s", collection_type)
+
+        return entries
+
+    # unwraps data from an AchieveForms response
+    def achieveFormsData(self, data):
+        return data.json()["integration"]["transformed"]["rows_data"]

package/python/vendor/waste_collection_schedule/source/chesterfield_gov_uk.py

@@ -0,0 +1,117 @@
+import json
+import logging
+from datetime import datetime
+
+import requests
+import urllib3
+from waste_collection_schedule import Collection, Icons
+
+# With verify=True the POST fails due to a SSLCertVerificationError.
+# Using verify=False works, but is not ideal. The following links may provide a better way of dealing with this:
+# https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings
+# https://urllib3.readthedocs.io/en/1.26.x/user-guide.html#ssl
+# This line suppresses the InsecureRequestWarning when using verify=False
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+
+TITLE = "Chesterfield Borough Council"
+DESCRIPTION = "Source for waste collection services for Chesterfield Borough Council"
+URL = "https://www.chesterfield.gov.uk/"
+
+HEADERS = {
+    "user-agent": "Mozilla/5.0",
+}
+
+TEST_CASES = {
+    "Test_001": {"uprn": 74023685},
+    "Test_002": {"uprn": "74009625"},
+    "Test_003": {"uprn": "74035689"},
+    "Test_004": {"uprn": "74020930"},
+}
+
+ICON_MAP = {
+    "DOMESTIC REFUSE": Icons.GENERAL_WASTE,
+    "DOMESTIC RECYCLING": Icons.GENERAL_WASTE,
+    "DOMESTIC ORGANIC": Icons.ORGANIC,
+    "DOMESTIC PAID GARDEN": Icons.GARDEN,
+}
+
+API_URLS = {
+    "session": "https://www.chesterfield.gov.uk/bins-and-recycling/bin-collections/check-bin-collections.aspx",
+    "fwuid": "https://myaccount.chesterfield.gov.uk/anonymous/c/cbc_VE_CollectionDaysLO.app?aura.format=JSON&aura.formatAdapter=LIGHTNING_OUT",
+    "search": "https://myaccount.chesterfield.gov.uk/anonymous/aura?r=2&aura.ApexAction.execute=1",
+}
+
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class Source:
+    def __init__(self, uprn=None):
+        self._uprn = str(uprn)
+
+    def fetch(self):
+
+        s = requests.Session()
+        r = s.get(API_URLS["session"], headers=HEADERS, verify=False)
+
+        # Capture fwuid value
+        r = s.get(
+            API_URLS["fwuid"],
+            verify=False,
+            headers=HEADERS,
+        )
+        resp = json.loads(r.content)
+        fwuid = resp["auraConfig"]["context"]["fwuid"]
+
+        if self._uprn:
+            # POST request returns schedule for matching uprn
+            payload = {
+                "message": '{"actions":[{"id":"4;a","descriptor":"aura://ApexActionController/ACTION$execute","callingDescriptor":"UNKNOWN","params":{"namespace":"","classname":"CBC_VE_CollectionDays","method":"getServicesByUPRN","params":{"propertyUprn":"'
+                + self._uprn
+                + '","executedFrom":"Main Website"},"cacheable":false,"isContinuation":false}}]}',
+                "aura.context": '{"mode":"PROD","fwuid":"'
+                + fwuid
+                + '","app":"c:cbc_VE_CollectionDaysLO","loaded":{"APPLICATION@markup://c:cbc_VE_CollectionDaysLO":"pqeNg7kPWCbx1pO8sIjdLA"},"dn":[],"globals":{},"uad":true}',
+                "aura.pageURI": "/bins-and-recycling/bin-collections/check-bin-collections.aspx",
+                "aura.token": "null",
+            }
+            r = s.post(
+                API_URLS["search"],
+                data=payload,
+                verify=False,
+                headers=HEADERS,
+            )
+            data = json.loads(r.content)
+            # print(data)
+
+        entries = []
+
+        # Extract waste types and dates from json
+        for item in data["actions"][0]["returnValue"]["returnValue"]["serviceUnits"]:
+            try:
+                waste_type = item["serviceTasks"][0]["taskTypeName"]
+            except IndexError:
+                # Commercial collection schedule for Residential properties is empty generating IndexError
+                pass
+            else:
+                waste_type = str(waste_type).replace("Collect ", "")
+                try:
+                    dt_zulu = item["serviceTasks"][0]["serviceTaskSchedules"][0][
+                        "nextInstance"
+                    ]["currentScheduledDate"]
+                except (IndexError, KeyError):
+                    #  keys are missing from some entries, or keys have no values
+                    pass
+                else:
+                    dt_utc = datetime.strptime(dt_zulu, "%Y-%m-%dT%H:%M:%S.%f%z")
+                    dt_local = dt_utc.astimezone(None)
+                    entries.append(
+                        Collection(
+                            date=dt_local.date(),
+                            t=waste_type,
+                            icon=ICON_MAP.get(waste_type.upper()),
+                        )
+                    )
+
+        return entries

package/python/vendor/waste_collection_schedule/source/chichester_gov_uk.py

@@ -0,0 +1,72 @@
+import re
+from datetime import datetime
+
+from bs4 import BeautifulSoup
+from curl_cffi import requests
+from waste_collection_schedule import Collection, Icons
+
+TITLE = "Chichester District Council"
+DESCRIPTION = "Source for chichester.gov.uk services for Chichester"
+URL = "chichester.gov.uk"
+
+TEST_CASES = {
+    "Test_001": {"uprn": "010002476348"},
+    "Test_002": {"uprn": "100062612654"},
+    "Test_003": {"uprn": "100061745708"},
+}
+
+ICON_MAP = {
+    "General Waste": Icons.GENERAL_WASTE,
+    "Recycling": Icons.RECYCLING,
+    "Garden Recycling": Icons.GARDEN,
+}
+
+
+class Source:
+    def __init__(self, uprn):
+        self._uprn = uprn
+
+    def fetch(self):
+        session = requests.Session(impersonate="chrome124")
+        # Start a session
+        r = session.get("https://www.chichester.gov.uk/checkyourbinday")
+        r.raise_for_status()
+        soup = BeautifulSoup(r.text, features="html.parser")
+
+        # Extract form submission url
+        form = soup.find(
+            "form", attrs={"id": re.compile(r"WASTECOLLECTIONCALENDARV\d+_FORM")}
+        )
+        ID = form["id"].split("_")[0]
+        form_url = form["action"]
+
+        # Submit form
+        form_data = {
+            f"{ID}_FORMACTION_NEXT": "Submit",
+            f"{ID}_CALENDAR_UPRN": self._uprn,
+        }
+        r = session.post(form_url, data=form_data)
+        r.raise_for_status()
+
+        # Extract collection dates
+        soup = BeautifulSoup(r.text, features="html.parser")
+        entries = []
+        bin_divs = soup.find_all("div", class_=re.compile(r"binType-"))
+        for bin_div in bin_divs:
+            # Each collection is a table row
+            bin_type = bin_div.text.strip().title()
+            date_div = bin_div.find_next_sibling("div")
+            if not date_div:
+                continue
+
+            # date format "Friday 27 February 2026"
+            date = datetime.strptime(date_div.text.strip(), "%A %d %B %Y").date()
+            entries.append(
+                Collection(
+                    date=date,
+                    t=bin_type,
+                    icon=ICON_MAP.get(bin_type),
+                )
+            )
+
+        return entries

package/python/vendor/waste_collection_schedule/source/chiemgau_recycling_lk_rosenheim.py

@@ -0,0 +1,40 @@
+from datetime import datetime
+
+import requests
+from waste_collection_schedule import Collection, Icons
+
+TITLE = "Chiemgau Recycling - Landkreis Rosenheim"  # Title will show up in README.md and info.md
+DESCRIPTION = "Source script for paper waste collection in Landkreis Rosenheim area"  # Describe your source
+URL = "https://chiemgau-recycling.de"  # Insert url to service homepage. URL will show up in README.md and info.md
+COUNTRY = "de"
+TEST_CASES = {  # Insert arguments for test cases to be used by test_sources.py script
+    "Bruckmühl 1": {"district": "Bruckmühl 1"}
+}
+
+ICON_MAP = {
+    "Papier": Icons.PAPER,
+}
+
+API_URL = "https://blauetonne.stkn.org/lk_rosenheim"
+
+
+class Source:
+    def __init__(self, district):
+        self.district = district
+
+    def fetch(self):
+        entries = []
+
+        r = requests.get(f"{API_URL}", params={"district": self.district})
+        r.raise_for_status()
+
+        for date in r.json():
+            entries.append(
+                Collection(
+                    date=datetime.fromisoformat(date).date(),  # Collection date
+                    t="Papier Tonne",  # Collection type
+                    icon=ICON_MAP.get("Papier"),  # Collection icon
+                )
+            )
+
+        return entries

package/python/vendor/waste_collection_schedule/source/chiltern_gov_uk.py

@@ -0,0 +1,20 @@
+import logging
+
+from waste_collection_schedule.source.iapp_itouchvision_com import (
+    Source as iapp_iTouchVision,
+)
+
+_LOGGER = logging.getLogger(__name__)
+TITLE = "Deprecated: Buckinghamshire"
+DESCRIPTION = "Deprecated: use the iapp_iTouchVision instead."
+URL = "https://chiltern.gov.uk"
+
+TEST_CASES = {
+    "Test_004": {"uprn": 10094593823},
+}
+
+
+class Source(iapp_iTouchVision):
+    def __init__(self, uprn):
+        super().__init__(uprn, "BUCKINGHAMSHIRE")
+        _LOGGER.warning("This source is deprecated. Use iapp_iTouchVision instead.")

package/python/vendor/waste_collection_schedule/source/chorley_gov_uk.py

@@ -0,0 +1,132 @@
+import datetime
+
+import requests
+from bs4 import BeautifulSoup
+from waste_collection_schedule import Collection, Icons  # type: ignore[attr-defined]
+
+TITLE = "Chorley Council"
+DESCRIPTION = "Source for chorley.gov.uk services for Chorley Council, UK."
+URL = "https://www.chorley.gov.uk"
+TEST_CASES = {
+    "20 Leatherland Drive": {
+        "postcode": "PR6 7YD",
+        "uprn": "010091497098",
+    },
+}
+
+ICON_MAP = {
+    "Residual Waste": Icons.GENERAL_WASTE,
+    "Dry Mixed Recycling": Icons.RECYCLING,
+    "Food Waste": Icons.BIO_KITCHEN,
+    "Garden Waste": Icons.GARDEN,
+    "Paper and Card": Icons.PAPER,
+}
+
+HOW_TO_GET_ARGUMENTS_DESCRIPTION = {
+    "en": (
+        "Go to https://www.chorley.gov.uk/bincollectiondays "
+        "and enter your postcode. Use browser dev tools to "
+        "find the UPRN (the option value in the address "
+        "dropdown)."
+    ),
+}
+
+PARAM_DESCRIPTIONS = {
+    "en": {
+        "postcode": "Your postcode, e.g. PR6 7YD.",
+        "uprn": "The UPRN from the address dropdown.",
+    }
+}
+
+PARAM_TRANSLATIONS = {
+    "en": {
+        "postcode": "Postcode",
+        "uprn": "UPRN",
+    }
+}
+
+FORM_URL = "https://forms.chorleysouthribble.gov.uk/xfp/form/71"
+FIELD_PREFIX = "qc576c657112a8277ba6f954ebc0490c946168363"
+
+
+class Source:
+    def __init__(self, postcode: str, uprn: str):
+        self._postcode = postcode.strip()
+        self._uprn = str(uprn).strip()
+
+    def fetch(self) -> list[Collection]:
+        session = requests.Session()
+        session.headers.update(
+            {
+                "User-Agent": (
+                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                    "AppleWebKit/537.36 (KHTML, like Gecko) "
+                    "Chrome/124.0.0.0 Safari/537.36"
+                ),
+            }
+        )
+
+        # Step 1: GET form and extract token
+        r = session.get(
+            FORM_URL,
+            params={"page": "198", "locale": "en_GB"},
+        )
+        r.raise_for_status()
+        soup = BeautifulSoup(r.text, "html.parser")
+        token = soup.find("input", {"name": "__token"}).attrs["value"]
+
+        # Step 2: POST postcode
+        data = {
+            "__token": token,
+            "page": "198",
+            "locale": "en_GB",
+            f"{FIELD_PREFIX}_0_0": self._postcode,
+            "next": "Next",
+        }
+        r = session.post(FORM_URL, data=data)
+        r.raise_for_status()
+        soup = BeautifulSoup(r.text, "html.parser")
+        token = soup.find("input", {"name": "__token"}).attrs["value"]
+
+        # Step 3: POST selected address
+        data = {
+            "__token": token,
+            "page": "198",
+            "locale": "en_GB",
+            f"{FIELD_PREFIX}_0_0": self._postcode,
+            f"{FIELD_PREFIX}_1_0": self._uprn,
+            "next": "Next",
+        }
+        r = session.post(FORM_URL, data=data)
+        r.raise_for_status()
+        soup = BeautifulSoup(r.text, "html.parser")
+
+        # Step 4: Parse collection table
+        entries = []
+        for tr in soup.find_all("tr")[1:]:
+            cells = tr.find_all("td")
+            if len(cells) < 2:
+                continue
+
+            service = cells[0].get_text(strip=True)
+            date_text = cells[1].get_text(strip=True)
+
+            # Remove " Collection Service" suffix
+            waste_type = service.replace(" Collection Service", "")
+
+            try:
+                collection_date = datetime.datetime.strptime(
+                    date_text, "%d/%m/%y"
+                ).date()
+            except ValueError:
+                continue
+
+            entries.append(
+                Collection(
+                    date=collection_date,
+                    t=waste_type,
+                    icon=ICON_MAP.get(waste_type),
+                )
+            )
+
+        return entries