@camstack/addon-waste-collections 0.0.0-stage → 0.1.1

package/python/vendor/waste_collection_schedule/source/cotswold_gov_uk.py

@@ -0,0 +1,328 @@
+import json
+import logging
+import re
+from datetime import date, datetime
+
+import requests
+import urllib3
+from dateutil.parser import parse
+from waste_collection_schedule import Collection, Icons  # type: ignore[attr-defined]
+from waste_collection_schedule.exceptions import (
+    SourceArgumentNotFoundWithSuggestions,
+)
+
+# With verify=True the POST fails due to a SSLCertVerificationError.
+# Using verify=False works, but is not ideal. The following links may provide a better way of dealing with this:
+# https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings
+# https://urllib3.readthedocs.io/en/1.26.x/user-guide.html#ssl
+# This line suppresses the InsecureRequestWarning when using verify=False
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+_LOGGER = logging.getLogger(__name__)
+
+TITLE = "Cotswold District Council"
+DESCRIPTION = "Source for Cotswold District Council, UK."
+URL = "https://www.cotswold.gov.uk/"
+COUNTRY = "uk"
+
+TEST_CASES = {
+    "Parsons Piece, 1 Glebe Lane, Kemble, Cirencester": {
+        "address": "PARSONS PIECE, 1 GLEBE LANE, KEMBLE, CIRENCESTER, GL7 6BD"
+    },
+}
+
+HOW_TO_GET_ARGUMENTS_DESCRIPTION = {
+    "en": (
+        "Go to the Cotswold District Council waste collection enquiry page "
+        "(https://community.cotswold.gov.uk/s/waste-collection-enquiry) and "
+        "start typing your postcode or street name into the address lookup. "
+        "Select your property from the autocomplete suggestions and use the "
+        "full address string exactly as it appears (e.g. "
+        "'10 HAVEN HILL, CIRENCESTER, GL7 1BN')."
+    ),
+}
+
+PARAM_TRANSLATIONS = {
+    "en": {"address": "Address"},
+}
+
+PARAM_DESCRIPTIONS = {
+    "en": {
+        "address": (
+            "The full address string as returned by the lookup on the "
+            "Cotswold District Council waste collection enquiry page."
+        ),
+    },
+}
+
+ICON_MAP = {
+    "bin": Icons.ORGANIC,
+    "refuse": Icons.GENERAL_WASTE,
+    "recycling": Icons.RECYCLING,
+    "caddy": Icons.BIO_KITCHEN,
+    "box": Icons.RECYCLING,
+    "garden": Icons.GARDEN,
+}
+
+REGEX_AURA_CONFIG = re.compile(r"var\s+auraConfig\s*=\s*(.*?),\n")
+
+API_URL = "https://community.cotswold.gov.uk/s/sfsites/aura"
+SEARCH_PAGE = "https://community.cotswold.gov.uk/s/waste-collection-enquiry"
+HEADERS = {
+    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:115.0) Gecko/20100101 Firefox/115.0",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+}
+
+
+class Source:
+    def __init__(self, address: str):
+        self._address: str = address
+        self._session = requests.Session()
+
+    @staticmethod
+    def _parse_date(date_str: str) -> date:
+        if date_str.lower() == "today":
+            return date.today()
+        if date_str.lower() == "tomorrow":
+            return date.today().replace(day=date.today().day + 1)
+
+        # Tue, 20 August
+        return parse(date_str, default=datetime.now()).date()
+
+    def _init_connection(self):
+        self._session = requests.Session()
+        r = self._session.get(SEARCH_PAGE, verify=False, headers=HEADERS)
+        r.raise_for_status()
+        aura_regex = REGEX_AURA_CONFIG.search(r.text, re.MULTILINE)
+
+        if not aura_regex:
+            raise Exception("Could not find aura config")
+        json_data = json.loads(aura_regex.group(1))
+        fwuid = json_data["context"]["fwuid"]
+        loaded = json_data["context"]["loaded"]
+        self._aura_context = json.dumps(
+            {
+                "mode": "PROD",
+                "fwuid": fwuid,
+                "app": "siteforce:communityApp",
+                "loaded": loaded,
+                "dn": [],
+                "globals": {},
+                "uad": False,
+            }
+        )
+
+    def _request(
+        self, r: int, aura_method: str, message: str | dict
+    ) -> requests.Response:
+        if isinstance(message, dict):
+            message = json.dumps(message)
+        response = self._session.post(
+            API_URL,
+            params={
+                "r": r,
+                aura_method: 1,
+            },
+            data={
+                "message": message,
+                "aura.context": self._aura_context,
+                "aura.pageURI": "/s/waste-collection-enquiry",
+                "aura.token": "null",
+            },
+            verify=False,
+            headers=HEADERS,
+        )
+        response.raise_for_status()
+        return response
+
+    def _match_address(self, address: str | None) -> bool:
+        if address is None:
+            return False
+        return self._address.lower().replace(" ", "").replace(".", "").replace(
+            ",", ""
+        ) == address.lower().replace(" ", "").replace(".", "").replace(",", "")
+
+    def fetch(self) -> list[Collection]:
+        self._init_connection()
+        entries = []
+        message1 = {
+            "actions": [
+                {
+                    "id": "85;a",
+                    "descriptor": "aura://FlowRuntimeConnectController/ACTION$startFlow",
+                    "callingDescriptor": "UNKNOWN",
+                    "params": {
+                        "flowDevName": "WebFormAlloyWasteCollectionEnquiry",
+                        "arguments": '[{"name":"vClientCode","type":"String","supportsRecordId":false,"value":"CDC"}]',
+                        "enableTrace": False,
+                        "enableRollbackMode": False,
+                        "debugAsUserId": "",
+                        "useLatestSubflow": False,
+                    },
+                }
+            ]
+        }
+        r = self._request(5, "aura.FlowRuntimeConnect.startFlow", message1)
+        serialized_state = r.json()["actions"][0]["returnValue"]["response"][
+            "serializedEncodedState"
+        ]
+
+        message2 = {
+            "actions": [
+                {
+                    "id": "89;a",
+                    "descriptor": "aura://LookupController/ACTION$lookup",
+                    "callingDescriptor": "UNKNOWN",
+                    "params": {
+                        "objectApiName": "Case",
+                        "fieldApiName": "Property__c",
+                        "pageParam": 1,
+                        "pageSize": 25,
+                        "q": self._address,
+                        "searchType": "TypeAhead",
+                        "targetApiName": "Property__c",
+                        "body": {
+                            "sourceRecord": {"apiName": "Case", "fields": {"Id": None}}
+                        },
+                    },
+                }
+            ]
+        }
+        r = self._request(1, "aura.LookupController.lookupaura.Lookup.lookup", message2)
+
+        addresses = r.json()["actions"][0]["returnValue"]["lookupResults"][
+            "Property__c"
+        ]["records"]
+        address_id = None
+        address_name = None
+        address_set = set()
+        for address in addresses:
+            potential_address = address["fields"]["Name"]["value"]
+            address_set.add(potential_address)
+            if self._match_address(potential_address):
+                address_id = address["id"]
+                address_name = potential_address
+                break
+        address_set -= {None, ""}
+        if not address_id:
+            raise SourceArgumentNotFoundWithSuggestions(
+                "address",
+                self._address,
+                address_set,
+            )
+
+        message3 = {
+            "actions": [
+                {
+                    "id": "114;a",
+                    "descriptor": "aura://RecordUiController/ACTION$getRecordWithFields",
+                    "callingDescriptor": "UNKNOWN",
+                    "params": {"recordId": address_id, "fields": ["Property__c.Name"]},
+                }
+            ]
+        }
+        r = self._request(25, "aura.RecordUi.getRecordWithFields", message3)
+
+        message4 = {
+            "actions": [
+                {
+                    "id": "123;a",
+                    "descriptor": "aura://FlowRuntimeConnectController/ACTION$navigateFlow",
+                    "callingDescriptor": "UNKNOWN",
+                    "params": {
+                        "request": {
+                            "action": "NEXT",
+                            "serializedState": serialized_state,
+                            "fields": [
+                                {
+                                    "field": "Property.recordId",
+                                    "value": address_id,
+                                    "isVisible": True,
+                                },
+                                {
+                                    "field": "Property.recordIds",
+                                    "value": [address_id],
+                                    "isVisible": True,
+                                },
+                                {
+                                    "field": "Property.recordName",
+                                    "value": address_name,
+                                    "isVisible": True,
+                                },
+                            ],
+                            "uiElementVisited": True,
+                            "enableTrace": False,
+                            "lcErrors": {},
+                        }
+                    },
+                }
+            ]
+        }
+
+        r = self._request(26, "aura.FlowRuntimeConnect.navigateFlow", message4)
+        serialized_state = r.json()["actions"][0]["returnValue"]["response"][
+            "serializedEncodedState"
+        ]
+
+        message5 = {
+            "actions": [
+                {
+                    "id": "125;a",
+                    "descriptor": "aura://FlowRuntimeConnectController/ACTION$navigateFlow",
+                    "callingDescriptor": "UNKNOWN",
+                    "params": {
+                        "request": {
+                            "action": "CONTINUE_AFTER_COMMIT",
+                            "serializedState": serialized_state,
+                            "fields": [],
+                            "uiElementVisited": True,
+                            "enableTrace": False,
+                        }
+                    },
+                }
+            ]
+        }
+
+        r = self._request(27, "aura.FlowRuntimeConnect.navigateFlow", message5)
+        serialized_state = r.json()["actions"][0]["returnValue"]["response"][
+            "serializedEncodedState"
+        ]
+
+        message6 = {
+            "actions": [
+                {
+                    "id": "127;a",
+                    "descriptor": "aura://FlowRuntimeConnectController/ACTION$navigateFlow",
+                    "callingDescriptor": "UNKNOWN",
+                    "params": {
+                        "request": {
+                            "action": "CONTINUE_AFTER_COMMIT",
+                            "serializedState": serialized_state,
+                            "fields": [],
+                            "uiElementVisited": True,
+                            "enableTrace": False,
+                        }
+                    },
+                }
+            ]
+        }
+        r = self._request(28, "aura.FlowRuntimeConnect.navigateFlow", message6)
+
+        table_value = r.json()["actions"][0]["returnValue"]["response"]["fields"][1][
+            "inputs"
+        ][3]["value"]
+
+        table = json.loads(table_value)
+
+        for row in table:
+            date_str = row["col2"]
+            bin_type = row["col1"]
+            try:
+                date_ = self._parse_date(date_str)
+            except ValueError:
+                _LOGGER.warning(f"date in unknown format: {date_str}")
+                continue
+            icon = ICON_MAP.get(bin_type.split()[-1].lower())
+            entries.append(Collection(date=date_, t=bin_type, icon=icon))
+
+        return entries

package/python/vendor/waste_collection_schedule/source/coventry_gov_uk.py

@@ -0,0 +1,178 @@
+import logging
+import re
+from datetime import date, datetime
+
+import requests
+from bs4 import BeautifulSoup
+from waste_collection_schedule import Collection, Icons  # type: ignore[attr-defined]
+from waste_collection_schedule.exceptions import (
+    SourceArgumentException,
+    SourceArgumentNotFound,
+)
+
+TITLE = "Coventry City Council"
+DESCRIPTION = "Source for waste collection services for Coventry City Council"
+URL = "https://www.coventry.gov.uk/"
+
+HEADERS = {"user-agent": "Mozilla/5.0"}
+API_URLS = {
+    "search": "https://www.coventry.gov.uk/directory/search",
+    "directory_record": "https://www.coventry.gov.uk",
+    # Where the council moved bin days to. This source cannot read it, so the
+    # only useful thing left to hand the user is the address of the service.
+    "my_account": "https://myaccount.coventry.gov.uk/service/find_my_bin_day",
+}
+TEST_CASES = {
+    "Test_001": {
+        "street": "Linwood Drive",
+    },
+    "Test_002": {
+        "street": "Cromwell Lane",
+    },
+    "Test_003": {
+        "street": "Lutterworth Road",
+    },
+}
+_LOGGER = logging.getLogger(__name__)
+ICON_MAP = {
+    "Recycling (blue-lidded bin)": Icons.RECYCLING,
+    "Household waste (green-lidded bin)": Icons.GENERAL_WASTE,
+    "Garden waste (brown-lidded bin)": Icons.GARDEN,
+    "Food waste caddy": Icons.BIO_KITCHEN,
+}
+
+PARAM_TRANSLATIONS = {
+    "en": {
+        "street": "Your street name, as it appears on the Coventry City Council website",
+    }
+}
+PARAM_DESCRIPTIONS = {
+    "en": {
+        "street": "Your street name, as it appears on the Coventry City Council website",
+    }
+}
+
+
+def _normalize_space(text: str) -> str:
+    return re.sub(r"\s+", " ", text.replace("\xa0", " ")).strip()
+
+
+class Source:
+    def __init__(self, street: str):
+        self._street: str = street
+
+    def fetch(self) -> list[Collection]:
+        s = requests.Session()
+
+        # search for address to get directory record
+        params: dict = {
+            "directoryID": "82",
+            "showInMap": "",
+            "keywords": self._street,
+            "search": "Search",
+        }
+        r = s.get(API_URLS["search"], headers=HEADERS, params=params, timeout=30)
+        if r.status_code == 404:
+            # The council retired the street directory this source searches;
+            # /directory/search now serves its "page not found" page, whose own
+            # navigation links carry the same CSS class the street results did.
+            # Without this check the site menu is offered as a list of streets.
+            raise SourceArgumentException(
+                "street",
+                "Coventry has retired the street directory this lookup uses "
+                f"({API_URLS['search']} now returns 404). Collection days have "
+                f"moved to {API_URLS['my_account']}, which this source does "
+                "not read yet.",
+            )
+        soup: BeautifulSoup = BeautifulSoup(r.content, "html.parser")
+        list_links: list = soup.find_all("a", {"class": "list__link"})
+        directory_record: str | None = None
+        for link in list_links:
+            if self._street.upper() in link.text.upper():
+                directory_record = link["href"]
+                break
+
+        if directory_record is None:
+            # RuntimeError reached the caller as a server error, so an ordinary
+            # misspelling was reported as an outage.
+            raise SourceArgumentNotFound(
+                "street",
+                self._street,
+                "The council's street directory returned no match. Check the "
+                "spelling, and include the street type (Road, Drive).",
+            )
+
+        # use directory record to get collection day
+        r = s.get(
+            API_URLS["directory_record"] + directory_record, headers=HEADERS, timeout=30
+        )
+        soup = BeautifulSoup(r.content, "html.parser")
+        buttons: list = soup.find_all("a", {"class": "button"})
+        schedule: str | None = None
+        for button in buttons:
+            if "bin" in button["href"]:
+                schedule = button["href"]
+                break
+
+        if schedule is None:
+            raise SourceArgumentNotFound(
+                "street",
+                self._street,
+                "The council's directory has this street but no bin collection "
+                "calendar for it.",
+            )
+
+        # Use the collection calendar link to get the current schedule.
+        r = s.get(schedule, headers=HEADERS, timeout=30)
+        r.raise_for_status()
+        soup = BeautifulSoup(r.content, "html.parser")
+        entries: list[Collection] = []
+
+        # Coventry changed its calendars from colon-delimited prose to one
+        # table per month. Each waste column contains Yes/No for every date.
+        today = date.today()
+        for table in soup.select("div.editor table"):
+            heading = table.find_previous(["h2", "h3"])
+            year_match = re.search(
+                r"\b(20\d{2})\b", heading.get_text() if heading else ""
+            )
+            if year_match is None:
+                continue
+            year = year_match.group(1)
+
+            headers = [
+                _normalize_space(cell.get_text(" ", strip=True))
+                for cell in table.select("thead th")
+            ]
+            if len(headers) < 2:
+                continue
+
+            for row in table.select("tbody tr"):
+                cells = row.find_all(["th", "td"])
+                if len(cells) != len(headers):
+                    continue
+                date_text = _normalize_space(cells[0].get_text(" ", strip=True))
+                try:
+                    waste_date = datetime.strptime(
+                        f"{date_text} {year}", "%A %d %B %Y"
+                    ).date()
+                except ValueError:
+                    _LOGGER.warning(
+                        "Could not parse Coventry collection date '%s'", date_text
+                    )
+                    continue
+                if waste_date < today:
+                    continue
+
+                for waste_type, cell in zip(headers[1:], cells[1:], strict=True):
+                    if not re.search(r"\byes\b", cell.get_text(" ", strip=True), re.I):
+                        continue
+                    entries.append(
+                        Collection(
+                            date=waste_date,
+                            t=waste_type,
+                            icon=ICON_MAP.get(waste_type, Icons.GENERAL_WASTE),
+                        )
+                    )
+
+        return entries

package/python/vendor/waste_collection_schedule/source/crawley_gov_uk.py

@@ -0,0 +1,107 @@
+# Credit where it's due:
+# This is based on the elmbridge_gov_uk source
+
+
+from datetime import datetime
+
+import requests
+from dateutil.parser import parse
+from waste_collection_schedule import Collection, Icons  # type: ignore[attr-defined]
+from waste_collection_schedule.service.AchieveForms import init_session, run_lookup
+
+TITLE = "Crawley Borough Council (myCrawley)"
+DESCRIPTION = "Source for Crawley Borough Council (myCrawley)."
+URL = "https://crawley.gov.uk/"
+TEST_CASES = {
+    "Feroners Cl": {"uprn": "100061775179"},
+    "Peterborough Road": {"uprn": 100061787552, "usrn": 9700731},
+}
+
+
+ICON_MAP = {
+    "Rubbish and Small Electricals Collection": Icons.ELECTRONICS,
+    "Glass": Icons.GLASS,
+    "GREENbin Garden Waste Collection": Icons.GARDEN,
+    "Paper": Icons.PAPER,
+    "Recycling and Textiles Collection": Icons.TEXTILE,
+}
+
+
+BASE_URL = "https://my.crawley.gov.uk"
+INTIAL_URL = f"{BASE_URL}/en/service/check_my_bin_collection"
+AUTH_URL = f"{BASE_URL}/authapi/isauthenticated"
+AUTH_TEST = f"{BASE_URL}/apibroker/domain/my.crawley.gov.uk"
+API_URL = f"{BASE_URL}/apibroker/runLookup"
+
+LOOKUP_ID = "5b4f0ec5f13f4"
+
+
+class Source:
+    def __init__(self, uprn: str | int, usrn: str | int | None = None):
+        self._uprn = str(uprn)
+        self._usrn = str(usrn) if usrn else None
+
+    def _get_payload(self) -> dict[str, dict]:
+        now = datetime.now()
+        return {
+            "Address": {
+                "address": {
+                    "value": {
+                        "Address": {
+                            "usrn": {"value": self._usrn or "0000"},
+                            "uprn": {"value": self._uprn},
+                        }
+                    }
+                },
+                "dayConverted": {"value": now.strftime("%d/%m/%Y")},
+                "getCollection": {"value": "true"},
+                "getWorksheets": {"value": "false"},
+            }
+        }
+
+    def get_collections(
+        self, session_key: str, session: requests.Session
+    ) -> list[Collection]:
+        result = run_lookup(
+            session,
+            API_URL,
+            session_key,
+            LOOKUP_ID,
+            self._get_payload(),
+        )
+        return list(result["integration"]["transformed"]["rows_data"].values())
+
+    def fetch(self) -> list[Collection]:
+        session = requests.Session()
+        session_key = init_session(
+            session,
+            INTIAL_URL,
+            AUTH_URL,
+            "elmbridge-self.achieveservice.com",
+            auth_test_url=AUTH_TEST,
+        )
+        collections = self.get_collections(session_key, session)
+        date_parse_failed = []
+        entries = []
+        for collection in collections:
+            for key in [
+                k for k in collection.keys() if k.endswith(("DateCurrent", "DateNext"))
+            ]:
+                date_str = collection[key]
+                try:
+                    date = parse(date_str, dayfirst=True).date()
+                except ValueError:
+                    date_parse_failed.append(date_str)
+                if not date_str:
+                    continue
+                bin_type = key.split("Date")[0]
+                icon = ICON_MAP.get(bin_type)
+                entries.append(Collection(date=date, t=bin_type, icon=icon))
+
+        if not entries:
+            if date_parse_failed:
+                raise ValueError(
+                    f"Failed to parse dates: {', '.join(date_parse_failed)}"
+                )
+            raise ValueError(f"No collections found for {self._uprn}")
+        return entries