@camstack/addon-waste-collections 0.0.0-stage → 0.1.2

package/python/vendor/waste_collection_schedule/source/southderbyshire_gov_uk.py

@@ -0,0 +1,65 @@
+import re
+from datetime import datetime
+
+from bs4 import BeautifulSoup
+from curl_cffi import requests
+from waste_collection_schedule import Collection, Icons
+from waste_collection_schedule.exceptions import SourceArgumentNotFound
+
+TITLE = "South Derbyshire District Council"
+DESCRIPTION = "Source for www.southderbyshire.gov.uk services for South Derbyshire "
+URL = "https://www.southderbyshire.gov.uk/"
+TEST_CASES = {
+    "test case 1": {"uprn": "100030233745"},
+    "test case 2": {"uprn": "10090304958"},
+}
+
+API_URL = "https://maps.southderbyshire.gov.uk/iShareLIVE.web/getdata.aspx?RequestType=LocalInfo&ms=mapsources/MyHouse&format=JSON&group=Recycling%20Bins%20and%20Waste|Next%20Bin%20Collections&uid="
+
+ICON_MAP = {
+    "Black": Icons.GENERAL_WASTE,
+    "Green": Icons.RECYCLING,
+    "Brown": Icons.ORGANIC,
+    "Podback": Icons.BIO_KITCHEN,
+}
+
+
+class Source:
+    def __init__(self, uprn: str):
+        self._uprn = uprn
+
+    def _extract_date(self, text):
+        # find date and return
+        date = re.search(r"(\d{2} \w+ \d{4})", text).group(1)
+        date = datetime.strptime(date, "%d %B %Y").date()
+        return date
+
+    def fetch(self):
+        entries = []
+
+        session = requests.Session(impersonate="chrome")
+
+        r = session.get(f"{API_URL}{self._uprn}")
+        soup = BeautifulSoup(
+            r.json()["Results"]["Next_Bin_Collections"]["_"], features="html.parser"
+        )
+        collections = soup.find_all("div", recursive=False)
+
+        if not collections:
+            raise SourceArgumentNotFound("uprn", self._uprn)
+
+        for collection in collections:
+            bintypes = re.findall(
+                r"Green|Brown|Black|Podback", collection.find("img")["alt"]
+            )
+
+            for bintype in bintypes:
+                entries.append(
+                    Collection(
+                        date=self._extract_date(collection.text),
+                        t=bintype if bintype == "Podback" else bintype + " bin",
+                        icon=ICON_MAP.get(bintype),
+                    )
+                )
+
+        return entries

package/python/vendor/waste_collection_schedule/source/southend_gov_uk.py

@@ -0,0 +1,69 @@
+from waste_collection_schedule import Collection, Icons  # type: ignore[attr-defined]
+from waste_collection_schedule.exceptions import SourceArgumentRequired
+from waste_collection_schedule.service.uk_cloud9_apps import Cloud9Client
+
+TITLE = "Southend-on-Sea City Council"
+DESCRIPTION = (
+    "Source for southend.gov.uk services for Southend-on-Sea City Council, UK."
+)
+URL = "https://www.southend.gov.uk"
+COUNTRY = "uk"
+TEST_CASES = {
+    "Test_001": {"uprn": 100090691871},
+    "Test_002": {"uprn": "100090700485"},
+    "Test_003": {
+        "postcode": "SS3 9JD",
+        "address": "38 Thorpedene Gardens, Shoeburyness",
+    },
+}
+HOW_TO_GET_ARGUMENTS_DESCRIPTION = {
+    "en": "You can find your UPRN by visiting [Find My Address](https://www.findmyaddress.co.uk) and entering your address details."
+}
+PARAM_TRANSLATIONS = {
+    "en": {
+        "uprn": "Unique Property Reference Number (UPRN)",
+        "postcode": "Postcode (legacy, use UPRN instead)",
+        "address": "Address (legacy, use UPRN instead)",
+    }
+}
+PARAM_DESCRIPTIONS = {
+    "en": {
+        "uprn": "Unique Property Reference Number (UPRN)",
+        "postcode": "Postcode (legacy, use UPRN instead)",
+        "address": "Address (legacy, use UPRN instead)",
+    }
+}
+ICON_MAP = {
+    "garden": Icons.GARDEN,
+    "food": Icons.BIO_KITCHEN,
+    "recycl": Icons.RECYCLING,
+    "refuse": Icons.GENERAL_WASTE,
+}
+
+
+class Source:
+    def __init__(
+        self,
+        uprn: str | int | None = None,
+        postcode: str | None = None,
+        address: str | None = None,
+    ):
+        self._client = Cloud9Client("southend", icon_keywords=ICON_MAP)
+        self._uprn = str(uprn) if uprn else None
+        self._postcode = postcode
+        self._address = address
+
+    def fetch(self) -> list[Collection]:
+        if self._uprn:
+            return self._client.fetch_by_uprn(self._uprn)
+        if not self._postcode:
+            raise SourceArgumentRequired("uprn", "Provide a UPRN or postcode + address")
+        return self._client.fetch_by_address(
+            postcode=self._postcode,
+            address_string=self._address or "",
+            # The client's suggestions are whole address strings and it reports
+            # the address it was given as the offending value, so they belong to
+            # the address argument: picking one has to refine `address`, not
+            # overwrite the postcode with a full address.
+            argument_name="address",
+        )

package/python/vendor/waste_collection_schedule/source/southglos_gov_uk.py

@@ -0,0 +1,59 @@
+import datetime
+
+import requests
+from waste_collection_schedule import Collection, Icons
+
+TITLE = "South Gloucestershire Council"  # Title will show up in README.md and info.md
+DESCRIPTION = "Source script for southglos.gov.uk"  # Describe your source
+URL = "https://southglos.gov.uk"  # Insert url to service homepage. URL will show up in README.md and info.md
+HEADERS = {"user-agent": "Mozilla/5.0"}
+TEST_CASES = {  # Insert arguments for test cases to be used by test_sources.py script
+    "Test_001": {"uprn": "643346"},
+    "Test_002": {"uprn": "641084"},
+}
+ICON_MAP = {
+    "BLACK BIN": Icons.GENERAL_WASTE,
+    "RECYCLING": Icons.RECYCLING,
+    "GARDEN WASTE": Icons.GARDEN,
+    "FOOD BIN": Icons.BIO_KITCHEN,
+    "AHP BIN": Icons.GENERAL_WASTE,
+}
+WASTE_MAP = {  # map new collection names to old collection names for compatibility
+    "Refuse": "BLACK BIN",
+    "Recycling": "RECYCLING",
+    "Garden": "GARDEN WASTE",
+    "Food": "FOOD BIN",
+    "AHP": "AHP BIN",
+}
+
+
+class Source:
+    def __init__(
+        self, uprn: str
+    ):  # argX correspond to the args dict in the source configuration
+        self._uprn = uprn
+
+    def fetch(self):
+        session = requests.Session()
+        r = session.get(
+            f"https://api.southglos.gov.uk/wastecomp/GetCollectionDetails?uprn={self._uprn}",
+            headers=HEADERS,
+        )
+        r.raise_for_status()
+        pickups = r.json()
+
+        entries = []
+        for item in pickups["value"]:
+            next_collection = item.get("hso_nextcollection")
+            if next_collection:
+                entries.append(
+                    Collection(
+                        date=datetime.datetime.strptime(
+                            item["hso_nextcollection"].split("T")[0], "%Y-%m-%d"
+                        ).date(),
+                        t=WASTE_MAP[item["hso_servicename"]],
+                        icon=ICON_MAP.get(WASTE_MAP[item["hso_servicename"]]),
+                    )
+                )
+
+        return entries

package/python/vendor/waste_collection_schedule/source/southkesteven_gov_uk.py

@@ -0,0 +1,114 @@
+import logging
+from datetime import datetime
+
+import bs4
+import requests
+from waste_collection_schedule import Collection, Icons  # type: ignore[attr-defined]
+from waste_collection_schedule.service.FirmstepSelfService import (
+    get_hidden_form_inputs,
+    lookup_addresses,
+)
+
+URL = "https://southkesteven.gov.uk"
+FORM_URL = (
+    "https://selfservice.southkesteven.gov.uk/"
+    "renderform?k=2074C945A63DDC0D18F1EB74DA230AC3122958B1&t=213"
+)
+RENDER_URL = "https://selfservice.southkesteven.gov.uk/RenderForm"
+ADDRESS_LOOKUP_URL = "https://selfservice.southkesteven.gov.uk/core/addresslookup"
+TEST_CASES = {
+    "Bourne": {"address_id": "PE10 0RX"},
+    "Long Bennington": {"address_id": "NG23 5EQ"},
+    "Grantham": {"address_id": "NG31 6NP"},
+}
+_LOGGER = logging.getLogger(__name__)
+ICON_MAP = {
+    "black": Icons.GENERAL_WASTE,
+    "gray": Icons.RECYCLING,
+    "green": Icons.ORGANIC,
+    "purple": Icons.NEWSPAPER,
+}
+
+
+TITLE = "South Kesteven District Council"
+DESCRIPTION = "Source for southkesteven.gov.uk services for South Kesteven, UK."
+
+
+class Source:
+    def __init__(self, address_id):
+        self._address_id = str(address_id)
+
+    def fetch(self):
+        session = requests.Session()
+        form_inputs = get_hidden_form_inputs(session, FORM_URL)
+        required = {
+            "__RequestVerificationToken",
+            "FormGuid",
+            "ObjectTemplateID",
+            "CurrentSectionID",
+        }
+        if not required.issubset(form_inputs.keys()):
+            raise ValueError("Unable to read South Kesteven form metadata")
+
+        payload = {
+            "__RequestVerificationToken": form_inputs["__RequestVerificationToken"],
+            "FormGuid": form_inputs["FormGuid"],
+            "ObjectTemplateID": form_inputs["ObjectTemplateID"],
+            "Trigger": "submit",
+            "CurrentSectionID": form_inputs["CurrentSectionID"],
+            "FF5265": self._address_id,
+            "FF5265lbltxt": "Collection Address",
+            "FF5265searchnlpg": "False",
+            "FF5265manualaddressentry": "False",
+            "FF5265classification": "",
+        }
+        data_resp = session.post(RENDER_URL, data=payload, timeout=30)
+        data_resp.raise_for_status()
+        data_soup = bs4.BeautifulSoup(data_resp.text, "html.parser")
+
+        collections: list[Collection] = []
+        for row in data_soup.select("table.Alloy-table tr"):
+            cols = row.find_all("td", class_="Alloy-table-col")
+            if len(cols) < 2:
+                continue
+            raw_date = cols[0].get_text(strip=True)
+            raw_type = cols[1].get_text(strip=True)
+            try:
+                dt = datetime.strptime(raw_date, "%A %d %B, %Y").date()
+            except ValueError:
+                continue
+            icon = ICON_MAP.get(raw_type.lower())
+            if icon is None:
+                lower = raw_type.lower()
+                if "refuse" in lower:
+                    icon = ICON_MAP["black"]
+                elif "recycling" in lower or "paper" in lower:
+                    icon = ICON_MAP["gray"]
+                elif "green" in lower:
+                    icon = ICON_MAP["green"]
+                elif "purple" in lower:
+                    icon = ICON_MAP["purple"]
+            collections.append(Collection(dt, raw_type, icon))
+
+        # If the new flow does not yield rows, try postcode lookup then resubmit.
+        if not collections and not self._address_id.startswith("U"):
+            addresses = lookup_addresses(
+                session, ADDRESS_LOOKUP_URL, self._address_id, search_nlpg="False"
+            )
+            if addresses:
+                self._address_id = next(iter(addresses.keys()))
+                return self.fetch()
+        if not collections:
+            _LOGGER.warning(
+                "South Kesteven returned no collection rows for address_id=%s",
+                self._address_id,
+            )
+
+        # filter out duplicate entries
+        collections = list(
+            {
+                (collection.date, collection.type): collection
+                for collection in collections
+            }.values()
+        )
+        return collections

package/python/vendor/waste_collection_schedule/source/southlanarkshire_gov_uk.py

@@ -0,0 +1,185 @@
+import json
+from datetime import datetime
+
+from bs4 import BeautifulSoup
+from curl_cffi import requests
+from waste_collection_schedule import Collection, Icons  # type: ignore[attr-defined]
+from waste_collection_schedule.exceptions import (
+    SourceArgumentNotFound,
+    SourceArgumentNotFoundWithSuggestions,
+)
+
+TITLE = "South Lanarkshire Council"
+DESCRIPTION = "Source for South Lanarkshire Council waste collection."
+URL = "https://wasteservices.southlanarkshire.gov.uk"
+COUNTRY = "uk"
+
+DASHBOARD_URL = "https://wasteservices.southlanarkshire.gov.uk/PublicDashboard"
+REQUEST_TIMEOUT = 30
+
+ICON_MAP = {
+    "food": Icons.BIO_KITCHEN,
+    "blue": Icons.RECYCLING,
+    "grey": Icons.GLASS_COLORED,
+    "burgundy": Icons.BIO_KITCHEN,
+    "black": Icons.GENERAL_WASTE,
+}
+
+
+def _icon_for(waste_type: str) -> str | None:
+    lower = waste_type.lower()
+    for keyword, icon in ICON_MAP.items():
+        if keyword in lower:
+            return icon
+    return None
+
+
+HOW_TO_GET_ARGUMENTS_DESCRIPTION = {
+    "en": (
+        "Find your UPRN using FindMyAddress (https://www.findmyaddress.co.uk/search) "
+        "or UPRN.uk (https://uprn.uk) - search for your address "
+        "and note the UPRN shown. Alternatively, visit "
+        "https://wasteservices.southlanarkshire.gov.uk/PublicDashboard, enter your "
+        "postcode, select your property, then inspect the dropdown option "
+        "(right-click > Inspect Element) and note the numeric value= attribute."
+    ),
+}
+
+PARAM_TRANSLATIONS = {
+    "en": {
+        "postcode": "Postcode",
+        "uprn": "UPRN",
+    }
+}
+
+PARAM_DESCRIPTIONS = {
+    "en": {
+        "postcode": "Your full UK postcode (e.g. G73 1UR).",
+        "uprn": "Unique Property Reference Number for your address (e.g. 484000600).",
+    }
+}
+
+TEST_CASES = {
+    "1 Clincarthill Road, Glasgow, G73 2LF": {"postcode": "G73 2LF", "uprn": 484129473},
+    "55 Chapel Court, Glasgow, G73 1UR": {"postcode": "G73 1UR", "uprn": 484000600},
+    "Flat 1 10, Burnside Lane, Hamilton, ML3 6QP": {
+        "postcode": "ML3 6QP",
+        "uprn": 484073020,
+    },
+    "2 Braxfield Road, Lanark, ML11 9AB": {"postcode": "ML11 9AB", "uprn": 484118513},
+}
+
+
+class Source:
+    def __init__(self, postcode: str, uprn: int):
+        self._postcode = postcode.strip().upper()
+        self._uprn = int(uprn)
+
+    def fetch(self) -> list[Collection]:
+        session = requests.Session(impersonate="chrome")
+
+        # Step 1: GET the dashboard to obtain the CSRF token
+        resp = session.get(DASHBOARD_URL, timeout=REQUEST_TIMEOUT)
+        resp.raise_for_status()
+
+        soup = BeautifulSoup(resp.text, "html.parser")
+        token_input = soup.find("input", {"name": "__RequestVerificationToken"})
+        if not token_input:
+            raise SourceArgumentNotFound(
+                "postcode",
+                self._postcode,
+                "could not load the South Lanarkshire dashboard form; the council site may have changed.",
+            )
+        token = token_input["value"]
+        # Step 2: POST to SelectPrem to retrieve the collection schedule
+        resp = session.post(
+            DASHBOARD_URL,
+            params={"handler": "SelectPrem"},
+            data={
+                "SelectedPostcode": self._postcode,
+                "SelectedPremises": str(self._uprn),
+                "__RequestVerificationToken": token,
+            },
+            timeout=REQUEST_TIMEOUT,
+        )
+        resp.raise_for_status()
+        premises = self._extract_premises(resp.text)
+
+        # Step 3: Extract the appointments JSON embedded in the response HTML
+        appointments = self._extract_appointments(resp.text)
+        if not appointments:
+            suggestions = [
+                f"{premise['Premises']} (UPRN {premise['UPRN']})"
+                for premise in premises
+                if isinstance(premise, dict)
+                and premise.get("Premises")
+                and premise.get("UPRN") is not None
+            ]
+            if suggestions:
+                raise SourceArgumentNotFoundWithSuggestions(
+                    "uprn", self._uprn, suggestions
+                )
+            raise SourceArgumentNotFound("uprn", self._uprn)
+
+        seen: set[tuple] = set()
+        entries: list[Collection] = []
+        for appt in appointments:
+            try:
+                date = datetime.fromisoformat(appt["StartTime"]).date()
+                subject = appt["Subject"]
+            except (KeyError, ValueError):
+                continue
+            key = (date, subject)
+            if key not in seen:
+                seen.add(key)
+                entries.append(
+                    Collection(date=date, t=subject, icon=_icon_for(subject))
+                )
+
+        entries.sort(key=lambda entry: (entry.date, entry.type))
+        return entries
+
+    @staticmethod
+    def _extract_appointments(html: str) -> list:
+        """Extract the appointments dataSource JSON embedded in the page HTML.
+
+        The page contains two dataSource blocks: one for the premises dropdown
+        and one for the scheduler appointments. The appointments block is
+        identified by containing a "Subject" key in each entry.
+        """
+        for data in Source._extract_data_sources(html):
+            if (
+                data
+                and isinstance(data[0], dict)
+                and "Subject" in data[0]
+                and "StartTime" in data[0]
+            ):
+                return data
+        return []
+
+    @staticmethod
+    def _extract_premises(html: str) -> list:
+        for data in Source._extract_data_sources(html):
+            if data and isinstance(data[0], dict) and "Premises" in data[0]:
+                return data
+        return []
+
+    @staticmethod
+    def _extract_data_sources(html: str) -> list:
+        marker = '"dataSource": ejs.data.DataUtil.parse.isJson('
+        search_from = 0
+        data_sources = []
+        while True:
+            idx = html.find(marker, search_from)
+            if idx == -1:
+                break
+            start = idx + len(marker)
+            try:
+                decoder = json.JSONDecoder()
+                data, _ = decoder.raw_decode(html, idx=start)
+                if isinstance(data, list):
+                    data_sources.append(data)
+            except ValueError:
+                pass
+            search_from = idx + 1
+        return data_sources

package/python/vendor/waste_collection_schedule/source/southperth_wa_gov_au.py

@@ -0,0 +1,120 @@
+import logging
+import re
+from datetime import datetime, timedelta
+
+from waste_collection_schedule import Collection, Icons  # type: ignore[attr-defined]
+from waste_collection_schedule.exceptions import SourceArgumentNotFound
+from waste_collection_schedule.service.IntraMaps import (
+    IntraMapsSearchError,
+    MapsClient,
+    MapsClientConfig,
+    extract_panel_fields,
+)
+
+TITLE = "City of South Perth"
+DESCRIPTION = "Source for City of South Perth waste collection."
+URL = "https://southperth.wa.gov.au"
+COUNTRY = "au"
+TEST_CASES = {
+    "Lansdowne Road": {"address": "156 Lansdowne Road KENSINGTON"},
+    "Roebuck Drive": {"address": "13 Roebuck Drive"},
+}
+ICON_MAP = {
+    "General Waste": Icons.GENERAL_WASTE,
+    "Recycling": Icons.RECYCLING,
+}
+PARAM_DESCRIPTIONS = {
+    "en": {
+        "address": "Street address with suburb (e.g. '156 Lansdowne Road KENSINGTON')",
+    },
+}
+PARAM_TRANSLATIONS = {
+    "en": {
+        "address": "Street Address",
+    },
+}
+INTRAMAPS_CONFIG = MapsClientConfig(
+    base_url="https://cosp.spatial.t1cloud.com",
+    instance="spatial/intramaps",
+    config_id="29b80b8c-2c27-4a14-8f10-678c7947f7be",
+    project="cf285c38-0d67-406e-93c2-35ae3d066521",
+)
+# Map IntraMaps field names to waste types
+FIELD_MAP = {
+    "Waste Pickup Day": "General Waste",
+    "Next Recycling Pickup": "Recycling",
+}
+WEEKDAYS = {
+    "monday": 0,
+    "tuesday": 1,
+    "wednesday": 2,
+    "thursday": 3,
+    "friday": 4,
+    "saturday": 5,
+    "sunday": 6,
+}
+LOGGER = logging.getLogger(__name__)
+
+
+class Source:
+    def __init__(self, address: str):
+        self._address = address.strip()
+
+    def fetch(self) -> list[Collection]:
+        try:
+            with MapsClient(INTRAMAPS_CONFIG) as client:
+                result = client.select_address(self._address)
+        except IntraMapsSearchError as e:
+            raise SourceArgumentNotFound("address", self._address) from e
+
+        response = result.get("response")
+        if not isinstance(response, dict):
+            raise SourceArgumentNotFound("address", self._address)
+
+        fields = extract_panel_fields(response)
+        entries: list[Collection] = []
+        for field_name, waste_type in FIELD_MAP.items():
+            value = fields.get(field_name, "")
+            if value:
+                if waste_type == "General Waste":
+                    entries.extend(self._parse_schedule(value, waste_type))
+                else:
+                    try:
+                        recycle_date = datetime.strptime(value, "%d %B %Y").date()
+                    except ValueError:
+                        LOGGER.warning(
+                            f"'{recycle_date}' did not match expected date format."
+                        )
+                    entries.append(
+                        Collection(
+                            date=recycle_date,
+                            t="Recycling",
+                            icon=ICON_MAP["Recycling"],
+                        )
+                    )
+
+        return entries
+
+    @staticmethod
+    def _parse_schedule(text: str, waste_type: str) -> list[Collection]:
+        """Parse schedule text like 'Every Friday'."""
+        icon = ICON_MAP.get(waste_type)
+        text_lower = text.strip().lower()
+        # "Every [Day]" — weekly
+        match = re.match(r"every\s+(\w+)", text_lower)
+        if match:
+            weekday = WEEKDAYS.get(match.group(1))
+            if weekday is not None:
+                today = datetime.now().date()
+                days_ahead = (weekday - today.weekday()) % 7
+                next_date = today + timedelta(days=days_ahead)
+                return [
+                    Collection(
+                        date=next_date + timedelta(weeks=i),
+                        t=waste_type,
+                        icon=icon,
+                    )
+                    for i in range(26)
+                ]
+
+        return []