@camstack/addon-waste-collections 0.0.0-stage → 0.1.2

package/python/vendor/waste_collection_schedule/source/communitywastedisposal_com.py

@@ -0,0 +1,301 @@
+import json
+import logging
+import re
+from datetime import date, datetime, timedelta
+
+import requests
+from dateutil.rrule import FR, MO, SA, SU, TH, TU, WE, WEEKLY, rrule
+from waste_collection_schedule import Collection, Icons  # type: ignore[attr-defined]
+from waste_collection_schedule.exceptions import SourceArgumentNotFound
+from waste_collection_schedule.service.ArcGis import (
+    ArcGisError,
+    query_feature_layer,
+)
+
+_LOGGER = logging.getLogger(__name__)
+
+TITLE = "Community Waste Disposal (CWD)"
+DESCRIPTION = "Source for Community Waste Disposal (CWD) in North Texas"
+URL = "https://www.communitywastedisposal.com"
+COUNTRY = "us"
+
+TEST_CASES = {
+    "Forney TX": {"address": "100 Princeton Cir, Forney, TX 75126"},
+    "Allen TX": {"address": "123 Main St, Allen, TX 75002"},
+    "Keller TX": {"address": "2016 Willis Ln, Keller, TX, 76248, USA"},
+}
+
+PARAM_DESCRIPTIONS = {
+    "en": {
+        "address": "Street address including city and ZIP (e.g. '123 Main St, Allen, TX 75002')",
+    },
+}
+
+PARAM_TRANSLATIONS = {
+    "en": {
+        "address": "Street Address",
+    },
+}
+
+LAYER_CONFIG = {
+    0: {"type": "HHW"},
+    1: {"type": "Recycling"},
+    2: {"type": "Trash"},
+    3: {"type": "Bulk Waste"},
+    4: {"type": "Yard Waste"},
+    5: {"type": "Compost"},
+}
+
+ICON_MAP = {
+    "HHW": Icons.HAZARDOUS,
+    "Recycling": Icons.RECYCLING,
+    "Trash": Icons.GENERAL_WASTE,
+    "Bulk Waste": Icons.BULKY,
+    "Yard Waste": Icons.GARDEN,
+    "Compost": Icons.ORGANIC,
+}
+
+# Fields (name differs in case between layers) that flag a waste stream as
+# "call to schedule" rather than an automatic recurring pickup. When set to
+# "Yes" the ArcGIS route data still carries a PickupDay/Frequency/TimingWeek
+# combination, but it does not represent an actual recurring collection date
+# -- the resident has to call CWD to arrange it. See #5072.
+CALL_TO_SCHEDULE_FIELDS = ("CallToSche", "CalltoSche")
+
+WEEKDAY_MAP = {
+    "Monday": MO,
+    "Tuesday": TU,
+    "Wednesday": WE,
+    "Thursday": TH,
+    "Friday": FR,
+    "Saturday": SA,
+    "Sunday": SU,
+}
+
+STR_TO_WEEKDAY = {
+    "Monday": 0,
+    "Tuesday": 1,
+    "Wednesday": 2,
+    "Thursday": 3,
+    "Friday": 4,
+    "Saturday": 5,
+    "Sunday": 6,
+}
+
+FEATURE_BASE = "https://services3.arcgis.com/xeSJphIgrY4QfLVq/arcgis/rest/services/CWD_Routes_View/FeatureServer"
+GEOCODE_URL = "https://geocode.arcgis.com/arcgis/rest/services/World/GeocodeServer"
+
+
+def _week_of_month(d: date) -> int:
+    """Return the 1-based ordinal occurrence of d's weekday within its month."""
+    return ((d.day - 1) // 7) + 1
+
+
+class Source:
+    def __init__(self, address: str):
+        self._address = address.strip()
+
+    def _geocode(self) -> tuple[dict, str]:
+        r = requests.get(
+            f"{GEOCODE_URL}/findAddressCandidates",
+            params={
+                "SingleLine": self._address,
+                "outSR": json.dumps({"wkid": 4326}),
+                "maxLocations": 1,
+                "outFields": "City",
+                "f": "json",
+            },
+            timeout=20,
+        )
+        r.raise_for_status()
+        candidates = r.json().get("candidates", [])
+        if not candidates:
+            raise SourceArgumentNotFound("address", self._address)
+        location = candidates[0]["location"]
+        city = candidates[0].get("attributes", {}).get("City", "")
+        return location, city
+
+    def _get_holidays(self, community: str) -> dict[str, dict[str, int]]:
+        try:
+            base_url = "https://support.newedgeservices.com/cwd/"
+            r = requests.get(base_url, timeout=15)
+            r.raise_for_status()
+            match = re.search(
+                r'<script[^>]+src=["\'](/cwd/static/js/main\.[0-9a-f]+\.js)["\']',
+                r.text,
+            )
+            if not match:
+                return {}
+            js_url = "https://support.newedgeservices.com" + match.group(1)
+            r = requests.get(js_url, timeout=15)
+            r.raise_for_status()
+            js_content = r.text
+        except requests.RequestException:
+            return {}
+
+        escaped = re.escape(community)
+        cm = re.search(rf'["\']?{escaped}["\']?\s*:\s*\[', js_content, re.IGNORECASE)
+        if not cm:
+            return {}
+
+        start = cm.start()
+        end_m = re.search(r"\],", js_content[start:])
+        end = start + end_m.end() if end_m else min(len(js_content), start + 50000)
+        segment = js_content[start:end]
+
+        hm = re.search(
+            r"service\s*:\s*Rt\.HOLIDAYS\s*,\s*dates\s*:\s*\[([^\]]+)\]", segment
+        )
+        if not hm:
+            return {}
+
+        holidays: dict[str, dict[str, int]] = {}
+        for start_str, end_str, skip_flag in re.findall(
+            r'start\s*:\s*At\(\)\(["\']([^"\']+)["\'].*?end\s*:\s*At\(\)\(["\']([^"\']+)["\'].*?skip\s*:\s*!([01])',
+            hm.group(1),
+        ):
+            delay = 1 if skip_flag == "1" else 0
+            if not delay:
+                continue
+            try:
+                parts = start_str.split("-")
+                start_date = date(int(parts[0]), int(parts[1]), int(parts[2]))
+            except (ValueError, TypeError, IndexError):
+                continue
+            end_date = start_date
+            if end_str:
+                try:
+                    parts = end_str.split("-")
+                    parsed = date(int(parts[0]), int(parts[1]), int(parts[2]))
+                    if parsed >= start_date:
+                        end_date = parsed
+                except (ValueError, TypeError, IndexError):
+                    pass
+            current = start_date
+            while current <= end_date:
+                holidays[current.strftime("%Y-%m-%d")] = {"delay": delay}
+                current += timedelta(days=1)
+
+        return holidays
+
+    def _generate_dates(
+        self,
+        day_name: str,
+        frequency: str,
+        timing_week: str,
+        start_date: date,
+        end_date: date,
+        holidays: dict[str, dict[str, int]],
+    ) -> list[date]:
+        weekday = WEEKDAY_MAP.get(day_name)
+        if not weekday:
+            return []
+
+        rule = rrule(
+            WEEKLY,
+            byweekday=weekday,
+            dtstart=datetime.combine(start_date, datetime.min.time()),
+            until=datetime.combine(end_date, datetime.min.time()),
+        )
+        all_dates = [d.date() for d in rule]
+
+        # TimingWeek can contain one or more ordinals, e.g. "1st", "3rd",
+        # "1st and 3rd" or "2nd and 4th". Whenever it gives us an explicit
+        # ordinal, honor it precisely (regardless of the -- sometimes
+        # inconsistent -- Frequency label) instead of only special-casing
+        # "1st" and silently falling back to "every single week" for
+        # anything else, which previously produced e.g. a collection every
+        # week for streams that only occur on the 2nd/4th week of the month.
+        ordinals = [int(n) for n in re.findall(r"\d+", timing_week)]
+
+        if ordinals:
+            dates = [d for d in all_dates if _week_of_month(d) in ordinals]
+        elif "biweekly" in frequency:
+            dates = [d for d in all_dates if d.isocalendar()[1] % 2 == 1]
+        elif not frequency or frequency in ("na", "weekly"):
+            # Blank/"NA"/"Weekly" -- either explicitly weekly, or no
+            # cadence info at all (in which case a weekly cadence on the
+            # given PickupDay is the best available assumption, matching
+            # prior behavior).
+            dates = all_dates
+        else:
+            # Frequency is "Monthly"/"BiMonthly"/"Calendar" (or something
+            # else unrecognised) but TimingWeek doesn't tell us which
+            # week(s) of the month apply. There is no way to derive that
+            # from the public route data alone, so skip rather than guess
+            # and show a wrong (e.g. weekly) schedule. See #5072.
+            dates = []
+
+        return [
+            (
+                d + timedelta(days=holidays[d.strftime("%Y-%m-%d")]["delay"])
+                if d.strftime("%Y-%m-%d") in holidays
+                else d
+            )
+            for d in dates
+        ]
+
+    def fetch(self) -> list[Collection]:
+        try:
+            location, community = self._geocode()
+        except SourceArgumentNotFound:
+            raise
+        except Exception as e:
+            raise SourceArgumentNotFound("address", self._address) from e
+
+        holidays = self._get_holidays(community) if community else {}
+
+        today = date.today()
+        end_date = today + timedelta(days=365)
+        entries: list[Collection] = []
+
+        for layer_id, config in LAYER_CONFIG.items():
+            layer_url = f"{FEATURE_BASE}/{layer_id}"
+            try:
+                features = query_feature_layer(layer_url, geometry=location)
+            except ArcGisError:
+                continue
+
+            attrs = features[0]
+
+            # Some streams (typically HHW, and occasionally Bulk/Yard Waste)
+            # are "call to schedule" rather than an automatic recurring
+            # pickup: the route data still carries a PickupDay/Frequency, but
+            # residents must call CWD to arrange collection. Don't surface
+            # those as if they were a fixed recurring date. See #5072.
+            if any(
+                isinstance(attrs.get(field), str)
+                and attrs[field].strip().lower() == "yes"
+                for field in CALL_TO_SCHEDULE_FIELDS
+            ):
+                continue
+
+            pickup_days = [
+                d.strip()
+                for key in ("PickupDay1", "PickupDay2")
+                if (d := attrs.get(key))
+                and isinstance(d, str)
+                and d.strip() in STR_TO_WEEKDAY
+            ]
+            if not pickup_days:
+                continue
+
+            frequency = str(attrs.get("Frequency", "Weekly")).lower()
+            timing_week = str(attrs.get("TimingWeek", "")).lower()
+
+            for day_name in pickup_days:
+                for collection_date in self._generate_dates(
+                    day_name, frequency, timing_week, today, end_date, holidays
+                ):
+                    entries.append(
+                        Collection(
+                            date=collection_date,
+                            t=config["type"],
+                            icon=ICON_MAP.get(config["type"]),
+                        )
+                    )
+
+        if not entries:
+            raise SourceArgumentNotFound("address", self._address)
+
+        return entries

package/python/vendor/waste_collection_schedule/source/comune_digital_it.py

@@ -0,0 +1,166 @@
+from datetime import date, timedelta
+
+import requests
+from waste_collection_schedule import Collection, Icons  # type: ignore[attr-defined]
+from waste_collection_schedule.exceptions import SourceArgumentNotFound
+
+TITLE = "Comune.Digital"
+DESCRIPTION = "Source for Italian municipalities using the Comune.Digital (GoodBarber) app platform."
+URL = "https://comune.digital"
+COUNTRY = "it"
+
+TEST_CASES = {
+    "Santo Stefano di Camastra": {
+        "city": "santostefanodicamastra",
+    },
+}
+
+ICON_MAP = {
+    "umido": Icons.BIO_KITCHEN,
+    "organico": Icons.ORGANIC,
+    "secco": Icons.GENERAL_WASTE,
+    "carta": Icons.PAPER,
+    "cartone": Icons.PAPER,
+    "plastica": Icons.PLASTIC_PACKAGING,
+    "vetro": Icons.GLASS,
+    "ingombranti": Icons.BULKY,
+}
+
+HOW_TO_GET_ARGUMENTS_DESCRIPTION = {
+    "en": "The city identifier is the subdomain of your municipality's Comune.Digital website. "
+    "For example, if the website is https://santostefanodicamastra.comune.digital, "
+    'the city identifier is "santostefanodicamastra".',
+    "it": "L'identificatore della città è il sottodominio del sito web Comune.Digital del tuo comune. "
+    "Ad esempio, se il sito è https://santostefanodicamastra.comune.digital, "
+    'l\'identificatore è "santostefanodicamastra".',
+}
+
+PARAM_TRANSLATIONS = {
+    "en": {
+        "city": "City identifier",
+    },
+}
+
+PARAM_DESCRIPTIONS = {
+    "en": {
+        "city": "Subdomain of the Comune.Digital website (e.g. santostefanodicamastra)",
+    },
+}
+
+SETTINGS_URL = "https://{city}.comune.digital/apiv4/getSettings?platform=webapp"
+ITEMS_URL = "https://api.ww-api.com/front/get_items/{webzine_id}/{section_id}/"
+SECTION_TYPE = "GBModuleTypeAgenda"
+WASTE_KEYWORDS = ("differenziata", "rifiut", "raccolt", "spazzatura")
+
+
+def _get_icon(title: str) -> str | None:
+    title_lower = title.lower()
+    for keyword, icon in ICON_MAP.items():
+        if keyword in title_lower:
+            return icon
+    return None
+
+
+class Source:
+    def __init__(self, city: str) -> None:
+        self._city = city.strip().lower()
+
+    def fetch(self) -> list[Collection]:
+        # Step 1: Discover webzine_id and waste section_id
+        settings_url = SETTINGS_URL.format(city=self._city)
+        resp = requests.get(settings_url, timeout=30)
+        if resp.status_code == 404:
+            raise SourceArgumentNotFound(
+                "city",
+                self._city,
+                "City not found on comune.digital. Check the subdomain is correct.",
+            )
+        resp.raise_for_status()
+
+        data = resp.json()
+        webzine_id = data.get("idWebzine")
+        if not webzine_id:
+            raise SourceArgumentNotFound(
+                "city",
+                self._city,
+                "Could not retrieve webzine ID from comune.digital.",
+            )
+
+        gs = data.get("gbsettings", {})
+        sections = gs.get("sections", {})
+
+        section_id = self._find_waste_section(sections)
+        if not section_id:
+            raise SourceArgumentNotFound(
+                "city",
+                self._city,
+                "No waste-collection schedule section found for this city on comune.digital.",
+            )
+
+        # Step 2: Fetch all waste-collection events (paginate while items are returned)
+        entries: list[Collection] = []
+        page = 1
+        per_page = 100
+
+        while True:
+            items_url = ITEMS_URL.format(webzine_id=webzine_id, section_id=section_id)
+            resp = requests.get(
+                items_url,
+                params={"page": page, "per_page": per_page},
+                timeout=30,
+            )
+            resp.raise_for_status()
+            payload = resp.json()
+            items = payload.get("items", [])
+            if not items:
+                break
+
+            for item in items:
+                raw_date_str = item.get("sortDate") or item.get("date", "")
+                if not raw_date_str:
+                    continue
+                title: str = item.get("title", "").strip()
+                if not title:
+                    continue
+
+                # Parse ISO datetime and extract the date portion.
+                # Events are "Stasera esporre …" (Tonight put out …) at 20:00 —
+                # the bin is actually collected the following morning, so we
+                # advance the date by one day.
+                try:
+                    event_date = date.fromisoformat(raw_date_str[:10])
+                except ValueError:
+                    continue
+
+                collection_date = event_date + timedelta(days=1)
+
+                # Strip the "Stasera esporre " prefix when present to get the
+                # clean waste-type string shown to the user.
+                waste_type = title
+                for prefix in ("Stasera esporre ", "stasera esporre "):
+                    if waste_type.startswith(prefix):
+                        waste_type = waste_type[len(prefix) :].capitalize()
+                        break
+
+                icon = _get_icon(waste_type)
+                entries.append(
+                    Collection(date=collection_date, t=waste_type, icon=icon)
+                )
+
+            page += 1
+
+        return entries
+
+    @staticmethod
+    def _find_waste_section(sections: dict) -> str | None:
+        """Return the section_id of the first Agenda section whose title
+        contains a waste-collection keyword."""
+        for section_id, section_data in sections.items():
+            if not isinstance(section_data, dict):
+                continue
+            if section_data.get("type") != SECTION_TYPE:
+                continue
+            title: str = section_data.get("title", "").lower()
+            if any(kw in title for kw in WASTE_KEYWORDS):
+                return section_id
+        return None

package/python/vendor/waste_collection_schedule/source/conwy_gov_uk.py

@@ -0,0 +1,57 @@
+from datetime import datetime
+
+import requests
+from bs4 import BeautifulSoup
+from waste_collection_schedule import Collection, Icons  # type: ignore[attr-defined]
+from waste_collection_schedule.exceptions import SourceArgumentNotFound
+
+TITLE = "Conwy County Borough Council"
+DESCRIPTION = "Source for Conwy County Borough Council."
+URL = "https://www.conwy.gov.uk/"
+TEST_CASES = {
+    "50000009637": {"uprn": 50000009637},
+    "100101037037": {"uprn": "100101037037"},
+    "50000007574": {"uprn": 50000007574},
+}
+
+
+ICON_MAP = {
+    "garden": Icons.GARDEN,
+    "electrical": Icons.ELECTRONICS,
+    "refuse": Icons.GENERAL_WASTE,
+    "recycle": Icons.RECYCLING,
+}
+
+
+API_URL = (
+    "https://www.conwy.gov.uk/Contensis-Forms/erf/collection-result-soap-xmas2025.asp"
+)
+
+
+class Source:
+    def __init__(self, uprn: str | int):
+        self._uprn: str | int = uprn
+
+    def fetch(self):
+        r = requests.get(API_URL, params={"uprn": self._uprn, "ilangid": 1})
+        r.raise_for_status()
+
+        entries = []
+
+        soup = BeautifulSoup(r.text, "html.parser")
+        collection_dates = soup.select(".containererf")
+
+        if not collection_dates:
+            raise SourceArgumentNotFound("uprn", self._uprn)
+
+        for collection in collection_dates:
+            date_str = collection.select_one("#main #content").text.strip()
+            bin_types = [el.text for el in collection.select("#main1 li")]
+
+            date = datetime.strptime(date_str, "%A, %d/%m/%Y").date()
+
+            for bin_type in bin_types:
+                icon = ICON_MAP.get(bin_type.split(" ")[0].lower())  # Collection icon
+                entries.append(Collection(date=date, t=bin_type, icon=icon))
+
+        return entries

package/python/vendor/waste_collection_schedule/source/cornwall_gov_uk.py

@@ -0,0 +1,86 @@
+from datetime import date, datetime
+
+import requests
+from bs4 import BeautifulSoup
+from waste_collection_schedule import Collection, Icons  # type: ignore[attr-defined]
+from waste_collection_schedule.exceptions import (
+    SourceArgumentNotFound,
+    SourceArgumentNotFoundWithSuggestions,
+)
+
+TITLE = "Cornwall Council"
+DESCRIPTION = "Source for cornwall.gov.uk services for Cornwall Council"
+URL = "https://cornwall.gov.uk"
+TEST_CASES = {
+    "known_uprn": {"uprn": "100040118005"},
+    "unknown_uprn": {"postcode": "TR261SP", "housenumberorname": "7"},
+    "unknown_uprn_int": {"postcode": "PL17 8PL", "housenumberorname": 3},
+    "uprn_with_garden_int_uprn": {"uprn": 100040080721},
+}
+
+SEARCH_URLS = {
+    "uprn_search": "https://www.cornwall.gov.uk/my-area/",
+    "collection_search": "https://www.cornwall.gov.uk/umbraco/Surface/Waste/MyCollectionDays?subscribe=False",
+}
+ICON_MAP = {
+    "Rubbish": Icons.GENERAL_WASTE,
+    "Recycling": Icons.RECYCLING,
+    "Garden": Icons.GARDEN,
+}
+
+
+class Source:
+    def __init__(
+        self, uprn=None, postcode=None, housenumberorname=None
+    ):  # argX correspond to the args dict in the source configuration
+        self._uprn = uprn
+        self._postcode = postcode
+        self._housenumberorname = str(housenumberorname) if housenumberorname else None
+
+    def fetch(self):
+        entries = []
+        session = requests.Session()
+
+        # Find the UPRN based on the postcode and the property name/number
+        if self._uprn is None:
+            args = {"Postcode": self._postcode}
+            r = session.get(SEARCH_URLS["uprn_search"], params=args)
+            r.raise_for_status()
+            soup = BeautifulSoup(r.text, features="html.parser")
+            propertyUprns = soup.find(id="Uprn").find_all("option")
+            if len(propertyUprns) == 0:
+                raise SourceArgumentNotFound(
+                    "postcode",
+                    self._postcode,
+                )
+            for match in propertyUprns:
+                if match.text.startswith(self._housenumberorname):
+                    self._uprn = match["value"]
+            if self._uprn is None:
+                raise SourceArgumentNotFoundWithSuggestions(
+                    "housenumberorname",
+                    self._housenumberorname,
+                    [match.text for match in propertyUprns],
+                )
+
+        # Get the collection days based on the UPRN (either supplied through arguments or searched for above)
+        args = {"uprn": self._uprn}
+        r = session.get(SEARCH_URLS["collection_search"], params=args)
+        r.raise_for_status()
+        soup = BeautifulSoup(r.text, features="html.parser")
+        for collection_div in soup.find_all("div", class_="collection"):
+            spans = collection_div.find_all("span")
+            if not spans:
+                continue
+            collection = spans[0].text
+            d = spans[-1].text + " " + str(date.today().year)
+
+            entries.append(
+                Collection(
+                    datetime.strptime(d, "%d %b %Y").date(),
+                    collection,
+                    icon=ICON_MAP.get(collection),
+                )
+            )
+
+        return entries