@camstack/addon-waste-collections 0.0.0-stage → 0.1.2

package/python/vendor/waste_collection_schedule/service/A_region_ch.py

@@ -0,0 +1,230 @@
+from typing import Literal
+
+import requests
+from bs4 import BeautifulSoup
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
+from waste_collection_schedule.exceptions import (
+    SourceArgumentNotFoundWithSuggestions,
+    SourceArgumentRequiredWithSuggestions,
+)
+from waste_collection_schedule.source.ics import Source as ICS
+
+
+def _session() -> requests.Session:
+    s = requests.Session()
+    retry = Retry(
+        total=3,
+        backoff_factor=2,
+        status_forcelist=[429, 500, 502, 503, 504],
+        respect_retry_after_header=True,
+    )
+    s.mount("https://", HTTPAdapter(max_retries=retry))
+    s.mount("http://", HTTPAdapter(max_retries=retry))
+    return s
+
+
+SERVICES = {
+    "winterthur": "https://m.winterthur.ch",
+    "a_region": "https://www.a-region.ch",
+    "koeniz": "https://koeniz.citymobile.ch",
+    "zab": "https://zab.citymobile.ch",
+}
+SERVICES_LITERALS = Literal["winterthur", "a_region", "koeniz", "zab"]
+
+
+class A_region_ch:
+    def __init__(
+        self,
+        service: SERVICES_LITERALS,
+        region_url: str,
+        district: str | None = None,
+        regex: str | None = None,
+        session: requests.Session | None = None,
+    ):
+        if service not in SERVICES:
+            raise Exception(f"service '{service}' not found")
+        self._base_url = SERVICES[service]
+
+        self._regex = regex
+
+        self._municipality_url = region_url
+        self._district = district
+        self._session = session or _session()
+
+    def fetch(self) -> list[ICS]:
+        waste_types = self.get_waste_types(self._municipality_url)
+
+        entries = []
+
+        for tour, link in waste_types.items():
+            entries += self.get_ICS_sources(link, tour)
+        return entries
+
+    def get_municipalities(self) -> dict[str, str]:
+        municipalities: dict[str, str] = {}
+
+        # get PHPSESSID
+        session = _session()
+        r = session.get(f"{self._base_url}")
+        r.raise_for_status()
+
+        # cookies = {'PHPSESSID': requests.utils.dict_from_cookiejar(r.cookies)['PHPSESSID']}
+
+        params: dict[str, str | int] = {"apid": "13875680", "apparentid": "4618613"}
+        r = session.get(f"{self._base_url}/index.php", params=params)
+        r.raise_for_status()
+        self.extract_municipalities(r.text, municipalities)
+
+        page = 1
+        while True:
+            params = {
+                "do": "searchFetchMore",
+                "hash": "606ee79ca61fc6eef434ab4fca0d5956",
+                "p": page,
+            }
+            headers = {
+                "cookie": "PHPSESSID=71v67j0et4ih04qa142d402ebm;"
+            }  # TODO: get cookie from first request
+            r = session.get(
+                f"{self._base_url}/appl/ajax/index.php", params=params, headers=headers
+            )
+            r.raise_for_status()
+            if r.text == "":
+                break
+            self.extract_municipalities(r.text, municipalities)
+            page = page + 1
+        return municipalities
+
+    def extract_municipalities(self, text: str, municipalities: dict[str, str]):
+        soup = BeautifulSoup(text, features="html.parser")
+        downloads = soup.find_all("a", href=True)
+        for download in downloads:
+            # href ::= "/index.hp"
+            href = download.get("href")
+            if "ref=search" in href:
+                for title in download.find_all("div", class_="title"):
+                    # title ::= "Abfallkalender Andwil"
+                    municipalities[title.string.removeprefix("Abfallkalender ")] = href
+
+    def get_waste_types(self, link: str) -> dict[str, str]:
+        if not link.startswith("http"):
+            link = f"{self._base_url}{link}"
+        r = self._session.get(link)
+        r.raise_for_status()
+
+        waste_types = {}
+
+        soup = BeautifulSoup(r.text, features="html.parser")
+        downloads = soup.find_all("a", href=True)
+        for download in downloads:
+            # href ::= "/index.php?apid=12731252&apparentid=5011362"
+            href = download.get("href")
+            if "download.php" in href:
+                # skip PDF/attachment downloads (e.g. "Abfall-Info" leaflets).
+                # These are not calendar pages, can be several MB in size, and
+                # previously slipped through because the old "PDF" check below
+                # compared bs4 Tag objects (from find_all) against a string,
+                # which never matched. Repeatedly fetching a several-MB PDF on
+                # every poll can also trip the provider's own anti-bot rate
+                # limiting.
+                continue
+            if (
+                download.find("div", class_="badgeIcon")
+                or download.find("img", class_="rowImg")
+                or download.find("img", class_="svgIconImg")
+            ):
+                titles = download.find_all("div", class_="title")
+                titles = [title.string for title in titles]
+                if not titles:
+                    titles = [download.get_text(strip=True)]
+                if any(title and "PDF" in title for title in titles):
+                    continue
+                for title in titles:
+                    if title is None:
+                        continue
+                    # title ::= "Altmetall"
+                    waste_types[title] = href
+
+        return waste_types
+
+    def get_ICS_sources(self, link: str, tour: str) -> list[ICS]:
+        if not link.startswith("http"):
+            link = f"{self._base_url}{link}"
+        r = self._session.get(link)
+        r.raise_for_status()
+
+        soup = BeautifulSoup(r.text, features="html.parser")
+
+        # check for additional districts
+        districts = {}
+        downloads = soup.find_all("a", href=True)
+        for download in downloads:
+            href = download.get("href")
+            if "apparentid" in href:
+                title = download.find("div", class_="title")
+                if title is not None:
+                    # additional district found ->
+                    district_name_split = title.string.split(": ")
+                    districts[
+                        district_name_split[1 if len(district_name_split) > 1 else 0]
+                    ] = href
+        if len(districts) > 0:
+            if len(districts) == 1:
+                # only one district found -> use it
+                return self.get_ICS_sources(next(iter(districts.values())), tour)
+            if self._district is None:
+                raise SourceArgumentRequiredWithSuggestions(
+                    "district",
+                    "Multiple districts found; specify which one to use.",
+                    districts.keys(),
+                )
+
+            if self._district not in districts:
+                raise SourceArgumentNotFoundWithSuggestions(
+                    "district", self._district, districts.keys()
+                )
+            return self.get_ICS_sources(districts[self._district], tour)
+
+        dates = []
+
+        downloads = soup.find_all("a", href=True)
+        for download in downloads:
+            # href ::= "/appl/ics.php?apid=12731252&from=2022-05-04%2013%3A00%3A00&to=2022-05-04%2013%3A00%3A00"
+            href = download.get("href")
+            if href.startswith("webcal") and "ical.php" in href:
+                dates.append(ICS(url=href, regex=self._regex))
+                break
+
+        return dates
+
+
+def get_region_url_by_street(
+    service: SERVICES_LITERALS,
+    street: str,
+    search_url: str,
+    district: str | None = None,
+    regex: str | None = None,
+) -> A_region_ch:
+    session = _session()
+    r = session.get(search_url, params={"q": street})
+    r.raise_for_status()
+
+    soup = BeautifulSoup(r.text, features="html.parser")
+    as_ = soup.select("a")
+    if len(as_) == 0:
+        raise Exception("No streets found")
+    streets = []
+    for a in as_:
+        href = a.get("href")
+        if not isinstance(href, str):
+            continue
+        streets.append(a.get_text(strip=True))
+
+        if a.get_text(strip=True).lower().replace(" ", "") == street.lower().replace(
+            " ", ""
+        ):
+            return A_region_ch(service, href, district, regex, session=session)
+
+    raise SourceArgumentNotFoundWithSuggestions("street", street, streets)

package/python/vendor/waste_collection_schedule/service/AbfallIO.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python3
+
+
+SERVICE_MAP = [
+    {
+        "title": "EGST Steinfurt",
+        "url": "https://www.egst.de/",
+        "service_id": "e21758b9c711463552fb9c70ac7d4273",
+    },
+    {
+        "title": "ALBA Berlin",
+        "url": "https://berlin.alba.info/",
+        "service_id": "9583a2fa1df97ed95363382c73b41b1b",
+    },
+    {
+        "title": "Landkreis Bayreuth",
+        "url": "https://www.landkreis-bayreuth.de/",
+        "service_id": "951da001077dc651a3bf437bc829964e",
+    },
+    {
+        "title": "Landkreis Calw",
+        "url": "https://www.kreis-calw.de/",
+        "service_id": "690a3ae4906c52b232c1322e2f88550c",
+    },
+    {
+        "title": "Abfallwirtschaft Landkreis Freudenstadt",
+        "url": "https://www.awb-fds.de/",
+        "service_id": "595f903540a36fe8610ec39aa3a06f6a",
+    },
+    {
+        "title": "Göttinger Entsorgungsbetriebe",
+        "url": "https://www.geb-goettingen.de/",
+        "service_id": "7dd0d724cbbd008f597d18fcb1f474cb",
+    },
+    {
+        "title": "Landkreis Heilbronn",
+        "url": "https://www.landkreis-heilbronn.de/",
+        "service_id": "1a1e7b200165683738adddc4bd0199a2",
+    },
+    {
+        "title": "Abfallwirtschaft Landkreis Kitzingen",
+        "url": "https://www.abfallwelt.de/",
+        "service_id": "594f805eb33677ad5bc645aeeeaf2623",
+    },
+    {
+        "title": "Abfallwirtschaft Landkreis Landsberg am Lech",
+        "url": "https://www.abfallberatung-landsberg.de/",
+        "service_id": "7df877d4f0e63decfb4d11686c54c5d6",
+    },
+    {
+        "title": "Stadt Landshut",
+        "url": "https://www.landshut.de/",
+        "service_id": "bd0c2d0177a0849a905cded5cb734a6f",
+    },
+    {
+        "title": "Ludwigshafen am Rhein",
+        "url": "https://www.ludwigshafen.de/",
+        "service_id": "6efba91e69a5b454ac0ae3497978fe1d",
+    },
+    {
+        "title": "MüllALARM / Schönmackers",
+        "url": "https://www.schoenmackers.de/",
+        "service_id": "e5543a3e190cb8d91c645660ad60965f",
+    },
+    {
+        "title": "Abfallbewirtschaftung Ostalbkreis",
+        "url": "https://www.goa-online.de/",
+        "service_id": "3ca331fb42d25e25f95014693ebcf855",
+    },
+    {
+        "title": "Landkreis Oldenburg",
+        "url": "https://www.oldenburg-kreis.de/",
+        "service_id": "27708a019a2e35de7eb4bbe7c851609f",
+    },
+    {
+        "title": "Landkreis Ostallgäu",
+        "url": "https://www.buerger-ostallgaeu.de/",
+        "service_id": "342cedd68ca114560ed4ca4b7c4e5ab6",
+    },
+    {
+        "title": "Rhein-Neckar-Kreis",
+        "url": "https://www.rhein-neckar-kreis.de/",
+        "service_id": "914fb9d000a9a05af4fd54cfba478860",
+    },
+    {
+        "title": "Landkreis Rotenburg (Wümme)",
+        "url": "https://lk-awr.de/",
+        "service_id": "645adb3c27370a61f7eabbb2039de4f1",
+    },
+    {
+        "title": "Landkreis Sigmaringen",
+        "url": "https://www.landkreis-sigmaringen.de/",
+        "service_id": "39886c5699d14e040063c0142cd0740b",
+    },
+    {
+        "title": "Landratsamt Traunstein",
+        "url": "https://www.traunstein.com/",
+        "service_id": "279cc5db4db838d1cfbf42f6f0176a90",
+    },
+    {
+        "title": "Landratsamt Unterallgäu",
+        "url": "https://www.landratsamt-unterallgaeu.de/",
+        "service_id": "c22b850ea4eff207a273e46847e417c5",
+    },
+    {
+        "title": "AWB Westerwaldkreis",
+        "url": "https://wab.rlp.de/",
+        "service_id": "248deacbb49b06e868d29cb53c8ef034",
+    },
+    {
+        "title": "Landkreis Limburg-Weilburg",
+        "url": "https://www.awb-lm.de/",
+        "service_id": "0ff491ffdf614d6f34870659c0c8d917",
+    },
+    {
+        "title": "Landkreis Weißenburg-Gunzenhausen",
+        "url": "https://www.landkreis-wug.de",
+        "service_id": "31fb9c7d783a030bf9e4e1994c7d2a91",
+    },
+    {
+        "title": "VIVO Landkreis Miesbach",
+        "url": "https://www.vivowarngau.de/",
+        "service_id": "4e33d4f09348fdcc924341bf2f27ec86",
+    },
+    {
+        "title": "Abfallzweckverband Rhein-Mosel-Eifel (Landkreis Mayen-Koblenz)",
+        "url": "https://www.azv-rme.de/",
+        "service_id": "8303df78b822c30ff2c2f98e405f86e6",
+    },
+    # Team Orange (Landkreis Würzburg) migrated off abfall.io; their service key
+    # now returns 403. Use the dedicated `team_orange_de` source instead. See #6041.
+    {
+        "title": "Landkreis Cuxhaven",
+        "url": "https://www.landkreis-cuxhaven.de/",
+        "service_id": "49fe8a63a056adbfc43f051f61dd4a44",
+    },
+    {
+        "title": "Landkreis Rottweil",
+        "url": "https://landkreis-rottweil.de",
+        "service_id": "d287412901d68d66825e588a60c94641",
+    },
+    {
+        "title": "AVR Kommunal, Rhein-Neckar-Kreis",
+        "url": "https://www.avr-kommunal.de/",
+        "service_id": "914fb9d000a9a05af4fd54cfba478860",
+    },
+    {
+        "title": "AWG Abfallwirtschaft Landkreis Calw",
+        "url": "https://www.awg-info.de/",
+        "service_id": "0813ea99f520c462373386564a99a51e",
+    },
+    {
+        "title": "Amt Bad Wilsnack/Weisen (Landkreis Prignitz)",
+        "url": "https://www.landkreis-prignitz.de/",
+        "service_id": "1e9592418582666e2a5d1c62b2683435",
+    },
+    {
+        "title": "Gemeinde Groß Pankow (Landkreis Prignitz)",
+        "url": "https://www.landkreis-prignitz.de/",
+        "service_id": "af91b65d2753a219309072837d8ea4e1",
+    },
+    {
+        "title": "Gemeinde Gumtow (Landkreis Prignitz)",
+        "url": "https://www.landkreis-prignitz.de/",
+        "service_id": "3cefa45ab357d231891bb497253c630f",
+    },
+    {
+        "title": "Gemeinde Karstädt (Landkreis Prignitz)",
+        "url": "https://www.landkreis-prignitz.de/",
+        "service_id": "798f59a75627f5d7686dab0c7226c877",
+    },
+    {
+        "title": "Amt Lenzen-Elbtalaue (Landkreis Prignitz)",
+        "url": "https://www.landkreis-prignitz.de/",
+        "service_id": "bb937857acd951dfc8de5be8b8a49f6d",
+    },
+    {
+        "title": "Amt Meyenburg (Landkreis Prignitz)",
+        "url": "https://www.landkreis-prignitz.de/",
+        "service_id": "4638881e7bebe6869e2e86de5f8aa09e",
+    },
+    {
+        "title": "Stadt Perleberg (Landkreis Prignitz)",
+        "url": "https://www.landkreis-prignitz.de/",
+        "service_id": "9fb3e2e5498e825250105ee272102a7b",
+    },
+    {
+        "title": "Gemeinde Plattenburg (Landkreis Prignitz)",
+        "url": "https://www.landkreis-prignitz.de/",
+        "service_id": "a0461612534502273c518e28d4f6f1e4",
+    },
+    {
+        "title": "Stadt Pritzwalk (Landkreis Prignitz)",
+        "url": "https://www.landkreis-prignitz.de/",
+        "service_id": "d92f59ef4066ae6d299478996d1d8430",
+    },
+    {
+        "title": "Amt Putlitz/Berge (Landkreis Prignitz)",
+        "url": "https://www.landkreis-prignitz.de/",
+        "service_id": "4f06df48f154246415e57ce12b26abe5",
+    },
+    {
+        "title": "Stadt Wittenberge (Landkreis Prignitz)",
+        "url": "https://www.landkreis-prignitz.de/",
+        "service_id": "b870ecfa6e1f882680758d374ba3fa2d",
+    },
+    {
+        "title": "Landkreis Landshut",
+        "url": "https://www.landkreis-landshut.de/",
+        "service_id": "04b7561b94f2cbaa171cd85bb6aa56de",
+    },
+]

package/python/vendor/waste_collection_schedule/service/AbfallIOGraphQL.py

@@ -0,0 +1,58 @@
+SERVICE_MAP = [
+    {
+        "title": "Landkreis Märkisch-Oderland",
+        "url": "https://www.maerkisch-oderland.de/",
+        "service_id": "efb75cbd1f08fae1d4e47ae72a85c655",
+    },
+    {
+        "title": "Holding Graz",
+        "url": "https://www.holding-graz.at/",
+        "service_id": "1c230a689579b6d3ddb9ceb5a56c6072",
+        "country": "at",
+    },
+    {
+        "title": "Landkreis Reutlingen",
+        "url": "https://www.kreis-reutlingen.de/",
+        "service_id": "15f69fab91c4cae50d9dbb5bcfd383f0",
+    },
+    {
+        "title": "Entsorgungsbetriebe Essen",
+        "url": "https://www.ebe-essen.de/",
+        "service_id": "51be67f3758f1fb57b420efe065c0663",
+    },
+    {
+        "title": "KELL Kommunalentsorgung Landkreis Leipzig GmbH",
+        "url": "https://kell-gmbh.de/",
+        "service_id": "0d7a92192ba3ae914c028ac37d73e222",
+    },
+    {
+        "title": "Wirtschaftsbetriebe Duisburg (WBD)",
+        "url": "https://www.wb-duisburg.de/",
+        "service_id": "80acad6c77fe9342ebafad29a8c58bf6",
+    },
+    {
+        "title": "Landkreis Göttingen",
+        "url": "https://www.landkreisgoettingen.de/",
+        "service_id": "4b5702d771c82b611c386ebbc7629026",
+    },
+    {
+        "title": "Abfallwirtschaft Landkreis Böblingen",
+        "url": "https://www.awb-bb.de/",
+        "service_id": "76bdaac8568082d77e7a90cb41129f9b",
+    },
+    {
+        "title": "ASG Nordsachsen",
+        "url": "https://www.asg-nordsachsen.de/",
+        "service_id": "2085afd95285e645e15ee9623d0c5172",
+    },
+    {
+        "title": "Amt für Abfallwirtschaft Schwarzwald-Baar-Kreis",
+        "url": "https://www.lrasbk.de/",
+        "service_id": "30628292bdd8b43db86a48f7e0d85f85",
+    },
+    {
+        "title": "ASO Abfall-Service Osterholz",
+        "url": "https://www.aso-ohz.de/",
+        "service_id": "8b016df0116d1d5094fa339bebea0c65",
+    },
+]