catui-agent 1.0.0-beta.0

package/dist/extensions/builtin/browser/agent-workspace/domain-skills/trustpilot/scraping.md

@@ -0,0 +1,375 @@
+# Trustpilot — Company Reviews Scraping
+
+Field-tested against trustpilot.com on 2026-04-18.
+`http_get` with a generic Mozilla/5.0 UA works — no JS challenge, no Cloudflare block.
+The Trustpilot Consumer API (`api.trustpilot.com`) returns 403 for all endpoints without an API key.
+
+---
+
+## Fastest Approach: `http_get` + `__NEXT_DATA__`
+
+Trustpilot is a Next.js SSR app. Every company review page embeds the full data payload in a
+`<script id="__NEXT_DATA__">` JSON block — no browser needed. This includes the business unit
+metadata, all 20 reviews for the current page, pagination info, and rating distribution.
+
+```python
+import re, json
+from helpers import http_get
+
+def get_trustpilot_page(domain, page=1, stars=None, languages='en', verified=False):
+    """
+    Fetch one page of reviews for a company domain.
+    Returns (business_unit, reviews, pagination, rating_distribution).
+    Returns (None, [], {}, {}) if page is beyond the cap or no data.
+    """
+    url = f"https://www.trustpilot.com/review/{domain}?languages={languages}&page={page}"
+    if stars:
+        url += f"&stars={stars}"
+    if verified:
+        url += "&verified=true"
+
+    html = http_get(url)
+    m = re.search(
+        r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>',
+        html, re.DOTALL
+    )
+    if not m:
+        return None, [], {}, {}
+
+    data = json.loads(m.group(1))
+    pp = data['props']['pageProps']
+    bu = pp['businessUnit']
+    filters = pp.get('filters') or {}
+    pagination = filters.get('pagination', {})
+    ratings = filters.get('reviewStatistics', {}).get('ratings', {})
+    reviews = pp.get('reviews', [])
+
+    return bu, reviews, pagination, ratings
+```
+
+---
+
+## Business Unit (Company) Metadata
+
+```python
+bu, reviews, pagination, ratings = get_trustpilot_page("amazon.com")
+
+# Confirmed fields (tested 2026-04-18):
+bu['id']               # '46ad346800006400050092d0'  — stable MongoDB ObjectId
+bu['displayName']      # 'Amazon'
+bu['identifyingName']  # 'www.amazon.com'
+bu['trustScore']       # 1.7  (float, 1.0–5.0)
+bu['stars']            # 1.5  (display stars: 1, 1.5, 2, 2.5 … 5)
+bu['numberOfReviews']  # 45228  — total across all languages
+bu['websiteUrl']       # 'https://www.amazon.com'
+bu['isClaimed']        # True/False
+bu['isClosed']         # True/False
+bu['isCollectingReviews']  # True/False
+
+# Rating distribution (from filters.reviewStatistics.ratings):
+ratings  # {'total': 45228, 'one': 29718, 'two': 2701, 'three': 1759, 'four': 2367, 'five': 8683}
+
+# Pagination (filtered count, default is English only):
+pagination  # {'currentPage': 1, 'perPage': 20, 'totalCount': 28039, 'totalPages': 1402}
+```
+
+---
+
+## Review Fields
+
+Each review in the `reviews` list has these confirmed fields:
+
+```python
+review = {
+    'id':      '69e3103e09f46d6b5910f3c1',  # hex ObjectId, unique
+    'rating':  1,                             # int 1–5
+    'title':   'UNDELIVERABLE',
+    'text':    'UNDELIVERABLE\nThis is the only explanation...',
+    'language': 'en',
+    'likes':   0,                             # upvote count
+    'source':  'Organic',                     # 'Organic' or 'Invitation'
+    'filtered': False,
+    'isPending': False,
+
+    'dates': {
+        'experiencedDate': '2026-03-29T00:00:00.000Z',  # when they used the service
+        'publishedDate':   '2026-04-18T07:01:50.000Z',  # when review was posted
+        'updatedDate':     None,
+        'submittedDate':   None,
+    },
+
+    'consumer': {
+        'id':              '5cafe2feb158a8533b443467',
+        'displayName':     'Baldy Bloke',
+        'imageUrl':        'https://user-images.trustpilot.com/...',
+        'numberOfReviews': 17,
+        'countryCode':     'GB',
+        'hasImage':        True,
+        'isVerified':      False,
+    },
+
+    'labels': {
+        'verification': {
+            'isVerified':        False,
+            'verificationLevel': 'not-verified',   # or 'verified'
+            'reviewSourceName':  'Organic',
+            'verificationSource': 'invitation',
+            'createdDateTime':   '2026-04-18T07:01:50.000Z',
+            'hasDachExclusion':  False,
+        },
+        'merged': None,
+    },
+
+    'reply': None,           # or {'message': '...', 'publishedDate': '...', 'updatedDate': None}
+    'location': None,        # populated for multi-location businesses
+    'productReviews': [],    # non-empty for product-level reviews
+}
+```
+
+---
+
+## Paginating — Collect Up to 200 Reviews
+
+**Hard cap: pages 1–10 work; page 11+ returns an empty `reviews` array (no error, just empty).**
+This cap applies per filter combination, so `stars=1` gives 200 reviews, `stars=2` gives another
+200, etc.
+
+```python
+import re, json, time
+from helpers import http_get
+
+def collect_reviews(domain, stars=None, languages='en', max_pages=10, delay=0.5):
+    """
+    Collect up to max_pages*20 = 200 reviews. Returns list of review dicts.
+    stars: 1-5 to filter by rating (None = all)
+    languages: 'en' (default), 'all', or ISO code like 'de'
+    delay: seconds between requests (0.5 is safe; tested 5 rapid reqs with no block)
+    """
+    base = f"https://www.trustpilot.com/review/{domain}"
+    params = f"?languages={languages}"
+    if stars:
+        params += f"&stars={stars}"
+
+    all_reviews = []
+    seen_ids = set()
+
+    for page in range(1, max_pages + 1):
+        url = f"{base}{params}&page={page}"
+        html = http_get(url)
+        m = re.search(
+            r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>',
+            html, re.DOTALL
+        )
+        if not m:
+            break
+        data = json.loads(m.group(1))
+        reviews = data['props']['pageProps'].get('reviews', [])
+        if not reviews:
+            break   # hit the page 10 cap or truly no more reviews
+
+        new = [r for r in reviews if r['id'] not in seen_ids]
+        seen_ids.update(r['id'] for r in reviews)
+        all_reviews.extend(new)
+
+        if page < max_pages:
+            time.sleep(delay)
+
+    return all_reviews
+
+
+# Usage — 200 reviews per call:
+reviews = collect_reviews("shopify.com")               # English only, all ratings
+reviews_1star = collect_reviews("amazon.com", stars=1) # 200 x 1-star reviews
+reviews_all = collect_reviews("stripe.com", languages='all')  # all languages
+```
+
+### Maximize unique reviews by sweeping all star ratings
+
+Since each star filter gives an independent 200-review window, you can collect up to 1,000
+reviews per company (pages are deduplicated across filters):
+
+```python
+all_reviews = {}
+for stars in range(1, 6):
+    for r in collect_reviews("amazon.com", stars=stars, delay=0.5):
+        all_reviews[r['id']] = r
+
+print(f"Total unique reviews: {len(all_reviews)}")
+```
+
+---
+
+## Filters Reference
+
+All filter params are appended to the base URL `https://www.trustpilot.com/review/{domain}`:
+
+| Param | Values | Notes |
+|---|---|---|
+| `page` | 1–10 | Page 11+ returns empty `reviews` (tested). 20 reviews per page. |
+| `languages` | `en`, `all`, `de`, `fr`, `it`, `nl`, `sv`, `da`… | Default is `en`. Use `all` for all languages. |
+| `stars` | `1`, `2`, `3`, `4`, `5` | Filter to that star rating only. Works correctly. |
+| `verified` | `true` | Returns only invitation-verified reviews. Amazon has only ~21 verified reviews total. |
+| `date` | `last30days`, `last6months`, `last12months` | Reflected in `filters.selected.date` but data volume unchanged vs no filter — server-side filtering may be best-effort. |
+| `sort` | `recency`, `highest_rated`, `lowest_rated`, `helpful` | The `sort` param is accepted but **ignored server-side** via SSR — `filters.selected.sort` always returns `recency`. Sort only works in browser JS navigation. |
+
+---
+
+## Pagination Object
+
+```python
+# From filters.pagination (present on pages 1–10 when data exists):
+pagination = {
+    'currentPage': 1,
+    'perPage':     20,
+    'totalCount':  28039,   # filtered count (e.g. English only)
+    'totalPages':  1402,    # math: ceil(totalCount / 20)
+}
+
+# NOTE: totalPages can be 1402 but you can only access pages 1–10 (200 reviews).
+# On page 11+ the reviews list is empty and pagination is absent.
+```
+
+---
+
+## Rate Limits and Anti-bot
+
+- **No Cloudflare, no DataDome** — plain HTTP with `Mozilla/5.0` UA works immediately (tested
+  5 rapid requests in <5 seconds without any block).
+- **No CAPTCHA** observed during any test run.
+- **No 429 / rate-limit headers** seen on rapid sequential requests.
+- Safe rate: 0.5s between requests is conservative. Tested 5 consecutive requests at natural
+  speed (0.2–1s each) with no issue.
+- **robots.txt** has `User-agent: * / Disallow: /` (all paths blocked for unnamed bots) and
+  explicitly blocks `anthropic-ai`, `ClaudeBot`, `Claude-User`, `Claude-SearchBot`, `GPTBot`,
+  `anthropic-ai`, `CCBot`, etc. Despite this, `http_get` with `Mozilla/5.0` UA is not blocked
+  server-side (robots.txt is advisory only). Respect the policy if operating at scale.
+
+---
+
+## Consumer API (`api.trustpilot.com`)
+
+All Consumer API endpoints require an API key (OAuth2 client credentials). Without a key:
+
+```
+GET https://api.trustpilot.com/v1/business-units/find?name=amazon.com  → 403 Forbidden
+GET https://api.trustpilot.com/v1/business-units/{id}/reviews          → 403 Forbidden
+```
+
+The Business Unit ID embedded in `__NEXT_DATA__` (`businessUnit.id`) is the same ID used in the
+Consumer API, so if you have an API key, you can use it directly without a separate lookup.
+
+---
+
+## Gotchas
+
+1. **Page cap is 10, not `totalPages`**: `filters.pagination.totalPages` may show 1402, but
+   requests for pages 11+ return `reviews: []` silently. The server-rendered SSR cap is
+   hard-coded at page 10 (200 reviews).
+
+2. **`totalCount` in pagination is language-filtered**: With `languages=en`, `totalCount` is the
+   English-only count (e.g. 28,039 for Amazon). `businessUnit.numberOfReviews` is the true total
+   across all languages (45,228). Use `languages=all` to see the full count in pagination.
+
+3. **Sort param ignored in SSR**: `?sort=highest_rated` is reflected in `filters.selected.sort`
+   in the JSON but the reviews returned are always `recency`-sorted. Sort only takes effect
+   via browser-side JS navigation.
+
+4. **Verified filter is narrow**: Amazon has 45,228 reviews but only 21 are `isVerified=True`
+   (verificationLevel = 'verified'). Most reviews are organic/not-verified. Page 1 of
+   `verified=true` shows a misleading `totalCount=28039` — page 2 corrects to `totalCount=21`.
+
+5. **`date` filter behavior**: The `date` param is reflected in `filters.selected.date` but the
+   total review counts and returned reviews do not visibly change vs no filter in testing. The
+   server may apply it only partially or it may affect ordering rather than filtering.
+
+6. **`languages=en` is the default** and the server returns it even without the param. Use
+   `languages=all` explicitly to get reviews in all languages.
+
+7. **No `__NEXT_DATA__` fallback**: Never observed an empty or missing `__NEXT_DATA__` on valid
+   company pages. If absent, the domain may not have a Trustpilot profile — check for a
+   redirect or 404 in the HTML title.
+
+8. **Stars `1.5` vs `2`**: `businessUnit.stars` uses half-star display values (1.5, 2.0, etc).
+   `businessUnit.trustScore` is the precise float (1.7). Use `trustScore` for numeric comparison.
+
+---
+
+## Complete One-Shot Example
+
+```python
+import re, json, time
+from helpers import http_get
+
+def scrape_trustpilot(domain, max_unique=200):
+    """
+    Scrape up to max_unique reviews. Returns (company_info, reviews_list).
+    With max_unique=1000, sweeps all 5 star ratings to maximize coverage.
+    """
+    def _fetch_page(domain, page, stars=None, languages='en'):
+        url = f"https://www.trustpilot.com/review/{domain}?languages={languages}&page={page}"
+        if stars:
+            url += f"&stars={stars}"
+        html = http_get(url)
+        m = re.search(
+            r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>',
+            html, re.DOTALL
+        )
+        if not m:
+            return None, []
+        d = json.loads(m.group(1))
+        pp = d['props']['pageProps']
+        return pp['businessUnit'], pp.get('reviews', [])
+
+    company_info = None
+    all_reviews = {}
+
+    # First page to get company info
+    bu, reviews = _fetch_page(domain, 1)
+    company_info = {
+        'id':           bu['id'],
+        'name':         bu['displayName'],
+        'domain':       bu['identifyingName'],
+        'trust_score':  bu['trustScore'],
+        'stars':        bu['stars'],
+        'total_reviews': bu['numberOfReviews'],
+        'is_claimed':   bu['isClaimed'],
+    }
+    for r in reviews:
+        all_reviews[r['id']] = r
+
+    if max_unique <= 20:
+        return company_info, list(all_reviews.values())
+
+    # Pages 2–10 (no star filter)
+    for page in range(2, 11):
+        if len(all_reviews) >= max_unique:
+            break
+        _, reviews = _fetch_page(domain, page)
+        if not reviews:
+            break
+        for r in reviews:
+            all_reviews[r['id']] = r
+        time.sleep(0.5)
+
+    # If we want more, sweep by star rating
+    if len(all_reviews) < max_unique and max_unique > 200:
+        for stars in range(1, 6):
+            for page in range(1, 11):
+                if len(all_reviews) >= max_unique:
+                    break
+                _, reviews = _fetch_page(domain, page, stars=stars)
+                if not reviews:
+                    break
+                for r in reviews:
+                    all_reviews[r['id']] = r
+                time.sleep(0.5)
+
+    return company_info, list(all_reviews.values())[:max_unique]
+
+
+# Run it:
+company, reviews = scrape_trustpilot("shopify.com", max_unique=200)
+print(f"{company['name']} — TrustScore {company['trust_score']} — {company['total_reviews']} total reviews")
+print(f"Collected: {len(reviews)} reviews")
+print(f"Sample: [{reviews[0]['rating']}★] {reviews[0]['title'][:60]}")
+```