@m13v/seo-components 0.35.0 → 0.36.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/package.json +1 -1
  2. package/src/lib/dm-short-link-redirect.ts +17 -2
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@m13v/seo-components",
3
- "version": "0.35.0",
3
+ "version": "0.36.0",
4
4
  "scripts": {
5
5
  "build:css": "tailwind -i src/_build.css -o dist/styles.css --minify",
6
6
  "lint:mobile-spans": "node scripts/lint-mobile-spans.mjs",
package/src/lib/dm-short-link-redirect.ts CHANGED
@@ -27,8 +27,20 @@ export interface DmShortLinkRedirectConfig {
27
27
  * but does NOT bump post_links.clicks / dm_links.clicks).
28
28
  * 2. Skips the PostHog `*_short_link_clicked` event (no fake conversions).
29
29
  * 3. Still 302s to target_url so previews render.
30
+ *
31
+ * 2026-05-07: extended after live measurement found ~71% of "human" hits in
32
+ * the last 24h were actually generic HTTP libs and stripped-header scrapers
33
+ * that the original "self-identifying bot" regex missed. New buckets:
34
+ * - Generic HTTP libs (axios, python-requests, python-urllib, node-fetch,
35
+ * bare "node"). No real browser ever sends these.
36
+ * - Specific scraper UAs caught in the wild: dataminr (Twitter intel),
37
+ * Anthill (content scanner).
38
+ * - Spoofed/empty fingerprints: bare "Mozilla/5.0" (no real browser sends
39
+ * just this), Chrome/70.x (2018 release, only seen from headless fleets).
40
+ * - Empty UA is also treated as bot in the caller (see isBot computation
41
+ * below) since real browsers always send a UA.
30
42
  */
31
- const BOT_UA_RE = /bot|crawler|spider|Twitterbot|LinkedInBot|Slackbot|facebookexternalhit|Discordbot|TelegramBot|WhatsApp|Applebot|Googlebot|Bingbot|YandexBot|DuckDuckBot|redditbot|Pinterest|Embedly|Snapchat/i;
43
+ const BOT_UA_RE = /bot|crawler|spider|Twitterbot|LinkedInBot|Slackbot|facebookexternalhit|Discordbot|TelegramBot|WhatsApp|Applebot|Googlebot|Bingbot|YandexBot|DuckDuckBot|redditbot|Pinterest|Embedly|Snapchat|axios\/|python-requests|python-urllib|node-fetch|^node$|dataminr|Anthill|^Mozilla\/5\.0$|Chrome\/70\./i;
32
44
 
33
45
  /**
34
46
  * Factory for `GET /r/[code]`.
@@ -86,7 +98,10 @@ export function createDmShortLinkRedirectHandler(config: DmShortLinkRedirectConf
86
98
  // agent presents passes through this; matched UAs do NOT count as a real
87
99
  // click and do NOT fire PostHog events.
88
100
  const ua = req.headers.get("user-agent") || "";
89
- const isBot = BOT_UA_RE.test(ua);
101
+ // Empty UA is treated as bot: real browsers always send one. Stripped-
102
+ // header scrapers were ~6% of "human" hits in the 24h post 2026-05-07
103
+ // measurement.
104
+ const isBot = !ua || BOT_UA_RE.test(ua);
90
105
  const referrer = req.headers.get("referer") || "";
91
106
 
92
107
  let target: string | null = null;