crawler-user-agents 1.49.0 → 1.51.0

package/.github/workflows/release-binary.yml

@@ -0,0 +1,25 @@
+# Build a statically linked clf-filter binary and attach it to the GitHub release for each tag.
+
+name: Release binary
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  release-binary:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - name: Build static binary
+        run: CGO_ENABLED=0 go build -ldflags="-w -s" -o clf-filter ./cmd/clf-filter/
+      - name: Upload binary to release
+        uses: softprops/action-gh-release@v2
+        with:
+          files: clf-filter

package/cmd/clf-filter/main.go

@@ -0,0 +1,51 @@
+// clf-filter reads Combined Log Format lines from stdin and writes them to stdout,
+// removing bot/crawler lines by default. Use --bot to keep only bot lines.
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+
+	agents "github.com/monperrus/crawler-user-agents"
+)
+
+func extractUserAgent(line string) (string, bool) {
+	// Combined Log Format ends with: "referer" "user-agent"
+	// Find the last quoted field.
+	end := strings.LastIndex(line, "\"")
+	if end < 1 {
+		return "", false
+	}
+	start := strings.LastIndex(line[:end], "\"")
+	if start < 0 {
+		return "", false
+	}
+	return line[start+1 : end], true
+}
+
+func main() {
+	botOnly := flag.Bool("bot", false, "keep only bot/crawler lines (default: remove bots)")
+	flag.Parse()
+
+	scanner := bufio.NewScanner(os.Stdin)
+	// Support long lines (e.g. large URLs).
+	scanner.Buffer(make([]byte, 1024*1024), 1024*1024)
+
+	for scanner.Scan() {
+		line := scanner.Text()
+		ua, ok := extractUserAgent(line)
+		isBot := ok && agents.IsCrawler(ua)
+
+		if *botOnly == isBot {
+			fmt.Println(line)
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		fmt.Fprintln(os.Stderr, "clf-filter: read error:", err)
+		os.Exit(1)
+	}
+}

package/crawler-user-agents.json

@@ -18363,5 +18363,17 @@
     "tags": [
       "monitoring"
     ]
+  },
+  {
+    "pattern": "GeedoShopProductFinder",
+    "addition_date": "2026/06/17",
+    "url": "https://geedo.com/product-finder/",
+    "instances": [
+      "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; GeedoShopProductFinder) Chrome/142.0.0.0 Safari/537.36"
+    ],
+    "description": "GeedoShopProductFinder is the automated crawler used by Geedo, a global product-search engine specialized in online retail.",
+    "tags": [
+      "search-engine"
+    ]
   }
 ]

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawler-user-agents",
-  "version": "1.49.0",
+  "version": "1.51.0",
   "main": "crawler-user-agents.json",
   "typings": "./index.d.ts",
   "exports": {