birdclaw 0.4.1 → 0.5.1

package/CHANGELOG.md

@@ -2,6 +2,51 @@
 
 ## Unreleased
 
+## 0.5.1 - 2026-05-15
+
+### Fixed
+
+- Harden the published CLI wrapper and release checks so the packaged `birdclaw` binary avoids `tsx` CLI IPC startup and stays covered by lint, format, and smoke tests.
+- Forward shutdown signals through the published CLI and bundled web server, and include referenced script helpers in npm packages.
+- Keep the selected DM conversation visible while its thread refreshes so the reply composer no longer flashes away mid-action.
+- Send the selected web account through manual sync controls so multi-account timelines sync the intended profile.
+- Run web sync requests as background jobs with status polling so the UI no longer holds one blocking sync request open.
+- Add typed web API fetch handling and explicit DMs loading/error/empty states so failed local reads surface cleanly.
+- Add explicit web app sync controls for home timeline, mentions, likes, bookmarks, and DMs so fresh live data can be pulled without leaving the UI.
+- Refine the web app sidebar tagline and theme selector so the brand chrome reads more clearly in compact layouts.
+- Add shared web feed loading/error/empty states with timeline-shaped skeleton rows and move conversation expansion into a cached single-thread surface with hover prefetch.
+- Use the Birdclaw crab-bird mark in the web app chrome, loading states, and empty states; soften dark-mode contrast and replace text-only reply warnings with conversation/replied indicators.
+- Allow the local web app to respond when Tailscale Serve forwards requests through the `clawmac.sheep-coho.ts.net` hostname.
+- Speed up the default home timeline load on large local databases and keep malformed archived media URL entities from crashing the web timeline.
+- Preserve tweet media aspect ratios, open timeline media in an inline viewer, and suppress duplicate media URL cards.
+
+## 0.5.0 - 2026-05-15
+
+### Added
+
+- Add `birdclaw import archive --select` for importing targeted archive slices while preserving unselected local data.
+- Add `birdclaw sync authored` for filling own-tweet gaps from `xurl` after the archive cutoff. Thanks @cavit99.
+- Add live `sync mentions` and `sync mention-threads --mode xurl` ingestion for current mention data and conversation context. Thanks @cavit99.
+- Add `birdclaw media fetch` plus archive bundled-media extraction and live media variant persistence for local originals caching. Thanks @cavit99.
+- Add a `/links` web lane for Hacker News-style top URL and video-provider insights with today, week, month, year, and all-time ranges.
+- Import archive `follower.js`/`following.js` files into the local follow graph and add archive-authored tweet edges so fresh archive imports are immediately queryable without live sync. Thanks @cavit99.
+- Add cache-first followers/following sync, local follow graph queries, and backup/export support for graph snapshots and churn events. Thanks @ma08.
+- Hydrate missing link-discussion profile avatars through `bird`/`xurl` so hover sheets can upgrade archive placeholders into real profile cards.
+- Add inline tweet conversation expansion in the web timeline, preserving the selected reply's parent chain before broad thread context.
+
+### Changed
+
+- Update npm dependencies, including React, Vite, Vitest, Playwright, Tailwind, Kysely, TanStack packages, oxlint, and oxfmt.
+
+### Fixed
+
+- Seed demo link insight data before direct `/links` route loads, so the lane is populated even when it is the first web route opened.
+- Isolate the default `bird` command config test from the maintainer's local `~/.birdclaw/config.json`.
+- Skip non-numeric archive placeholder IDs such as self-DM conversation IDs when hydrating profiles through X, so one malformed local ID no longer aborts the batch. Thanks @nfarina.
+- Include expanded short URLs and link occurrences in Git-friendly backups so linked-tweet search survives backup restore.
+- Prefer `bird` for follow graph sync in `auto` mode, keeping `xurl` as an explicit fallback for accounts where OAuth2 follow reads work.
+- Update the docs site and app icons to use the Birdclaw crab-bird mark instead of the generic bird logo.
+
 ## 0.4.1 - 2026-05-11
 
 ### Added
@@ -51,6 +96,13 @@
 - Allow Playwright e2e runs to use an alternate local port when `3000` is already occupied.
 - Replace maintainer-local documentation links with repo-relative links and align the setup docs with the Node version file. Thanks @stainlu.
 
+## Unreleased
+
+### Fixed
+
+- Fix live `xurl` status detection when the CLI is installed but not authenticated; thanks @kyupark.
+- Default local `bird` integration to `bird` on PATH and report stale configured command paths with setup guidance.
+
 ## 0.2.1 - 2026-04-27
 
 ### Changed

package/README.md

@@ -20,20 +20,27 @@ Status: WIP. Real and usable. Not done. Expect schema churn, transport gaps, and
 - one shared SQLite DB for multiple accounts, with canonical tweets/profiles and account-scoped timeline/collection edges
 - FTS5 search over tweets and DMs
 - archive autodiscovery on macOS
-- archive import for tweets, likes, profiles, and full DMs
+- archive import for tweets, likes, followers/following, profiles, and full DMs
+- selective archive re-imports for one stale slice without wiping the rest of the local store
 - archive import for bookmark exports when present
-- live likes and bookmarks sync through `xurl` or `bird`
+- archive import streams bundled media files into the local originals cache and extracts `video_info.variants[]` for video and animated-GIF rows
+- live authored sync through `xurl`, plus likes and bookmarks through `xurl` or `bird`
+- cache-first followers/following sync through `bird` or `xurl`
+- local follow graph queries for top followers, unfollows, mutuals, and non-mutual following
 - Git-friendly text backups with yearly tweet shards and per-conversation DM shards
 - profile hydration from live Twitter metadata
 - profile-change history, affiliation badge edges, and extracted bio entities for local identity lookups
 - local avatar cache
 - local media cache root under `~/.birdclaw`
+- live syncers persist tweet media variants so `media fetch` can pull originals from `pbs.twimg.com` and `video.twimg.com` on a separate schedule
+- `media fetch` reuses bytes already extracted by `import archive` before falling back to CDN
 
 ### Web UI
 
 - `Home` timeline
 - `Mentions` queue
 - `Likes` and `Bookmarks` review lanes
+- `Links` for Hacker News-style top URLs, video-provider links, and the comments around them across today/week/month/year/all-time windows
 - `DMs` workspace with two-column layout
 - `Inbox` for mixed mention + DM triage
 - `Blocks` for local blocklist maintenance
@@ -74,7 +81,7 @@ Status: WIP. Real and usable. Not done. Expect schema churn, transport gaps, and
 ## Still In Progress
 
 - broader resumable live sync beyond the targeted paths already wired
-- fuller media fetch pipeline
+- thumbnail generation on top of the originals cache
 - richer multi-account UX
 - more complete transport coverage
 - more archive edge-case handling
@@ -102,6 +109,7 @@ Important paths:
 
 - DB: `~/.birdclaw/birdclaw.sqlite`
 - media cache: `~/.birdclaw/media`
+- archive-extracted media: `~/.birdclaw/media/originals/archive/<kind>/<id>/<filename>` where `<kind>` is one of `tweets`, `dms`, `community`, `deleted`, `profile`, `moments`, `dmGroup`
 - avatar cache: `~/.birdclaw/media/thumbs/avatars`
 - Playwright test home: `.playwright-home`
 
@@ -111,6 +119,53 @@ Override the root:
 export BIRDCLAW_HOME=/path/to/custom/root
 ```
 
+### Media fetch
+
+`birdclaw media fetch` fills the local originals cache at
+`~/.birdclaw/media/originals/<media_key>.<ext>` for tweet media URLs already
+stored in `tweets.media_json`. Images come from `pbs.twimg.com`, videos and
+animated GIFs from `video.twimg.com` (highest-bitrate mp4 variant; HLS-only
+media is skipped).
+
+Live syncers (`sync mentions`, `sync mention-threads`, `sync likes`, `sync
+bookmarks`, `sync timeline`) persist `media_json` with `variants[]` ride-along
+metadata so `media fetch` has URLs to download from. Archive-imported tweets
+already carry that shape. Before falling back to HTTP, `media fetch` looks for
+bytes already extracted by `import archive` under
+`~/.birdclaw/media/originals/archive/tweets/<tweetId>/` and copies those into
+the canonical path; reuses are counted in the JSON output as
+`reused_from_archive` and never spend CDN bandwidth.
+
+Legal posture: this is a respectful client-rendering cache, not a scraper. The
+command never enumerates, crawls, or derives Twitter/X CDN URLs. It only
+fetches URLs that birdclaw already has from an archive or API/live sync
+record, skips files that already exist locally, sends a birdclaw user agent,
+paces image requests sequentially by default, caps optional image parallelism
+at five, runs video downloads serially with their own `--video-pacing-ms`,
+streams response bodies to a `.tmp` file with `Range: bytes=<size>-` resume,
+caps each file at `--max-bytes` (100MB default), backs off on `429`, and
+relies on the local file cache for idempotency.
+
+Thumbnail generation and automatic invocation from sync commands are
+intentionally left out. Run it separately, for example from cron or launchd
+every few hours:
+
+```bash
+birdclaw media fetch --json
+birdclaw media fetch --dry-run --limit 20
+birdclaw media fetch --include-video --video-pacing-ms 1500 --max-bytes 209715200 --json
+birdclaw media fetch --no-include-video --parallel 3 --pacing-ms 250 --json
+```
+
+Notes:
+
+- `--include-video` is on by default; pass `--no-include-video` for images
+  only
+- `--kind`, `--since`, and `--limit` scope which tweet rows are inspected
+- `--parallel` applies to image fetches only; video fetches stay serial
+- JSON output reports `images_fetched`, `videos_fetched`, `gifs_fetched`,
+  `reused_from_archive`, and per-kind byte counters
+
 ## Requirements
 
 - Node `25.8.1` or Node 26.x
@@ -163,9 +218,26 @@ Find and import an archive:
 birdclaw archive find --json
 birdclaw import archive --json
 birdclaw import archive ~/Downloads/twitter-archive-2025.zip --json
+```
+
+Optional profile hydration can improve bios, follower counts, and avatars, but it performs live X profile reads and can spend API credits on large archives:
+
+```bash
 birdclaw import hydrate-profiles --json
 ```
 
+`import archive` is idempotent. Re-running parses follower/following edges into the local follow graph, streams bundled media files under `data/tweets_media/`, `data/direct_messages_media/`, and the other archive media folders into `~/.birdclaw/media/originals/archive/<kind>/<id>/`, and pulls `video_info.variants[]` so archive video and animated-GIF rows carry mp4 URLs for the live media fetcher. Already-extracted files are skipped when size matches.
+
+Re-import only one part of a newer archive when you already have live or local data you want to keep:
+
+```bash
+birdclaw import archive ~/Downloads/twitter-archive-2026.zip --select tweets --json
+birdclaw import archive ~/Downloads/twitter-archive-2026.zip --select likes,bookmarks --json
+birdclaw import archive ~/Downloads/twitter-archive-2026.zip --select directMessages --json
+```
+
+Valid `--select` slices are `tweets`, `likes`, `bookmarks`, `profiles`, `directMessages`, `followers`, and `following`. `dms` and `direct-messages` are accepted aliases for `directMessages`.
+
 Back up the local SQLite store as canonical JSONL text:
 
 ```bash
@@ -184,6 +256,10 @@ Start the app:
 birdclaw serve
 ```
 
+Use the Sync button in Home, Mentions, Likes, Bookmarks, or DMs to run the matching live sync from the web UI and then reload the local view. These controls are explicit because live reads can be slow, auth-dependent, or rate-limited.
+
+When running behind a trusted reverse proxy such as Tailscale Serve, add any extra proxy hostnames to `BIRDCLAW_ALLOWED_HOSTS`. The clawmac Tailscale hostname is allowed by default.
+
 First moderation pass:
 
 ```bash
@@ -204,21 +280,47 @@ pnpm cli search tweets --liked --limit 20 --json
 pnpm cli search tweets --bookmarked --limit 20 --json
 ```
 
-### Sync likes, bookmarks, and home timeline
+### Sync authored tweets, likes, bookmarks, home timeline, and mentions
 
-`auto` tries `xurl` first, then falls back to `bird`. Use `bird` directly when the API path is unavailable for the account/token you have locally.
+`auto` tries `xurl` first for likes/bookmarks, then falls back to `bird`. Use `bird` directly when the API path is unavailable for the account/token you have locally. For repeated xurl collection syncs, add `--early-stop` to stop paging once a whole page already exists locally; without `--all` or `--max-pages`, it caps at 10 pages.
 
 ```bash
+pnpm cli sync authored --mode xurl --limit 100 --json
 pnpm cli sync likes --mode auto --limit 100 --refresh --json
 pnpm cli sync bookmarks --mode auto --limit 100 --refresh --json
+pnpm cli sync likes --mode auto --limit 100 --max-pages 5 --early-stop --refresh --json
+pnpm cli sync bookmarks --mode auto --limit 100 --max-pages 5 --early-stop --refresh --json
 pnpm cli sync bookmarks --mode bird --all --max-pages 5 --limit 100 --refresh --json
 pnpm cli sync timeline --limit 100 --refresh --json
+pnpm cli sync mentions --mode xurl --limit 100 --max-pages 3 --refresh --json
 pnpm cli sync mention-threads --limit 30 --delay-ms 1500 --timeout-ms 15000 --json
 ```
 
+Mention context is a two-step sync pipeline: run `sync mentions` to ingest recent mention rows with `kind='mention'`, then run `sync mention-threads --mode xurl` to fill parent/root conversation context.
+
+### Follow graph queries
+
+Follow graph sync is cache-first and defaults to dry-run so repeated agent queries do not keep spending live reads. `auto` prefers `bird` for this path and falls back to `xurl`.
+
+```bash
+pnpm cli sync followers --json
+pnpm cli sync following --json
+pnpm cli sync followers --yes --json
+pnpm cli sync following --yes --json
+pnpm cli sync followers --mode bird --yes --json
+pnpm cli graph summary --json
+pnpm cli graph events --since 2026-05-01 --json
+pnpm cli graph top-followers --limit 20 --json
+pnpm cli graph unfollowed --date 2026-05-01 --json
+pnpm cli graph non-mutual-following --sort followers --limit 100 --json
+pnpm cli graph mutuals --json
+```
+
+Use `--refresh` only when you intentionally want a new live fetch. The `graph` commands are local SQLite reads and never call X. See [follow-graph.md](docs/follow-graph.md) for long-term agent usage notes.
+
 ### Export mentions for agents
 
-Default `birdclaw` mode returns normalized items with `text`, `plainText`, `markdown`, author metadata, and canonical URLs:
+Default `birdclaw` mode exports DB-backed mention items with `text`, `plainText`, `markdown`, author metadata, and canonical URLs:
 
 ```bash
 pnpm cli mentions export "agent" --unreplied --limit 10
@@ -260,7 +362,8 @@ Notes:
 - `actions.transport` accepts `auto`, `bird`, or `xurl`
 - `bird` mode uses your local `bird` CLI and caches its mentions output into birdclaw's canonical store
 - filters still work in `xurl` mode; filtered payloads are rebuilt from the local canonical store after sync
-- `sync likes`, `sync bookmarks`, `sync timeline`, and `sync mention-threads` store live results in the canonical local store; per-account home/mention/like/bookmark membership is kept as edges so shared tweets do not clobber account ownership
+- `sync authored`, `sync mentions`, `sync mention-threads`, `sync likes`, `sync bookmarks`, and `sync timeline` store live results in the canonical local store; per-account authored/home/mention/like/bookmark membership is kept as edges so shared tweets do not clobber account ownership
+- the web UI has explicit Sync buttons for home timeline, mentions, likes, bookmarks, and DMs; they call the same sync paths and then reload the local DB-backed view
 
 ### Research bookmarks and threads
 
@@ -510,12 +613,14 @@ pnpm test
 pnpm coverage
 pnpm build
 pnpm e2e
+pnpm perf:browser -- --scenario=links,links-toggle --iterations=5
 ```
 
 Current bar:
 
 - branch coverage above `80%`
 - Playwright coverage for core UI flows
+- browser perf smoke reports ready/action timings plus API endpoint fan-out
 
 ## CI
 
@@ -533,3 +638,4 @@ Workflow: [ci.yml](.github/workflows/ci.yml)
 - [spec.md](docs/spec.md)
 - [cli.md](docs/cli.md)
 - [data-architecture.md](docs/data-architecture.md)
+- [follow-graph.md](docs/follow-graph.md)

package/bin/birdclaw.mjs

@@ -1,30 +1,69 @@
 #!/usr/bin/env node
-import { spawnSync } from "node:child_process";
+import { spawn } from "node:child_process";
 import { createRequire } from "node:module";
 import { dirname, join } from "node:path";
-import { fileURLToPath } from "node:url";
+import { fileURLToPath, pathToFileURL } from "node:url";
 
 const packageRoot = dirname(dirname(fileURLToPath(import.meta.url)));
 const require = createRequire(import.meta.url);
-const tsxCli = require.resolve("tsx/cli");
+const tsxLoader = pathToFileURL(require.resolve("tsx")).href;
 const birdclawCli = join(packageRoot, "src", "cli.ts");
 
-const result = spawnSync(
+const child = spawn(
 	process.execPath,
-	[tsxCli, birdclawCli, ...process.argv.slice(2)],
+	["--import", tsxLoader, birdclawCli, ...process.argv.slice(2)],
 	{
 		stdio: "inherit",
 		env: process.env,
+		detached: process.platform !== "win32",
 	},
 );
 
-if (result.error) {
-	console.error(result.error.message);
-	process.exit(1);
+const forwardedSignals = ["SIGINT", "SIGTERM", "SIGHUP", "SIGQUIT"];
+
+function removeSignalHandlers() {
+	for (const signal of forwardedSignals) {
+		process.removeListener(signal, forwardSignal);
+	}
+}
+
+function forwardSignal(signal) {
+	if (child.exitCode === null && child.signalCode === null) {
+		signalChild(signal);
+	}
+}
+
+function signalChild(signal) {
+	if (child.pid === undefined) {
+		return;
+	}
+	const targetPid = process.platform === "win32" ? child.pid : -child.pid;
+	try {
+		process.kill(targetPid, signal);
+	} catch (error) {
+		if (error?.code !== "ESRCH") {
+			throw error;
+		}
+	}
 }
 
-if (result.signal) {
-	process.kill(process.pid, result.signal);
+for (const signal of forwardedSignals) {
+	process.on(signal, forwardSignal);
 }
 
-process.exit(result.status ?? 0);
+child.on("error", (error) => {
+	removeSignalHandlers();
+	console.error(error.message);
+	process.exit(1);
+});
+
+child.on("exit", (code, signal) => {
+	removeSignalHandlers();
+
+	if (signal) {
+		process.kill(process.pid, signal);
+		return;
+	}
+
+	process.exit(code ?? 0);
+});

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "birdclaw",
-	"version": "0.4.1",
+	"version": "0.5.1",
 	"description": "Local Twitter memory in SQLite for archives, DMs, likes, bookmarks, and moderation",
 	"homepage": "https://github.com/steipete/birdclaw#readme",
 	"license": "MIT",
@@ -13,6 +13,7 @@
 	},
 	"files": [
 		"bin/",
+		"scripts/",
 		"src/",
 		"!src/**/*.test.ts",
 		"!src/**/*.test.tsx",
@@ -38,54 +39,55 @@
 		"dev": "vite dev --port 3000",
 		"build": "vite build",
 		"preview": "vite preview",
-		"test": "env -u NODE_OPTIONS node ./scripts/run-vitest.mjs run",
-		"coverage": "env -u NODE_OPTIONS node ./scripts/run-vitest.mjs run --coverage",
+		"test": "node ./scripts/run-vitest.mjs run",
+		"coverage": "node ./scripts/run-vitest.mjs run --coverage",
 		"e2e": "playwright test",
 		"typecheck": "tsgo --noEmit",
-		"format": "oxfmt --write src scripts playwright vite.config.ts vitest.config.ts playwright.config.ts",
-		"format:check": "oxfmt --check src scripts playwright vite.config.ts vitest.config.ts playwright.config.ts",
-		"lint": "oxlint --import-plugin --node-plugin --vitest-plugin --deny-warnings -A require-mock-type-parameters -A no-control-regex src scripts playwright vite.config.ts vitest.config.ts playwright.config.ts",
+		"format": "oxfmt --write bin src scripts playwright vite.config.ts vitest.config.ts playwright.config.ts",
+		"format:check": "oxfmt --check bin src scripts playwright vite.config.ts vitest.config.ts playwright.config.ts",
+		"lint": "oxlint --import-plugin --node-plugin --vitest-plugin --deny-warnings -A require-mock-type-parameters -A no-control-regex bin src scripts playwright vite.config.ts vitest.config.ts playwright.config.ts",
 		"check": "pnpm run format:check && pnpm run lint",
+		"perf:browser": "node ./scripts/browser-perf.mjs",
 		"cli": "tsx src/cli.ts",
 		"docs:site": "node scripts/build-docs-site.mjs"
 	},
 	"dependencies": {
-		"@steipete/sweet-cookie": "^0.2.0",
-		"@tailwindcss/vite": "^4.2.4",
-		"@tanstack/devtools-vite": "0.6.0",
-		"@tanstack/react-devtools": "0.10.2",
-		"@tanstack/react-router": "1.169.1",
+		"@steipete/sweet-cookie": "^0.3.0",
+		"@tailwindcss/vite": "^4.3.0",
+		"@tanstack/devtools-vite": "0.7.0",
+		"@tanstack/react-devtools": "0.10.5",
+		"@tanstack/react-router": "1.169.2",
 		"@tanstack/react-router-devtools": "1.166.13",
 		"@tanstack/react-router-ssr-query": "1.166.12",
-		"@tanstack/react-start": "1.167.61",
-		"@tanstack/router-plugin": "^1.167.32",
+		"@tanstack/react-start": "1.167.65",
+		"@tanstack/router-plugin": "^1.167.35",
 		"@vitejs/plugin-react": "^6.0.1",
 		"commander": "^14.0.3",
-		"kysely": "^0.28.17",
-		"lucide-react": "^1.14.0",
-		"react": "^19.2.5",
-		"react-dom": "^19.2.5",
-		"tailwindcss": "^4.2.4",
-		"tsx": "^4.21.0",
-		"vite": "^8.0.10",
-		"zod": "^4.4.2"
+		"kysely": "^0.29.0",
+		"lucide-react": "^1.16.0",
+		"react": "^19.2.6",
+		"react-dom": "^19.2.6",
+		"tailwindcss": "^4.3.0",
+		"tsx": "^4.22.0",
+		"vite": "^8.0.13",
+		"zod": "^4.4.3"
 	},
 	"devDependencies": {
-		"@playwright/test": "^1.59.1",
+		"@playwright/test": "^1.60.0",
 		"@tailwindcss/typography": "^0.5.19",
 		"@testing-library/dom": "^10.4.1",
 		"@testing-library/jest-dom": "^6.9.1",
 		"@testing-library/react": "^16.3.2",
-		"@types/node": "^25.6.0",
+		"@types/node": "^25.8.0",
 		"@types/react": "^19.2.14",
 		"@types/react-dom": "^19.2.3",
-		"@typescript/native-preview": "7.0.0-dev.20260505.1",
-		"@vitest/coverage-v8": "^4.1.5",
+		"@typescript/native-preview": "7.0.0-dev.20260514.1",
+		"@vitest/coverage-v8": "^4.1.6",
 		"jsdom": "^29.1.1",
-		"oxfmt": "^0.47.0",
-		"oxlint": "^1.62.0",
+		"oxfmt": "^0.49.0",
+		"oxlint": "^1.64.0",
 		"typescript": "^6.0.3",
-		"vitest": "^4.1.5"
+		"vitest": "^4.1.6"
 	},
 	"engines": {
 		"node": ">=25.8.1 <27"

package/playwright.config.ts

@@ -28,6 +28,7 @@ export default defineConfig({
 		env: {
 			BIRDCLAW_PLAYWRIGHT_PORT: port,
 			BIRDCLAW_HOME: testHome,
+			BIRDCLAW_DISABLE_LIVE_PROFILE_LOOKUP: "1",
 			BIRDCLAW_DISABLE_LIVE_WRITES: "1",
 		},
 	},

package/public/manifest.json

@@ -1,6 +1,6 @@
 {
-  "short_name": "TanStack App",
-  "name": "Create TanStack App Sample",
+  "short_name": "birdclaw",
+  "name": "birdclaw",
   "icons": [
     {
       "src": "favicon.ico",