birdclaw 0.2.1 → 0.4.0

package/CHANGELOG.md

@@ -1,6 +1,49 @@
 # CHANGELOG
 
-All notable changes to this project will be documented in this file.
+## Unreleased
+
+## 0.4.0 - 2026-05-09
+
+### Added
+
+- Preserve richer X profile metadata during `bird`/`xurl` profile hydration, including profile URLs, profile URL entities, locations, verification type, raw profile JSON, and X affiliation/highlighted-label metadata.
+- Add first-class `profile_affiliations` storage, backup/export/import support, and `whois --json` `profileEvidence` so identity lookups can explain whether a match came from bio text, profile URLs, affiliation badges, DM context, or expanded links.
+- Add profile-change snapshots for hydrated profiles, preserving prior bio/profile URL/location/verification/affiliation states so identity searches can surface current and previous affiliation evidence.
+- Add first-class bio entity extraction for profile bios and profile URLs, including `@handle`, domain, and company-phrase evidence used by fuzzy identity searches such as `whois "blacksmith guy"`.
+- Add a derived `identity_search_index` and `whois` filters for affiliation-oriented identity lookups: `--affiliation`, `--current-affiliation`, and `--exclude-domain-only`.
+
+### Changed
+
+- Use Node's native `node:sqlite` runtime instead of `better-sqlite3`, removing the native npm dependency while preserving the existing synchronous SQLite API surface.
+- Allow Node 26.x in the package engine range and update install docs for the native SQLite runtime.
+- Improve DM `whois` ranking with Sweetistics-style profile evidence scoring: profile URLs and affiliation badges now boost relevant candidates, while cached profile and URL lookups still avoid repeated API/network work.
+- Resolve synthetic X highlighted-label organization badges into real local organization profile ids when `bird` can hydrate the org handle.
+- Rank current affiliation and bio identity evidence above plain profile domains in `whois`, group human output into ambiguity buckets, and explain "why this person?" with the strongest typed evidence first.
+- Use `bird profiles --json` for batch profile hydration when available, falling back to single-profile `bird user --profile-only --json`.
+
+## 0.3.0 - 2026-05-05
+
+### Added
+
+- Add research mode for turning bookmarked Twitter threads into Markdown briefs, with shared `xurl`/`bird` tweet lookup fallback for thread expansion. Thanks @anupamchugh.
+- Add live home timeline and mention-thread sync commands so local triage can pull current `bird` context into the SQLite store.
+- Add search snippets for tweet and DM results, including deterministic DM snippets when multiple messages in a conversation match. Thanks @mvanhorn.
+- Add `--min-likes` and `--quality-reason` controls for tweet search quality filtering. Thanks @mvanhorn.
+- Store Twitter following counts on profiles and include them in JSONL backups.
+
+### Changed
+
+- Use the native TypeScript preview compiler for the `typecheck` script.
+- Refresh TypeScript and related development dependencies.
+
+### Fixed
+
+- Use the existing Twitter web cookie fallback as the final `auto` transport for block and unblock actions. Thanks @pejmanjohn.
+- Resolve the `bird` transport from `PATH` before falling back to the local development checkout. Thanks @vyctorbrzezowski.
+- Stabilize the presenter timestamp test across local time zones. Thanks @pejmanjohn.
+- Clean up the DMs route render test so CI does not leave React work running after jsdom teardown.
+- Allow Playwright e2e runs to use an alternate local port when `3000` is already occupied.
+- Replace maintainer-local documentation links with repo-relative links and align the setup docs with the Node version file. Thanks @stainlu.
 
 ## 0.2.1 - 2026-04-27
 

package/README.md

@@ -17,7 +17,7 @@ Status: WIP. Real and usable. Not done. Expect schema churn, transport gaps, and
 
 ### Local data + storage
 
-- one shared SQLite DB for multiple accounts
+- one shared SQLite DB for multiple accounts, with canonical tweets/profiles and account-scoped timeline/collection edges
 - FTS5 search over tweets and DMs
 - archive autodiscovery on macOS
 - archive import for tweets, likes, profiles, and full DMs
@@ -25,6 +25,7 @@ Status: WIP. Real and usable. Not done. Expect schema churn, transport gaps, and
 - live likes and bookmarks sync through `xurl` or `bird`
 - Git-friendly text backups with yearly tweet shards and per-conversation DM shards
 - profile hydration from live Twitter metadata
+- profile-change history, affiliation badge edges, and extracted bio entities for local identity lookups
 - local avatar cache
 - local media cache root under `~/.birdclaw`
 
@@ -112,7 +113,7 @@ export BIRDCLAW_HOME=/path/to/custom/root
 
 ## Requirements
 
-- Node `25.8.1`
+- Node `25.8.1` or Node 26.x
 - `pnpm`
 - macOS recommended for Spotlight archive discovery
 - `xurl` optional for live reads / writes
@@ -203,7 +204,7 @@ pnpm cli search tweets --liked --limit 20 --json
 pnpm cli search tweets --bookmarked --limit 20 --json
 ```
 
-### Sync likes and bookmarks
+### Sync likes, bookmarks, and home timeline
 
 `auto` tries `xurl` first, then falls back to `bird`. Use `bird` directly when the API path is unavailable for the account/token you have locally.
 
@@ -211,6 +212,8 @@ pnpm cli search tweets --bookmarked --limit 20 --json
 pnpm cli sync likes --mode auto --limit 100 --refresh --json
 pnpm cli sync bookmarks --mode auto --limit 100 --refresh --json
 pnpm cli sync bookmarks --mode bird --all --max-pages 5 --limit 100 --refresh --json
+pnpm cli sync timeline --limit 100 --refresh --json
+pnpm cli sync mention-threads --limit 30 --delay-ms 1500 --timeout-ms 15000 --json
 ```
 
 ### Export mentions for agents
@@ -257,18 +260,49 @@ Notes:
 - `actions.transport` accepts `auto`, `bird`, or `xurl`
 - `bird` mode uses your local `bird` CLI and caches its mentions output into birdclaw's canonical store
 - filters still work in `xurl` mode; filtered payloads are rebuilt from the local canonical store after sync
-- `sync likes` and `sync bookmarks` store live results in the same local timeline table, so `search tweets --liked` and `search tweets --bookmarked` work across archive and live data
+- `sync likes`, `sync bookmarks`, `sync timeline`, and `sync mention-threads` store live results in the canonical local store; per-account home/mention/like/bookmark membership is kept as edges so shared tweets do not clobber account ownership
+
+### Research bookmarks and threads
+
+`birdclaw research` turns bookmarked tweets into a markdown brief with local thread expansion, live ancestor lookup when needed, and extracted links/handles:
+
+```bash
+birdclaw research "codex" --limit 20 --thread-depth 10 --json
+birdclaw research --account acct_primary --out ~/research/codex.md
+```
 
 ### Search and triage DMs
 
 ```bash
 pnpm cli search dms "prototype" --json
 pnpm cli search dms "layout" --min-followers 1000 --min-influence-score 120 --sort influence --json
+pnpm cli search dms "blacksmith" --context 4 --resolve-profiles --expand-urls --no-xurl-fallback --json
+pnpm cli whois "blacksmith guy" --context 4 --no-xurl-fallback --json
+pnpm cli whois "github guy" --current-affiliation github --exclude-domain-only --no-xurl-fallback
+pnpm cli whois "blacksmith" --tweets --context 4 --no-xurl-fallback --json
 pnpm cli dms sync --limit 50 --refresh --json
 pnpm cli dms list --refresh --limit 10 --json
 pnpm cli dms list --unreplied --min-followers 500 --min-influence-score 90 --sort influence --json
 ```
 
+`--resolve-profiles` fills archive-imported numeric DM profiles through the local
+cache first, then `bird`, then `xurl` unless `--no-xurl-fallback` is set.
+Resolved profiles keep bio, location, profile URL, verification type, structured
+URL entities, raw profile JSON, and any X affiliation badge metadata Birdclaw can
+see. When a highlighted-label badge only gives a synthetic label plus handle,
+Birdclaw tries to hydrate that org handle through `bird` and rewrites the edge to
+the real local organization profile id. Profile changes are snapshotted over
+time, and bios are indexed for `@handles`, domains, and company phrases.
+`whois` uses that profile context plus DM context and cached URL expansion to
+return typed evidence such as `profile_bio`, `profile_url`, `profile_bio_url`,
+`affiliation`, `bio_handle`, `bio_domain`, `bio_company`, `profile_history`,
+`dm_context`, and `expanded_url`. It keeps a derived `identity_search_index`
+for fast local profile-evidence lookups, ranks current affiliation and bio
+identity evidence above plain domains, and groups human output into likely
+affiliated, ecosystem, link-only, and DM-context buckets. Use
+`--affiliation`, `--current-affiliation`, and `--exclude-domain-only` when you
+want "GitHub people" rather than anyone with a `github.com` link.
+
 ### AI inbox
 
 ```bash
@@ -293,7 +327,7 @@ pnpm cli unban @amelia --account acct_primary --transport bird --json
 Notes:
 
 - `ban` / `unban` accept `--transport auto|bird|xurl`
-- `auto` tries `bird` first, then falls back to `xurl` when bird fails
+- `auto` tries `bird` first, then falls back to `xurl`, then `x-web` cookie-backed block/unblock when both fail
 - forced `xurl` writes still verify through `bird status` before sqlite changes
 - Twitter still rejects pure OAuth2 block writes, so `auto` is the safe default for block/unblock
 - `blocks import` accepts newline-delimited blocklists with comments and markdown bullets
@@ -366,8 +400,13 @@ Layout:
 manifest.json
 data/accounts.jsonl
 data/profiles.jsonl
+data/profile_affiliations.jsonl
+data/profile_snapshots.jsonl
+data/profile_bio_entities.jsonl
 data/tweets/YYYY.jsonl
 data/tweets/unknown.jsonl
+data/timeline_edges/home.jsonl
+data/timeline_edges/mention.jsonl
 data/collections/likes.jsonl
 data/collections/bookmarks.jsonl
 data/dms/conversations.jsonl
@@ -376,7 +415,7 @@ data/moderation/blocks.jsonl
 data/moderation/mutes.jsonl
 ```
 
-Tweets are sharded by year for human browsing and yearly analysis. Collection-only tweets whose real creation date is unknown go into `data/tweets/unknown.jsonl` instead of pretending they belong to 1970. DMs are sharded by year with `conversation_id` in each row; this keeps Git fast while preserving conversation membership. Likes and bookmarks are stored as collection edges in `data/collections` and mirrored into the timeline rows for current query compatibility.
+Tweets are sharded by year for human browsing and yearly analysis. Collection-only tweets whose real creation date is unknown go into `data/tweets/unknown.jsonl` instead of pretending they belong to 1970. Timeline membership is stored in `data/timeline_edges`; likes and bookmarks are stored as account-scoped collection edges in `data/collections`. DMs are sharded by year with `conversation_id` in each row; this keeps Git fast while preserving conversation membership.
 
 Use `backup sync` when the target is a private Git repo. It pulls first, merge-imports the remote backup into local SQLite, exports the local union back into text shards, commits, and pushes.
 
@@ -466,11 +505,11 @@ pnpm cli auth status --json
 ## Testing
 
 ```bash
-fnm exec --using 25.8.1 pnpm check
-fnm exec --using 25.8.1 pnpm test
-fnm exec --using 25.8.1 pnpm coverage
-fnm exec --using 25.8.1 pnpm build
-fnm exec --using 25.8.1 pnpm e2e
+pnpm check
+pnpm test
+pnpm coverage
+pnpm build
+pnpm e2e
 ```
 
 Current bar:
@@ -487,10 +526,10 @@ GitHub Actions runs:
 - `pnpm build`
 - `pnpm e2e`
 
-Workflow: [ci.yml](/Users/steipete/Projects/birdclaw/.github/workflows/ci.yml)
+Workflow: [ci.yml](.github/workflows/ci.yml)
 
 ## Docs
 
-- [spec.md](/Users/steipete/Projects/birdclaw/docs/spec.md)
-- [cli.md](/Users/steipete/Projects/birdclaw/docs/cli.md)
-- [data-architecture.md](/Users/steipete/Projects/birdclaw/docs/data-architecture.md)
+- [spec.md](docs/spec.md)
+- [cli.md](docs/cli.md)
+- [data-architecture.md](docs/data-architecture.md)

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "birdclaw",
-	"version": "0.2.1",
+	"version": "0.4.0",
 	"description": "Local Twitter memory in SQLite for archives, DMs, likes, bookmarks, and moderation",
 	"homepage": "https://github.com/steipete/birdclaw#readme",
 	"license": "MIT",
@@ -46,29 +46,29 @@
 		"format:check": "oxfmt --check src scripts playwright vite.config.ts vitest.config.ts playwright.config.ts",
 		"lint": "oxlint --import-plugin --node-plugin --vitest-plugin --deny-warnings -A require-mock-type-parameters -A no-control-regex src scripts playwright vite.config.ts vitest.config.ts playwright.config.ts",
 		"check": "pnpm run format:check && pnpm run lint",
-		"cli": "tsx src/cli.ts"
+		"cli": "tsx src/cli.ts",
+		"docs:site": "node scripts/build-docs-site.mjs"
 	},
 	"dependencies": {
 		"@steipete/sweet-cookie": "^0.2.0",
 		"@tailwindcss/vite": "^4.2.4",
 		"@tanstack/devtools-vite": "0.6.0",
 		"@tanstack/react-devtools": "0.10.2",
-		"@tanstack/react-router": "1.168.24",
+		"@tanstack/react-router": "1.169.1",
 		"@tanstack/react-router-devtools": "1.166.13",
 		"@tanstack/react-router-ssr-query": "1.166.12",
-		"@tanstack/react-start": "1.167.49",
-		"@tanstack/router-plugin": "^1.167.27",
+		"@tanstack/react-start": "1.167.61",
+		"@tanstack/router-plugin": "^1.167.32",
 		"@vitejs/plugin-react": "^6.0.1",
-		"better-sqlite3": "^12.9.0",
 		"commander": "^14.0.3",
-		"kysely": "^0.28.16",
-		"lucide-react": "^1.11.0",
+		"kysely": "^0.28.17",
+		"lucide-react": "^1.14.0",
 		"react": "^19.2.5",
 		"react-dom": "^19.2.5",
 		"tailwindcss": "^4.2.4",
 		"tsx": "^4.21.0",
 		"vite": "^8.0.10",
-		"zod": "^4.3.6"
+		"zod": "^4.4.2"
 	},
 	"devDependencies": {
 		"@playwright/test": "^1.59.1",
@@ -76,24 +76,23 @@
 		"@testing-library/dom": "^10.4.1",
 		"@testing-library/jest-dom": "^6.9.1",
 		"@testing-library/react": "^16.3.2",
-		"@types/better-sqlite3": "^7.6.13",
 		"@types/node": "^25.6.0",
 		"@types/react": "^19.2.14",
 		"@types/react-dom": "^19.2.3",
+		"@typescript/native-preview": "7.0.0-dev.20260505.1",
 		"@vitest/coverage-v8": "^4.1.5",
-		"jsdom": "^29.1.0",
-		"oxfmt": "^0.46.0",
-		"oxlint": "^1.61.0",
+		"jsdom": "^29.1.1",
+		"oxfmt": "^0.47.0",
+		"oxlint": "^1.62.0",
 		"typescript": "^6.0.3",
 		"vitest": "^4.1.5"
 	},
 	"engines": {
-		"node": ">=25.8.1 <26"
+		"node": ">=25.8.1 <27"
 	},
 	"packageManager": "pnpm@10.31.0",
 	"pnpm": {
 		"onlyBuiltDependencies": [
-			"better-sqlite3",
 			"esbuild",
 			"lightningcss"
 		]

package/playwright.config.ts

@@ -2,6 +2,8 @@ import path from "node:path";
 import { defineConfig, devices } from "@playwright/test";
 
 const testHome = path.join(process.cwd(), ".playwright-home");
+const port = process.env.BIRDCLAW_PLAYWRIGHT_PORT ?? "3000";
+const baseURL = `http://127.0.0.1:${port}`;
 
 export default defineConfig({
 	testDir: "./playwright",
@@ -9,7 +11,7 @@ export default defineConfig({
 	retries: 0,
 	workers: 1,
 	use: {
-		baseURL: "http://127.0.0.1:3000",
+		baseURL,
 		trace: "on-first-retry",
 	},
 	projects: [
@@ -20,10 +22,11 @@ export default defineConfig({
 	],
 	webServer: {
 		command: "node ./scripts/start-test-server.mjs",
-		url: "http://127.0.0.1:3000",
+		url: baseURL,
 		reuseExistingServer: false,
 		timeout: 120000,
 		env: {
+			BIRDCLAW_PLAYWRIGHT_PORT: port,
 			BIRDCLAW_HOME: testHome,
 			BIRDCLAW_DISABLE_LIVE_WRITES: "1",
 		},