birdclaw 0.8.3 → 0.8.4

package/src/lib/network-map.ts

@@ -1,382 +0,0 @@
-import { getNativeDb } from "./db";
-import {
-	geocodeLocation,
-	GeocodeRateLimitError,
-	getOpenCageApiKey,
-	readCachedGeocodes,
-	readSuppressedGeocodeKeys,
-	type GeocodeResult,
-} from "./geocoding";
-import { isMeaningfulLocation, normalizeLocationKey } from "./location";
-import type { Database } from "./sqlite";
-
-export type NetworkMapKind = "all" | "followers" | "following" | "mutual";
-
-export interface NetworkMapProfileProperties {
-	profileId: string;
-	handle: string;
-	name: string;
-	avatarUrl: string | null;
-	location: string;
-	resolvedLocation: string | null;
-	followersCount: number;
-	followingCount: number;
-	verified: boolean | null;
-	relationship: "followers" | "following" | "mutual";
-	approxRadiusM: number | null;
-}
-
-export interface NetworkMapFeature {
-	type: "Feature";
-	geometry: { type: "Point"; coordinates: [number, number] };
-	properties: NetworkMapProfileProperties;
-}
-
-export interface NetworkMapResponse {
-	type: "FeatureCollection";
-	features: NetworkMapFeature[];
-	meta: {
-		accountId: string;
-		type: NetworkMapKind;
-		totalProfiles: number;
-		profilesWithLocation: number;
-		meaningfulProfiles: number;
-		locatedProfiles: number;
-		missingGeocodes: number;
-		geocodedThisRun: number;
-		suppressedGeocodes: number;
-		opencageConfigured: boolean;
-		mapboxTokenConfigured: boolean;
-	};
-	config: {
-		mapboxToken: string | null;
-	};
-}
-
-interface ProfileLocationRow {
-	id: string;
-	handle: string;
-	display_name: string;
-	followers_count: number;
-	following_count: number;
-	avatar_url: string | null;
-	location: string | null;
-	verified_type: string | null;
-	in_followers: number;
-	in_following: number;
-}
-
-interface NetworkMapOptions {
-	account?: string;
-	type?: NetworkMapKind;
-	limit?: number;
-	geocodeLimit?: number;
-	refresh?: boolean;
-	signal?: AbortSignal;
-}
-
-const DEFAULT_LIMIT = 10_000;
-const MAX_LIMIT = 50_000;
-const DEFAULT_GEOCODE_LIMIT = 80;
-const MAX_GEOCODE_LIMIT = 500;
-const OPENCAGE_REQUEST_DELAY_MS = 1100;
-
-function abortableDelay(ms: number, signal?: AbortSignal) {
-	if (signal?.aborted) return Promise.reject(new Error("geocode aborted"));
-	return new Promise<void>((resolve, reject) => {
-		const timeout = setTimeout(() => {
-			signal?.removeEventListener("abort", onAbort);
-			resolve();
-		}, ms);
-		function onAbort() {
-			clearTimeout(timeout);
-			reject(new Error("geocode aborted"));
-		}
-		signal?.addEventListener("abort", onAbort, { once: true });
-	});
-}
-
-export function getPublicMapboxToken() {
-	const token =
-		process.env.NEXT_PUBLIC_MAPBOX_ACCESS_TOKEN?.trim() ||
-		process.env.BIRDCLAW_MAPBOX_ACCESS_TOKEN?.trim() ||
-		null;
-	return token?.startsWith("pk.") ? token : null;
-}
-
-function parseLimit(
-	value: number | undefined,
-	fallback: number,
-	max: number,
-	min = 1,
-) {
-	if (!Number.isFinite(value) || value === undefined || value < min)
-		return fallback;
-	return Math.min(max, Math.floor(value));
-}
-
-function resolveAccountId(db: Database, accountId?: string) {
-	const row = accountId
-		? (db
-				.prepare("select id from accounts where id = ? or handle = ? limit 1")
-				.get(accountId, accountId.replace(/^@/, "")) as
-				| { id: string }
-				| undefined)
-		: (db
-				.prepare(
-					"select id from accounts order by is_default desc, created_at asc limit 1",
-				)
-				.get() as { id: string } | undefined);
-	if (!row) throw new Error(`Unknown account: ${accountId ?? "default"}`);
-	return row.id;
-}
-
-function relationshipForRow(
-	row: Pick<ProfileLocationRow, "in_followers" | "in_following">,
-) {
-	if (row.in_followers && row.in_following) return "mutual";
-	if (row.in_followers) return "followers";
-	return "following";
-}
-
-function fetchNetworkRows({
-	db,
-	accountId,
-	type,
-	limit,
-}: {
-	db: Database;
-	accountId: string;
-	type: NetworkMapKind;
-	limit: number;
-}) {
-	const having =
-		type === "mutual"
-			? "having max(case when fe.direction = 'followers' then 1 else 0 end) = 1 and max(case when fe.direction = 'following' then 1 else 0 end) = 1"
-			: type === "followers"
-				? "having max(case when fe.direction = 'followers' then 1 else 0 end) = 1"
-				: type === "following"
-					? "having max(case when fe.direction = 'following' then 1 else 0 end) = 1"
-					: "";
-	const rows = db
-		.prepare(
-			`
-      select
-        p.id,
-        p.handle,
-        p.display_name,
-        p.followers_count,
-        p.following_count,
-        p.avatar_url,
-        p.location,
-        p.verified_type,
-        max(case when fe.direction = 'followers' then 1 else 0 end) as in_followers,
-        max(case when fe.direction = 'following' then 1 else 0 end) as in_following
-      from follow_edges fe
-      join profiles p on p.id = fe.profile_id
-      where fe.account_id = ?
-        and fe.current = 1
-      group by p.id
-      ${having}
-      order by p.followers_count desc, p.handle asc
-      limit ?
-      `,
-		)
-		.all(accountId, limit) as ProfileLocationRow[];
-	return rows;
-}
-
-function collectKeys(rows: ProfileLocationRow[]) {
-	const originalByKey = new Map<string, string>();
-	const keyByProfile = new Map<string, string>();
-	const seen = new Set<string>();
-	const keys: string[] = [];
-	for (const row of rows) {
-		const location = row.location;
-		if (!location || !isMeaningfulLocation(location)) continue;
-		const key = normalizeLocationKey(location);
-		if (!key) continue;
-		keyByProfile.set(row.id, key);
-		if (!originalByKey.has(key)) originalByKey.set(key, location);
-		if (!seen.has(key)) {
-			seen.add(key);
-			keys.push(key);
-		}
-	}
-	return { keys, keyByProfile, originalByKey };
-}
-
-async function fillMissingGeocodes({
-	db,
-	keys,
-	originalByKey,
-	refresh,
-	geocodeLimit,
-	signal,
-}: {
-	db: Database;
-	keys: string[];
-	originalByKey: Map<string, string>;
-	refresh: boolean;
-	geocodeLimit: number;
-	signal?: AbortSignal;
-}) {
-	const cache = readCachedGeocodes(keys, db);
-	const suppressed = readSuppressedGeocodeKeys(keys, db);
-	const coordinateKeys = keys.filter(
-		(key) => key.startsWith("coords:") && (refresh || !cache.has(key)),
-	);
-	const uncachedKeys = keys.filter(
-		(key) =>
-			!key.startsWith("coords:") &&
-			!cache.has(key) &&
-			(refresh || !suppressed.has(key)),
-	);
-	const refreshKeys = refresh
-		? keys.filter((key) => !key.startsWith("coords:") && cache.has(key))
-		: [];
-	const openCageKeys = getOpenCageApiKey()
-		? [...uncachedKeys, ...refreshKeys].slice(0, geocodeLimit)
-		: [];
-	let geocoded = 0;
-	for (const key of coordinateKeys) {
-		if (signal?.aborted) break;
-		const original = originalByKey.get(key);
-		if (!original) continue;
-		const result = await geocodeLocation(original, db, signal).catch(
-			() => null,
-		);
-		if (result) geocoded += 1;
-	}
-	for (let index = 0; index < openCageKeys.length; index += 1) {
-		if (signal?.aborted) break;
-		const key = openCageKeys[index];
-		if (!key) continue;
-		if (index > 0) {
-			await abortableDelay(OPENCAGE_REQUEST_DELAY_MS, signal).catch(() => null);
-			if (signal?.aborted) break;
-		}
-		const original = originalByKey.get(key);
-		if (!original) continue;
-		try {
-			const result = await geocodeLocation(original, db, signal);
-			if (result) geocoded += 1;
-		} catch (error) {
-			if (signal?.aborted) break;
-			if (error instanceof GeocodeRateLimitError) break;
-		}
-	}
-	const updatedCache = readCachedGeocodes(keys, db);
-	const updatedSuppressed = readSuppressedGeocodeKeys(keys, db);
-	return {
-		cache: updatedCache,
-		missingCount: keys.filter(
-			(key) => !updatedCache.has(key) && !updatedSuppressed.has(key),
-		).length,
-		suppressedCount: updatedSuppressed.size,
-		geocoded,
-	};
-}
-
-function buildFeatures({
-	rows,
-	keyByProfile,
-	cache,
-}: {
-	rows: ProfileLocationRow[];
-	keyByProfile: Map<string, string>;
-	cache: Map<string, GeocodeResult>;
-}) {
-	const byKey = new Map<string, ProfileLocationRow[]>();
-	for (const row of rows) {
-		const key = keyByProfile.get(row.id);
-		if (!key || !cache.has(key)) continue;
-		const group = byKey.get(key);
-		if (group) group.push(row);
-		else byKey.set(key, [row]);
-	}
-
-	const features: NetworkMapFeature[] = [];
-	for (const [key, members] of byKey) {
-		const geo = cache.get(key);
-		if (!geo) continue;
-		for (let index = 0; index < members.length; index += 1) {
-			const row = members[index];
-			if (!row?.location) continue;
-			features.push({
-				type: "Feature",
-				geometry: { type: "Point", coordinates: [geo.lng, geo.lat] },
-				properties: {
-					profileId: row.id,
-					handle: row.handle,
-					name: row.display_name,
-					avatarUrl: row.avatar_url,
-					location: row.location,
-					resolvedLocation: geo.formatted ?? null,
-					followersCount: Number(row.followers_count ?? 0),
-					followingCount: Number(row.following_count ?? 0),
-					verified:
-						row.verified_type && row.verified_type !== "none" ? true : null,
-					relationship: relationshipForRow(row),
-					approxRadiusM: geo.approxRadiusM ?? null,
-				},
-			});
-		}
-	}
-	return features;
-}
-
-export async function getNetworkMap(
-	options: NetworkMapOptions = {},
-	db = getNativeDb(),
-): Promise<NetworkMapResponse> {
-	const accountId = resolveAccountId(db, options.account);
-	const type = options.type ?? "all";
-	const limit = parseLimit(options.limit, DEFAULT_LIMIT, MAX_LIMIT);
-	const geocodeLimit = parseLimit(
-		options.geocodeLimit,
-		DEFAULT_GEOCODE_LIMIT,
-		MAX_GEOCODE_LIMIT,
-		0,
-	);
-	const rows = fetchNetworkRows({ db, accountId, type, limit });
-	const rowsWithLocation = rows.filter((row) => row.location);
-	const meaningfulRows = rowsWithLocation.filter((row) =>
-		row.location ? isMeaningfulLocation(row.location) : false,
-	);
-	const { keys, keyByProfile, originalByKey } = collectKeys(meaningfulRows);
-	const geocodes = await fillMissingGeocodes({
-		db,
-		keys,
-		originalByKey,
-		refresh: options.refresh === true,
-		geocodeLimit,
-		signal: options.signal,
-	});
-	const features = buildFeatures({
-		rows: meaningfulRows,
-		keyByProfile,
-		cache: geocodes.cache,
-	});
-	const token = getPublicMapboxToken();
-	return {
-		type: "FeatureCollection",
-		features,
-		meta: {
-			accountId,
-			type,
-			totalProfiles: rows.length,
-			profilesWithLocation: rowsWithLocation.length,
-			meaningfulProfiles: meaningfulRows.length,
-			locatedProfiles: features.length,
-			missingGeocodes: geocodes.missingCount,
-			geocodedThisRun: geocodes.geocoded,
-			suppressedGeocodes: geocodes.suppressedCount,
-			opencageConfigured: Boolean(getOpenCageApiKey()),
-			mapboxTokenConfigured: Boolean(token),
-		},
-		config: {
-			mapboxToken: token,
-		},
-	};
-}

package/src/lib/openai-response-runtime.ts

@@ -1,251 +0,0 @@
-import { Effect } from "effect";
-import { tryPromise } from "./effect-runtime";
-import {
-	defaultRuntimeServices,
-	type RuntimeServices,
-} from "./runtime-services";
-
-const DEFAULT_DELIMITER_PATTERN = /\n---\s*\n/;
-const DEFAULT_DELIMITER_HOLD = 8;
-
-export interface OpenAIStreamState {
-	eventBuffer: string;
-	rawText: string;
-	pendingVisible: string;
-	jsonMode: boolean;
-	responseId?: string;
-	usage?: unknown;
-	error?: string;
-}
-
-export interface OpenAIStreamResult {
-	rawText: string;
-	responseId?: string;
-	usage?: unknown;
-}
-
-function toError(error: unknown) {
-	return error instanceof Error ? error : new Error(String(error));
-}
-
-export function createOpenAIStreamState(): OpenAIStreamState {
-	return {
-		eventBuffer: "",
-		rawText: "",
-		pendingVisible: "",
-		jsonMode: false,
-	};
-}
-
-function emitVisibleDelta(
-	state: OpenAIStreamState,
-	delta: string,
-	onDelta: ((delta: string) => void) | undefined,
-	delimiterPattern: RegExp,
-	delimiterHold: number,
-) {
-	state.rawText += delta;
-	if (state.jsonMode) return;
-
-	const combined = state.pendingVisible + delta;
-	const delimiterIndex = combined.search(delimiterPattern);
-	if (delimiterIndex >= 0) {
-		const visible = combined.slice(0, delimiterIndex);
-		if (visible) onDelta?.(visible);
-		state.pendingVisible = "";
-		state.jsonMode = true;
-		return;
-	}
-
-	if (combined.length <= delimiterHold) {
-		state.pendingVisible = combined;
-		return;
-	}
-
-	const visible = combined.slice(0, -delimiterHold);
-	state.pendingVisible = combined.slice(-delimiterHold);
-	if (visible) onDelta?.(visible);
-}
-
-function handleOpenAIEvent(
-	state: OpenAIStreamState,
-	event: Record<string, unknown>,
-	onDelta: ((delta: string) => void) | undefined,
-	delimiterPattern: RegExp,
-	delimiterHold: number,
-) {
-	const type = typeof event.type === "string" ? event.type : "";
-	if (
-		type === "response.output_text.delta" &&
-		typeof event.delta === "string"
-	) {
-		emitVisibleDelta(
-			state,
-			event.delta,
-			onDelta,
-			delimiterPattern,
-			delimiterHold,
-		);
-		return;
-	}
-	if (type === "response.completed") {
-		const response = event.response;
-		if (response && typeof response === "object") {
-			const record = response as Record<string, unknown>;
-			state.responseId = typeof record.id === "string" ? record.id : undefined;
-			state.usage = record.usage;
-		}
-		return;
-	}
-	if (type === "response.error" || type === "error") {
-		const error = event.error;
-		state.error =
-			error && typeof error === "object" && "message" in error
-				? String((error as { message?: unknown }).message)
-				: "OpenAI stream failed";
-		return;
-	}
-	if (type === "response.failed" || type === "response.incomplete") {
-		const response = event.response;
-		const record =
-			response && typeof response === "object"
-				? (response as Record<string, unknown>)
-				: {};
-		const error = record.error;
-		const incomplete = record.incomplete_details;
-		state.error =
-			error && typeof error === "object" && "message" in error
-				? String((error as { message?: unknown }).message)
-				: incomplete && typeof incomplete === "object" && "reason" in incomplete
-					? `OpenAI response incomplete: ${String((incomplete as { reason?: unknown }).reason)}`
-					: "OpenAI stream failed";
-	}
-}
-
-export function processOpenAIResponseSseChunk(
-	state: OpenAIStreamState,
-	chunk: string,
-	{
-		onDelta,
-		delimiterPattern = DEFAULT_DELIMITER_PATTERN,
-		delimiterHold = DEFAULT_DELIMITER_HOLD,
-	}: {
-		onDelta?: (delta: string) => void;
-		delimiterPattern?: RegExp;
-		delimiterHold?: number;
-	} = {},
-) {
-	state.eventBuffer += chunk;
-	let boundary = state.eventBuffer.indexOf("\n\n");
-	while (boundary >= 0) {
-		const block = state.eventBuffer.slice(0, boundary);
-		state.eventBuffer = state.eventBuffer.slice(boundary + 2);
-		const data = block
-			.split("\n")
-			.filter((line) => line.startsWith("data:"))
-			.map((line) => line.slice(5).trimStart())
-			.join("\n");
-		if (data && data !== "[DONE]") {
-			try {
-				handleOpenAIEvent(
-					state,
-					JSON.parse(data) as Record<string, unknown>,
-					onDelta,
-					delimiterPattern,
-					delimiterHold,
-				);
-			} catch {
-				// The feature parser decides whether partial output remains usable.
-			}
-		}
-		boundary = state.eventBuffer.indexOf("\n\n");
-	}
-}
-
-export function readOpenAIResponseStreamEffect(
-	response: Response,
-	options: {
-		onDelta?: (delta: string) => void;
-		delimiterPattern?: RegExp;
-		delimiterHold?: number;
-	} = {},
-): Effect.Effect<OpenAIStreamResult, Error> {
-	const reader = response.body?.getReader();
-	if (!reader) {
-		return Effect.fail(new Error("OpenAI response did not include a stream"));
-	}
-	const decoder = new TextDecoder();
-
-	return Effect.gen(function* () {
-		const state = createOpenAIStreamState();
-		for (;;) {
-			const { done, value } = yield* tryPromise(() => reader.read()).pipe(
-				Effect.mapError(toError),
-			);
-			if (!done) {
-				processOpenAIResponseSseChunk(
-					state,
-					decoder.decode(value, { stream: true }),
-					options,
-				);
-				continue;
-			}
-			if (!state.jsonMode && state.pendingVisible) {
-				options.onDelta?.(state.pendingVisible);
-			}
-			if (state.error) {
-				return yield* Effect.fail(new Error(state.error));
-			}
-			return {
-				rawText: state.rawText,
-				...(state.responseId ? { responseId: state.responseId } : {}),
-				...(state.usage === undefined ? {} : { usage: state.usage }),
-			};
-		}
-	}).pipe(
-		Effect.ensuring(
-			Effect.sync(() => {
-				reader.releaseLock();
-			}),
-		),
-	);
-}
-
-export function requestOpenAIResponseEffect({
-	body,
-	signal,
-	runtime = defaultRuntimeServices,
-}: {
-	body: unknown;
-	signal?: AbortSignal;
-	runtime?: RuntimeServices;
-}): Effect.Effect<Response, Error> {
-	return Effect.gen(function* () {
-		const apiKey = runtime.env("OPENAI_API_KEY");
-		if (!apiKey) {
-			return yield* Effect.fail(new Error("OPENAI_API_KEY is not set"));
-		}
-		const response = yield* tryPromise(() =>
-			runtime.fetch("https://api.openai.com/v1/responses", {
-				method: "POST",
-				signal,
-				headers: {
-					authorization: `Bearer ${apiKey}`,
-					"content-type": "application/json",
-				},
-				body: JSON.stringify(body),
-			}),
-		).pipe(Effect.mapError(toError));
-		if (!response.ok) {
-			const text = yield* tryPromise(() => response.text()).pipe(
-				Effect.mapError(toError),
-			);
-			return yield* Effect.fail(
-				new Error(
-					`OpenAI request failed: ${String(response.status)} ${text.slice(0, 400)}`,
-				),
-			);
-		}
-		return response;
-	});
-}