@rkarim08/sia 1.0.0

package/src/sync/hlc.ts

@@ -0,0 +1,117 @@
+// Module: hlc — Hybrid Logical Clock utilities for team sync
+// HLC is a plain bigint: upper 48 bits = wall-clock ms, lower 16 bits = counter.
+
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { SIA_HOME } from "@/shared/config";
+
+/** HLC is a plain bigint — no mutable struct. */
+export type HLC = bigint;
+
+const COUNTER_BITS = 16n;
+const COUNTER_MASK = (1n << COUNTER_BITS) - 1n;
+const MAX_COUNTER = 0xffff;
+
+/** Pack wall-clock ms and counter into a single bigint. */
+export function pack(wallMs: number, counter: number): bigint {
+	return (BigInt(wallMs) << COUNTER_BITS) | BigInt(counter);
+}
+
+/** Unpack a bigint HLC into wall-clock ms and counter. */
+export function unpack(value: bigint): { wallMs: number; counter: number } {
+	const counter = Number(value & COUNTER_MASK);
+	const wallMs = Number(value >> COUNTER_BITS);
+	return { wallMs, counter };
+}
+
+/**
+ * Generate the next local HLC value. NO mutation — returns a new bigint.
+ * Monotonic: if wall clock hasn't advanced, increments the counter.
+ */
+export function hlcNow(local: bigint): bigint {
+	const now = Date.now();
+	const { wallMs, counter } = unpack(local);
+	if (now > wallMs) {
+		return pack(now, 0);
+	}
+	if (counter >= MAX_COUNTER) {
+		return pack(wallMs + 1, 0);
+	}
+	return pack(wallMs, counter + 1);
+}
+
+/**
+ * Merge a remote HLC into the local clock (Lamport/HLC merge rules).
+ * NO mutation — returns a new bigint that is causally after both inputs.
+ */
+export function hlcReceive(local: bigint, remote: bigint): bigint {
+	const now = Date.now();
+	const l = unpack(local);
+	const r = unpack(remote);
+	const maxWall = Math.max(l.wallMs, r.wallMs, now);
+
+	if (maxWall === l.wallMs && maxWall === r.wallMs) {
+		const newCounter = Math.max(l.counter, r.counter) + 1;
+		if (newCounter > MAX_COUNTER) {
+			return pack(maxWall + 1, 0);
+		}
+		return pack(maxWall, newCounter);
+	}
+	if (maxWall === l.wallMs) {
+		if (l.counter >= MAX_COUNTER) {
+			return pack(maxWall + 1, 0);
+		}
+		return pack(maxWall, l.counter + 1);
+	}
+	if (maxWall === r.wallMs) {
+		if (r.counter >= MAX_COUNTER) {
+			return pack(maxWall + 1, 0);
+		}
+		return pack(maxWall, r.counter + 1);
+	}
+	// now is strictly greater than both
+	return pack(maxWall, 0);
+}
+
+/**
+ * Persist HLC to disk at `{siaHome}/repos/{repoHash}/hlc.json`.
+ * Creates directory if needed. Writes the bigint as a decimal string.
+ */
+export function persistHlc(repoHash: string, hlc: bigint, siaHome: string = SIA_HOME): void {
+	const dir = join(siaHome, "repos", repoHash);
+	mkdirSync(dir, { recursive: true });
+	const filePath = join(dir, "hlc.json");
+	writeFileSync(filePath, JSON.stringify({ hlc: hlc.toString() }), "utf-8");
+}
+
+/**
+ * Load HLC from disk for the given repo.
+ * Falls back to `pack(Date.now(), 0)` on any error (missing file, parse error, etc.).
+ */
+export function loadHlc(repoHash: string, siaHome: string = SIA_HOME): bigint {
+	const filePath = join(siaHome, "repos", repoHash, "hlc.json");
+	if (!existsSync(filePath)) {
+		return pack(Date.now(), 0);
+	}
+	try {
+		const raw = JSON.parse(readFileSync(filePath, "utf-8")) as { hlc?: string };
+		if (raw.hlc !== undefined) {
+			return BigInt(raw.hlc);
+		}
+		return pack(Date.now(), 0);
+	} catch {
+		return pack(Date.now(), 0);
+	}
+}
+
+/**
+ * Safely convert database column values to BigInt.
+ * Null or undefined -> 0n.
+ */
+export function hlcFromDb(value: unknown): bigint {
+	if (value === null || value === undefined) return 0n;
+	if (typeof value === "bigint") return value;
+	if (typeof value === "number") return BigInt(value);
+	if (typeof value === "string") return BigInt(value);
+	throw new TypeError("Unsupported HLC column type");
+}

package/src/sync/keychain.ts

@@ -0,0 +1,144 @@
+// Module: keychain — OS keychain integration with file fallback
+
+import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { SIA_HOME } from "@/shared/config";
+
+const SERVICE_NAME = "sia-sync";
+const FALLBACK_PATH = join(SIA_HOME, ".tokens");
+const FALLBACK_ENV_FLAG = "SIA_KEYCHAIN_FALLBACK";
+
+function shouldUseFallback(): boolean {
+	return process.env[FALLBACK_ENV_FLAG] === "1";
+}
+
+type KeyringModule = typeof import("@napi-rs/keyring");
+
+let keyringModulePromise: Promise<KeyringModule | null> | null = null;
+
+async function getKeyring(): Promise<KeyringModule | null> {
+	if (!keyringModulePromise) {
+		keyringModulePromise = import("@napi-rs/keyring").catch(() => null);
+	}
+	return keyringModulePromise;
+}
+
+type KeychainEntry = {
+	setPassword(pw: string): Promise<void>;
+	getPassword(): Promise<string | null>;
+	deletePassword(): Promise<void>;
+};
+
+async function getKeychainEntry(serverUrl: string): Promise<KeychainEntry | null> {
+	const keyring = await getKeyring();
+	if (!keyring) return null;
+
+	const kr = keyring as Record<string, unknown>;
+	const krDefault = kr.default as Record<string, unknown> | undefined;
+	const EntryCtor =
+		(kr.Entry as (new (...args: unknown[]) => KeychainEntry) | undefined) ??
+		(krDefault?.Entry as (new (...args: unknown[]) => KeychainEntry) | undefined) ??
+		(kr.Keyring as (new (...args: unknown[]) => KeychainEntry) | undefined) ??
+		(krDefault as unknown as (new (...args: unknown[]) => KeychainEntry) | undefined);
+
+	if (!EntryCtor) return null;
+	try {
+		return new EntryCtor(SERVICE_NAME, serverUrl) as KeychainEntry;
+	} catch {
+		return null;
+	}
+}
+
+function readFallback(): Record<string, string> {
+	if (!existsSync(FALLBACK_PATH)) return {};
+	try {
+		const parsed = JSON.parse(readFileSync(FALLBACK_PATH, "utf-8")) as Record<string, string>;
+		return parsed;
+	} catch {
+		return {};
+	}
+}
+
+function writeFallback(map: Record<string, string>): void {
+	const dir = dirname(FALLBACK_PATH);
+	mkdirSync(dir, { recursive: true });
+	writeFileSync(FALLBACK_PATH, JSON.stringify(map, null, 2), { encoding: "utf-8", mode: 0o600 });
+	try {
+		chmodSync(FALLBACK_PATH, 0o600);
+	} catch {
+		// best effort — continue
+	}
+}
+
+export async function storeToken(serverUrl: string, token: string): Promise<void> {
+	if (shouldUseFallback()) {
+		const map = readFallback();
+		map[serverUrl] = token;
+		writeFallback(map);
+		return;
+	}
+
+	const entry = await getKeychainEntry(serverUrl);
+	if (entry) {
+		try {
+			await entry.setPassword(token);
+			return;
+		} catch {
+			// Fall back to file store on keychain errors
+		}
+	}
+
+	console.warn("OS keychain unavailable — falling back to file storage");
+	const map = readFallback();
+	map[serverUrl] = token;
+	writeFallback(map);
+}
+
+export async function getToken(serverUrl: string): Promise<string | null> {
+	if (shouldUseFallback()) {
+		const map = readFallback();
+		return map[serverUrl] ?? null;
+	}
+
+	const entry = await getKeychainEntry(serverUrl);
+	if (entry) {
+		try {
+			const result = await entry.getPassword();
+			return result ?? null;
+		} catch {
+			// Fall back to file store
+		}
+	}
+
+	console.warn("OS keychain unavailable — falling back to file storage");
+	const map = readFallback();
+	return map[serverUrl] ?? null;
+}
+
+export async function deleteToken(serverUrl: string): Promise<void> {
+	if (shouldUseFallback()) {
+		const map = readFallback();
+		if (map[serverUrl]) {
+			delete map[serverUrl];
+			writeFallback(map);
+		}
+		return;
+	}
+
+	const entry = await getKeychainEntry(serverUrl);
+	if (entry) {
+		try {
+			await entry.deletePassword();
+			return;
+		} catch {
+			// Fall back to file store
+		}
+	}
+
+	console.warn("OS keychain unavailable — falling back to file storage");
+	const map = readFallback();
+	if (map[serverUrl]) {
+		delete map[serverUrl];
+		writeFallback(map);
+	}
+}

package/src/sync/pull.ts

@@ -0,0 +1,232 @@
+// Module: pull — Pull remote changes via libSQL sync
+
+import type { CandidateFact } from "@/capture/types";
+import { writeAuditEntry } from "@/graph/audit";
+import type { SiaDb } from "@/graph/db-interface";
+import type { SyncConfig } from "@/shared/config";
+import { hlcFromDb, hlcReceive, loadHlc, persistHlc } from "@/sync/hlc";
+
+export interface PullResult {
+	entitiesReceived: number;
+	edgesReceived: number;
+	vssRefreshed: number;
+}
+
+export async function pullChanges(
+	db: SiaDb,
+	_bridgeDb: SiaDb,
+	config: SyncConfig,
+	repoHash?: string,
+	siaHome?: string,
+	metaDb?: SiaDb,
+): Promise<PullResult> {
+	if (!config.enabled) {
+		return { entitiesReceived: 0, edgesReceived: 0, vssRefreshed: 0 };
+	}
+
+	// Trigger replication if available (safe optional call)
+	await db.sync?.();
+
+	// Get received entities: non-private with hlc_modified > synced_at (or synced_at IS NULL)
+	const entityRows = await db.execute(
+		`SELECT * FROM graph_nodes
+		 WHERE visibility != 'private'
+		   AND hlc_modified IS NOT NULL
+		   AND (synced_at IS NULL OR synced_at < hlc_modified)`,
+	);
+	const edgeRowsResult = await db.execute(
+		`SELECT id, from_id, to_id, type, weight, confidence, trust_tier,
+		        t_created, t_expired, t_valid_from, t_valid_until,
+		        hlc_created, hlc_modified, source_episode, extraction_method
+		 FROM graph_edges
+		 WHERE hlc_modified IS NOT NULL
+		   AND (t_valid_until IS NULL)`,
+	);
+	const receivedEdges = edgeRowsResult.rows as Array<Record<string, unknown>>;
+
+	const receivedEntities = entityRows.rows as Array<Record<string, unknown>>;
+
+	// --- Consolidation via @/capture/consolidate ---
+	// Convert received entities to CandidateFact[] for consolidation.
+	// We dynamically import to avoid circular dependency issues.
+	const processedEntityIds: string[] = [];
+	if (receivedEntities.length > 0) {
+		try {
+			const { consolidate } = await import("@/capture/consolidate");
+			const candidates: CandidateFact[] = receivedEntities.map((row) => ({
+				type: row.type as string as CandidateFact["type"],
+				name: row.name as string,
+				content: row.content as string,
+				summary: row.summary as string,
+				tags: (() => {
+					try {
+						return JSON.parse((row.tags as string) || "[]") as string[];
+					} catch {
+						return [];
+					}
+				})(),
+				file_paths: (() => {
+					try {
+						return JSON.parse((row.file_paths as string) || "[]") as string[];
+					} catch {
+						return [];
+					}
+				})(),
+				trust_tier: row.trust_tier as number as CandidateFact["trust_tier"],
+				confidence: row.confidence as number,
+				extraction_method: (row.extraction_method as string) ?? undefined,
+			}));
+
+			await consolidate(db, candidates);
+
+			for (const row of receivedEntities) {
+				processedEntityIds.push(row.id as string);
+			}
+		} catch (err) {
+			// If consolidation fails, still track the entities as received
+			console.warn("Consolidation during pull failed:", err);
+			for (const row of receivedEntities) {
+				processedEntityIds.push(row.id as string);
+			}
+		}
+	}
+
+	// --- Update local HLC high-water ---
+	const maxHlcRow = await db.execute(
+		"SELECT MAX(hlc_modified) as max_hlc FROM graph_nodes WHERE hlc_modified IS NOT NULL",
+	);
+	const maxHlc = hlcFromDb((maxHlcRow.rows[0] as { max_hlc?: unknown })?.max_hlc ?? 0n);
+
+	if (repoHash) {
+		let localHlc = loadHlc(repoHash, siaHome);
+		if (maxHlc > 0n) {
+			localHlc = hlcReceive(localHlc, maxHlc);
+		}
+		persistHlc(repoHash, localHlc, siaHome);
+	} else if (maxHlc > 0n) {
+		// No repoHash — merge into a fresh clock (result discarded)
+		hlcReceive(0n, maxHlc);
+	}
+
+	// --- sync_peers ---
+	// Update all known peers' last_seen_at and last_seen_hlc (to the max HLC received).
+	if (metaDb) {
+		const maxHlcNumber = Number(maxHlc);
+		await metaDb.execute(`UPDATE sync_peers SET last_seen_at = ?, last_seen_hlc = ?`, [
+			Date.now(),
+			maxHlcNumber,
+		]);
+	}
+
+	// Audit the received items.
+	for (const row of receivedEntities) {
+		await writeAuditEntry(db, "SYNC_RECV", { entity_id: row.id as string });
+	}
+
+	// --- Apply received edges to local graph ---
+	// For each edge row fetched, INSERT if new or UPDATE if it already exists locally.
+	for (const edge of receivedEdges) {
+		const existingRow = await db.execute("SELECT id FROM graph_edges WHERE id = ?", [edge.id]);
+		if ((existingRow.rows as Array<unknown>).length > 0) {
+			// UPDATE existing edge
+			await db.execute(
+				`UPDATE graph_edges SET
+					from_id = ?, to_id = ?, type = ?, weight = ?, confidence = ?, trust_tier = ?,
+					t_created = ?, t_expired = ?, t_valid_from = ?, t_valid_until = ?,
+					hlc_created = ?, hlc_modified = ?, source_episode = ?, extraction_method = ?
+				 WHERE id = ?`,
+				[
+					edge.from_id,
+					edge.to_id,
+					edge.type,
+					edge.weight,
+					edge.confidence,
+					edge.trust_tier,
+					edge.t_created,
+					edge.t_expired ?? null,
+					edge.t_valid_from ?? null,
+					edge.t_valid_until ?? null,
+					edge.hlc_created ?? null,
+					edge.hlc_modified ?? null,
+					edge.source_episode ?? null,
+					edge.extraction_method ?? null,
+					edge.id,
+				],
+			);
+		} else {
+			// INSERT new edge
+			await db.execute(
+				`INSERT INTO graph_edges (
+					id, from_id, to_id, type, weight, confidence, trust_tier,
+					t_created, t_expired, t_valid_from, t_valid_until,
+					hlc_created, hlc_modified, source_episode, extraction_method
+				) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+				[
+					edge.id,
+					edge.from_id,
+					edge.to_id,
+					edge.type,
+					edge.weight,
+					edge.confidence,
+					edge.trust_tier,
+					edge.t_created,
+					edge.t_expired ?? null,
+					edge.t_valid_from ?? null,
+					edge.t_valid_until ?? null,
+					edge.hlc_created ?? null,
+					edge.hlc_modified ?? null,
+					edge.source_episode ?? null,
+					edge.extraction_method ?? null,
+				],
+			);
+		}
+		await writeAuditEntry(db, "SYNC_RECV", { edge_id: edge.id as string });
+	}
+
+	// --- Scoped VSS refresh ---
+	// Only refresh entities that were actually received (use their IDs, not full table).
+	let vssRefreshed = 0;
+	const sqlite = db.rawSqlite();
+	if (sqlite && processedEntityIds.length > 0) {
+		try {
+			// Build scoped query for just the received entity IDs
+			const placeholders = processedEntityIds.map(() => "?").join(", ");
+			const embedRows = await db.execute(
+				`SELECT rowid, embedding FROM graph_nodes
+				 WHERE id IN (${placeholders}) AND embedding IS NOT NULL AND archived_at IS NULL`,
+				processedEntityIds,
+			);
+
+			// Use rawSqlite() directly for VSS INSERT
+			for (const row of embedRows.rows as Array<{ rowid: number; embedding: Uint8Array }>) {
+				try {
+					sqlite
+						.prepare("INSERT OR REPLACE INTO graph_nodes_vss(rowid, embedding) VALUES (?, ?)")
+						.run(row.rowid, row.embedding);
+					vssRefreshed++;
+				} catch {
+					// Ignore individual VSS insert failures
+				}
+			}
+
+			if (vssRefreshed > 0) {
+				for (const row of embedRows.rows as Array<{ rowid: number }>) {
+					await writeAuditEntry(db, "VSS_REFRESH", { entity_id: String(row.rowid) });
+				}
+			}
+		} catch {
+			// Ignore if extension/table not available
+		}
+	} else if (!sqlite && processedEntityIds.length > 0) {
+		console.warn("rawSqlite() returned null — skipping VSS refresh for received entities");
+	}
+
+	// Trigger replication after processing (safe optional call)
+	await db.sync?.();
+
+	return {
+		entitiesReceived: receivedEntities.length,
+		edgesReceived: receivedEdges.length,
+		vssRefreshed,
+	};
+}

package/src/sync/push.ts

@@ -0,0 +1,131 @@
+// Module: push — Push local changes to remote via libSQL sync
+
+import { writeAuditEntry } from "@/graph/audit";
+import type { SiaDb } from "@/graph/db-interface";
+import type { SyncConfig } from "@/shared/config";
+import { hlcNow, loadHlc, persistHlc } from "@/sync/hlc";
+
+export interface PushResult {
+	entitiesPushed: number;
+	edgesPushed: number;
+	bridgeEdgesPushed: number;
+}
+
+export async function pushChanges(
+	db: SiaDb,
+	config: SyncConfig,
+	bridgeDb?: SiaDb,
+	repoHash?: string,
+	siaHome?: string,
+): Promise<PushResult> {
+	if (!config.enabled) {
+		return { entitiesPushed: 0, edgesPushed: 0, bridgeEdgesPushed: 0 };
+	}
+
+	// Determine the timestamp for synced_at — use HLC if repoHash is provided, else Date.now()
+	let syncedAt: number;
+	let hlc: bigint | undefined;
+	if (repoHash) {
+		hlc = loadHlc(repoHash, siaHome);
+		hlc = hlcNow(hlc);
+		syncedAt = Number(hlc >> 16n); // wall-clock ms from HLC
+	} else {
+		syncedAt = Date.now();
+	}
+
+	// --- Push entities ---
+	const candidates = await db.execute(
+		`SELECT id FROM graph_nodes
+		 WHERE visibility != 'private'
+		   AND (synced_at IS NULL OR (hlc_modified IS NOT NULL AND synced_at < hlc_modified))`,
+	);
+
+	const entityIds = (candidates.rows as Array<{ id: string }>).map((r) => r.id);
+	const entityIdSet = new Set(entityIds);
+
+	if (entityIds.length > 0) {
+		// Batch update synced_at in chunks of 500
+		for (let i = 0; i < entityIds.length; i += 500) {
+			const chunk = entityIds.slice(i, i + 500);
+			const placeholders = chunk.map(() => "?").join(", ");
+			await db.execute(`UPDATE graph_nodes SET synced_at = ? WHERE id IN (${placeholders})`, [
+				syncedAt,
+				...chunk,
+			]);
+		}
+		for (const id of entityIds) {
+			await writeAuditEntry(db, "SYNC_SEND", { entity_id: id });
+		}
+	}
+
+	// --- Push edges ---
+	// Query edges where both from_id AND to_id are in pushed entity IDs set
+	let edgesPushed = 0;
+	if (entityIdSet.size > 0) {
+		const allEdges = await db.execute(
+			`SELECT id, from_id, to_id FROM graph_edges
+			 WHERE t_valid_until IS NULL`,
+		);
+
+		const eligibleEdgeIds: string[] = [];
+		for (const row of allEdges.rows as Array<{ id: string; from_id: string; to_id: string }>) {
+			if (entityIdSet.has(row.from_id) && entityIdSet.has(row.to_id)) {
+				eligibleEdgeIds.push(row.id);
+			}
+		}
+
+		// Batch update hlc_modified on edges in chunks of 500
+		for (let i = 0; i < eligibleEdgeIds.length; i += 500) {
+			const chunk = eligibleEdgeIds.slice(i, i + 500);
+			const placeholders = chunk.map(() => "?").join(", ");
+			await db.execute(`UPDATE graph_edges SET hlc_modified = ? WHERE id IN (${placeholders})`, [
+				syncedAt,
+				...chunk,
+			]);
+		}
+		for (const id of eligibleEdgeIds) {
+			await writeAuditEntry(db, "SYNC_SEND", { edge_id: id });
+		}
+		edgesPushed = eligibleEdgeIds.length;
+	}
+
+	// --- Push bridge edges ---
+	let bridgeEdgesPushed = 0;
+	if (bridgeDb && entityIdSet.size > 0) {
+		const bridgeEdges = await bridgeDb.execute(
+			`SELECT id, source_entity_id, target_entity_id FROM cross_repo_edges
+			 WHERE t_valid_until IS NULL`,
+		);
+
+		const eligibleBridgeIds: string[] = [];
+		for (const row of bridgeEdges.rows as Array<{
+			id: string;
+			source_entity_id: string;
+			target_entity_id: string;
+		}>) {
+			if (entityIdSet.has(row.source_entity_id) && entityIdSet.has(row.target_entity_id)) {
+				eligibleBridgeIds.push(row.id);
+			}
+		}
+
+		for (let i = 0; i < eligibleBridgeIds.length; i += 500) {
+			const chunk = eligibleBridgeIds.slice(i, i + 500);
+			const placeholders = chunk.map(() => "?").join(", ");
+			await bridgeDb.execute(
+				`UPDATE cross_repo_edges SET hlc_modified = ? WHERE id IN (${placeholders})`,
+				[syncedAt, ...chunk],
+			);
+		}
+		bridgeEdgesPushed = eligibleBridgeIds.length;
+	}
+
+	// Persist updated HLC after push
+	if (repoHash && hlc !== undefined) {
+		persistHlc(repoHash, hlc, siaHome);
+	}
+
+	// Trigger replication if available (safe optional call)
+	await db.sync?.();
+
+	return { entitiesPushed: entityIds.length, edgesPushed, bridgeEdgesPushed };
+}

package/src/types/chokidar.d.ts

@@ -0,0 +1,23 @@
+declare module "chokidar" {
+	export interface WatchOptions {
+		ignoreInitial?: boolean;
+		ignored?:
+			| ((path: string, stats?: { isDirectory(): boolean }) => boolean)
+			| RegExp
+			| string
+			| string[];
+	}
+
+	export interface FSWatcher {
+		on(event: "add" | "change" | "unlink", listener: (path: string) => void): this;
+		close(): Promise<void> | void;
+	}
+
+	export function watch(paths: string | string[], options?: WatchOptions): FSWatcher;
+
+	const chokidar: {
+		watch: typeof watch;
+	};
+
+	export default chokidar;
+}