@nothumanwork/nn 0.1.0

package/src/db/client.ts

@@ -0,0 +1,333 @@
+import type { Client } from "@libsql/client";
+
+import { projectDir } from "../config/paths.ts";
+import {
+  contentHash,
+  eventIdFor,
+  projectIdFor,
+  sessionIdFor,
+} from "../ir/event-id.ts";
+import type {
+  NormalizedEvent,
+  ProviderId,
+  SourceImportBatch,
+  SourceSessionRef,
+} from "../ir/types.ts";
+import { getDb, withDb } from "./migrate.ts";
+
+export interface StoredSession {
+  id: string;
+  source: ProviderId;
+  sourceSessionId: string;
+  workspaceRoot: string | null;
+  sourcePath: string | null;
+  title: string | null;
+}
+
+export async function upsertProject(
+  client: Client,
+  canonicalPath: string,
+): Promise<string> {
+  const id = projectIdFor(canonicalPath);
+  const now = Date.now();
+  await client.execute({
+    sql: `INSERT INTO projects(id, canonical_path, display_name, created_at, updated_at)
+          VALUES (?, ?, ?, ?, ?)
+          ON CONFLICT(id) DO UPDATE SET updated_at = excluded.updated_at`,
+    args: [id, canonicalPath, canonicalPath, now, now],
+  });
+  return id;
+}
+
+export async function upsertSession(
+  client: Client,
+  projectId: string,
+  session: SourceSessionRef,
+): Promise<string> {
+  const id = sessionIdFor(projectId, session.source, session.sessionId);
+  const now = Date.now();
+  await client.execute({
+    sql: `INSERT INTO sessions(
+            id, project_id, source, source_session_id, workspace_root, source_path,
+            title, model, started_at, updated_at, created_at, modified_at
+          ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+          ON CONFLICT(project_id, source, source_session_id) DO UPDATE SET
+            workspace_root = COALESCE(excluded.workspace_root, sessions.workspace_root),
+            source_path = COALESCE(excluded.source_path, sessions.source_path),
+            title = COALESCE(excluded.title, sessions.title),
+            model = COALESCE(excluded.model, sessions.model),
+            updated_at = COALESCE(excluded.updated_at, sessions.updated_at),
+            modified_at = excluded.modified_at`,
+    args: [
+      id,
+      projectId,
+      session.source,
+      session.sessionId,
+      session.workspaceRoot ?? null,
+      session.path,
+      session.title ?? null,
+      session.model ?? null,
+      new Date().toISOString(),
+      new Date().toISOString(),
+      now,
+      now,
+    ],
+  });
+  return id;
+}
+
+export async function getNextSeq(
+  client: Client,
+  sessionId: string,
+): Promise<number> {
+  const result = await client.execute({
+    sql: "SELECT COALESCE(MAX(seq), 0) AS max_seq FROM events WHERE session_id = ?",
+    args: [sessionId],
+  });
+  const maxSeq = Number(result.rows[0]?.max_seq ?? 0);
+  return maxSeq + 1;
+}
+
+export async function insertEvents(
+  client: Client,
+  sessionId: string,
+  events: NormalizedEvent[],
+): Promise<number> {
+  if (events.length === 0) {
+    return 0;
+  }
+
+  let seq = await getNextSeq(client, sessionId);
+  let inserted = 0;
+  const now = Date.now();
+
+  for (const event of events) {
+    const id = eventIdFor(event);
+    const result = await client.execute({
+      sql: `INSERT OR IGNORE INTO events(
+              id, session_id, seq, boundary, role, timestamp, content_text,
+              tool_name, tool_call_id, tool_input_json, tool_output_json,
+              raw_json, parser_version, raw_local_ref, line_key, content_hash, created_at
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      args: [
+        id,
+        sessionId,
+        seq,
+        event.boundary,
+        event.role,
+        event.timestamp,
+        event.contentText,
+        event.toolName ?? null,
+        event.toolCallId ?? null,
+        event.toolInputJson ?? null,
+        event.toolOutputJson ?? null,
+        event.rawJson,
+        event.parserVersion,
+        event.rawLocalRef,
+        event.lineKey,
+        contentHash(event.contentText),
+        now,
+      ],
+    });
+    if (result.rowsAffected > 0) {
+      inserted += 1;
+      seq += 1;
+    }
+  }
+
+  return inserted;
+}
+
+export async function setSourceCursor(
+  client: Client,
+  source: ProviderId,
+  cursorKey: string,
+  lastByteOffset: number,
+  parserVersion: string,
+  cursorValue: string,
+  lastLineUuid?: string | null,
+): Promise<void> {
+  await client.execute({
+    sql: `INSERT INTO source_cursors(source, cursor_key, last_byte_offset, last_line_uuid, parser_version, cursor_value, updated_at)
+          VALUES (?, ?, ?, ?, ?, ?, ?)
+          ON CONFLICT(source, cursor_key) DO UPDATE SET
+            last_byte_offset = excluded.last_byte_offset,
+            last_line_uuid = excluded.last_line_uuid,
+            parser_version = excluded.parser_version,
+            cursor_value = excluded.cursor_value,
+            updated_at = excluded.updated_at`,
+    args: [
+      source,
+      cursorKey,
+      lastByteOffset,
+      lastLineUuid ?? null,
+      parserVersion,
+      cursorValue,
+      Date.now(),
+    ],
+  });
+}
+
+export async function getSourceCursor(
+  client: Client,
+  source: ProviderId,
+  cursorKey: string,
+): Promise<number> {
+  const result = await client.execute({
+    sql: "SELECT last_byte_offset FROM source_cursors WHERE source = ? AND cursor_key = ?",
+    args: [source, cursorKey],
+  });
+  return Number(result.rows[0]?.last_byte_offset ?? 0);
+}
+
+export async function ingestBatch(
+  batch: SourceImportBatch,
+): Promise<{ inserted: number; sessionId: string }> {
+  const client = await getDb();
+  const projectId = await upsertProject(client, projectDir());
+  const sessionId = await upsertSession(client, projectId, batch.session);
+  const inserted = await insertEvents(client, sessionId, batch.events);
+  await setSourceCursor(
+    client,
+    batch.source,
+    batch.cursorKey,
+    batch.consumedBytes,
+    batch.parserVersion,
+    `line:${batch.emittedLines};bytes:${batch.consumedBytes}`,
+  );
+  return { inserted, sessionId };
+}
+
+export async function listSessions(
+  provider?: ProviderId,
+): Promise<StoredSession[]> {
+  return withDb(async (client) => {
+    const sql =
+      provider ?
+        "SELECT id, source, source_session_id, workspace_root, source_path, title FROM sessions WHERE source = ? ORDER BY modified_at DESC"
+      : "SELECT id, source, source_session_id, workspace_root, source_path, title FROM sessions ORDER BY modified_at DESC";
+    const args = provider ? [provider] : [];
+    const result = await client.execute({ sql, args });
+    return result.rows.map((row) => ({
+      id: String(row.id),
+      source: String(row.source) as ProviderId,
+      sourceSessionId: String(row.source_session_id),
+      workspaceRoot: row.workspace_root ? String(row.workspace_root) : null,
+      sourcePath: row.source_path ? String(row.source_path) : null,
+      title: row.title ? String(row.title) : null,
+    }));
+  });
+}
+
+export async function getSessionEvents(sessionId: string): Promise<
+  Array<{
+    seq: number;
+    boundary: string;
+    role: string;
+    timestamp: string;
+    contentText: string;
+    toolName: string | null;
+    toolCallId: string | null;
+    toolInputJson: string | null;
+    toolOutputJson: string | null;
+    rawJson: string;
+    source: ProviderId;
+    sourceSessionId: string;
+  }>
+> {
+  return withDb(async (client) => {
+    const result = await client.execute({
+      sql: `SELECT e.seq, e.boundary, e.role, e.timestamp, e.content_text, e.tool_name,
+                   e.tool_call_id, e.tool_input_json, e.tool_output_json, e.raw_json,
+                   s.source, s.source_session_id
+            FROM events e
+            JOIN sessions s ON s.id = e.session_id
+            WHERE e.session_id = ?
+            ORDER BY e.seq ASC`,
+      args: [sessionId],
+    });
+    return result.rows.map((row) => ({
+      seq: Number(row.seq),
+      boundary: String(row.boundary),
+      role: String(row.role),
+      timestamp: String(row.timestamp),
+      contentText: String(row.content_text),
+      toolName: row.tool_name ? String(row.tool_name) : null,
+      toolCallId: row.tool_call_id ? String(row.tool_call_id) : null,
+      toolInputJson: row.tool_input_json ? String(row.tool_input_json) : null,
+      toolOutputJson:
+        row.tool_output_json ? String(row.tool_output_json) : null,
+      rawJson: String(row.raw_json),
+      source: String(row.source) as ProviderId,
+      sourceSessionId: String(row.source_session_id),
+    }));
+  });
+}
+
+export async function searchEvents(
+  query: string,
+  limit = 20,
+): Promise<
+  Array<{
+    sessionId: string;
+    seq: number;
+    contentText: string;
+    boundary: string;
+  }>
+> {
+  return withDb(async (client) => {
+    const result = await client.execute({
+      sql: `SELECT e.session_id, e.seq, e.content_text, e.boundary
+            FROM events_fts fts
+            JOIN events e ON e.rowid = fts.rowid
+            WHERE events_fts MATCH ?
+            ORDER BY rank
+            LIMIT ?`,
+      args: [query, limit],
+    });
+    return result.rows.map((row) => ({
+      sessionId: String(row.session_id),
+      seq: Number(row.seq),
+      contentText: String(row.content_text),
+      boundary: String(row.boundary),
+    }));
+  });
+}
+
+export async function recordExport(
+  sessionId: string,
+  targetProvider: string,
+  outputPath: string,
+  lossy: boolean,
+  warnings: string[],
+): Promise<void> {
+  await withDb(async (client) => {
+    const id = `exp:v1:${contentHash(`${sessionId}|${targetProvider}|${outputPath}|${Date.now()}`)}`;
+    await client.execute({
+      sql: `INSERT INTO exports(id, session_id, target_provider, output_path, exported_at, lossy, warnings_json)
+            VALUES (?, ?, ?, ?, ?, ?, ?)`,
+      args: [
+        id,
+        sessionId,
+        targetProvider,
+        outputPath,
+        Date.now(),
+        lossy ? 1 : 0,
+        JSON.stringify(warnings),
+      ],
+    });
+  });
+}
+
+export async function findSessionBySourceId(
+  source: ProviderId,
+  sourceSessionId: string,
+): Promise<string | null> {
+  return withDb(async (client) => {
+    const result = await client.execute({
+      sql: "SELECT id FROM sessions WHERE source = ? AND source_session_id = ? LIMIT 1",
+      args: [source, sourceSessionId],
+    });
+    return result.rows[0]?.id ? String(result.rows[0].id) : null;
+  });
+}

package/src/db/libsql-native.ts

@@ -0,0 +1,66 @@
+import { embeddedFiles } from "bun";
+import { createRequire } from "node:module";
+import Module from "node:module";
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+
+import { isCompiledExecutable } from "../config/runtime.ts";
+
+function libsqlPackageName(): string {
+  if (process.platform === "darwin" && process.arch === "arm64") {
+    return "@libsql/darwin-arm64";
+  }
+  if (process.platform === "darwin" && process.arch === "x64") {
+    return "@libsql/darwin-x64";
+  }
+  if (process.platform === "linux" && process.arch === "arm64") {
+    return "@libsql/linux-arm64-gnu";
+  }
+  if (process.platform === "linux" && process.arch === "x64") {
+    return "@libsql/linux-x64-gnu";
+  }
+  if (process.platform === "win32" && process.arch === "x64") {
+    return "@libsql/win32-x64-msvc";
+  }
+  throw new Error(
+    `unsupported platform for libsql: ${process.platform}-${process.arch}`,
+  );
+}
+
+function libsqlPackageSuffix(packageName: string): string {
+  return packageName.replace("@libsql/", "");
+}
+
+async function installCompiledLibsqlNative(): Promise<void> {
+  const packageName = libsqlPackageName();
+  const suffix = libsqlPackageSuffix(packageName);
+  const blob =
+    embeddedFiles.find(
+      (file) => file.name.includes("libsql") && file.name.includes(suffix),
+    ) ?? embeddedFiles.find((file) => file.name.endsWith(".node"));
+  if (!blob) {
+    throw new Error(
+      `compiled nn is missing embedded libsql native addon for ${packageName}`,
+    );
+  }
+
+  const dir = join(tmpdir(), "nn-libsql", suffix);
+  mkdirSync(dir, { recursive: true });
+  const nativePath = join(dir, "index.node");
+  writeFileSync(nativePath, await blob.arrayBuffer());
+
+  const require = createRequire(import.meta.url);
+  const native = require(nativePath);
+  const originalRequire = Module.prototype.require;
+  Module.prototype.require = function (id: string) {
+    if (id === packageName) {
+      return native;
+    }
+    return originalRequire.call(this, id);
+  };
+}
+
+if (isCompiledExecutable()) {
+  await installCompiledLibsqlNative();
+}

package/src/db/lock.ts

@@ -0,0 +1,72 @@
+import { createHash } from "node:crypto";
+import {
+  closeSync,
+  existsSync,
+  mkdirSync,
+  openSync,
+  statSync,
+  unlinkSync,
+  writeFileSync,
+} from "node:fs";
+import { dirname } from "node:path";
+
+import { lockPath } from "../config/paths.ts";
+import schemaSql from "./schema.sql" with { type: "text" };
+
+const LOCK_MAX_WAIT_MS = 5_000;
+const LOCK_RETRY_MS = 25;
+const LOCK_STALE_MS = 60_000;
+
+export function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export async function withIngestLock<T>(fn: () => Promise<T> | T): Promise<T> {
+  const path = lockPath();
+  mkdirSync(dirname(path), { recursive: true });
+  const deadline = Date.now() + LOCK_MAX_WAIT_MS;
+
+  while (Date.now() < deadline) {
+    try {
+      if (existsSync(path)) {
+        try {
+          const stale = Date.now() - statSync(path).mtimeMs > LOCK_STALE_MS;
+          if (stale) {
+            unlinkSync(path);
+          }
+        } catch {
+          // ignore
+        }
+      }
+
+      const fd = openSync(path, "wx");
+      writeFileSync(fd, `${process.pid}\n${Date.now()}\n`);
+      closeSync(fd);
+
+      try {
+        return await fn();
+      } finally {
+        try {
+          unlinkSync(path);
+        } catch {
+          // ignore
+        }
+      }
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "EEXIST") {
+        throw error;
+      }
+      await sleep(LOCK_RETRY_MS);
+    }
+  }
+
+  throw new Error("[nn] timed out waiting for ingest lock");
+}
+
+export function checksum(content: string): string {
+  return createHash("sha256").update(content, "utf8").digest("hex");
+}
+
+export function readSchemaSql(): string {
+  return schemaSql;
+}

package/src/db/migrate.ts

@@ -0,0 +1,246 @@
+import { createClient, type Client } from "@libsql/client";
+import { existsSync, mkdirSync } from "node:fs";
+import { dirname } from "node:path";
+
+import { syncConfig, tursoConfigured } from "../config/sync.ts";
+import { dbPath } from "../config/paths.ts";
+import { checksum, readSchemaSql, withIngestLock } from "./lock.ts";
+import {
+  findPendingBackup,
+  importedBackupDir,
+  importPendingLocalBackup,
+  isReplicaConflictError,
+  prepareForEmbeddedReplica,
+  recoverEmbeddedReplica,
+} from "./replica-migrate.ts";
+
+const MIGRATION_VERSION = 1;
+const MIGRATION_NAME = "initial_schema";
+
+let cachedClient: Client | null = null;
+let activeDbClient: Client | null = null;
+
+export function createDbClient(path = dbPath()): Client {
+  mkdirSync(dirname(path), { recursive: true });
+  const url = path.startsWith("file:") ? path : `file:${path}`;
+
+  if (tursoConfigured()) {
+    prepareForEmbeddedReplica(path);
+    const config = syncConfig();
+    return createClient({
+      url,
+      syncUrl: process.env.TURSO_DATABASE_URL!,
+      authToken: process.env.TURSO_AUTH_TOKEN!,
+      syncInterval:
+        config.syncIntervalMs > 0 ? config.syncIntervalMs : undefined,
+    });
+  }
+
+  return createClient({ url });
+}
+
+export async function withDb<T>(
+  fn: (client: Client) => Promise<T>,
+): Promise<T> {
+  return withIngestLock(async () => {
+    if (tursoConfigured()) {
+      const client = await openDbClient();
+      activeDbClient = client;
+      try {
+        return await fn(client);
+      } finally {
+        client.close();
+        activeDbClient = null;
+        resetDbCache();
+      }
+    }
+
+    const client = await getDb();
+    return fn(client);
+  });
+}
+
+export async function getDb(): Promise<Client> {
+  if (activeDbClient) {
+    return activeDbClient;
+  }
+
+  if (!cachedClient) {
+    cachedClient = createDbClient();
+    await initializeLocalClient(cachedClient, dbPath());
+  }
+  return cachedClient;
+}
+
+async function openDbClient(
+  path = dbPath(),
+  allowRecovery = true,
+): Promise<Client> {
+  try {
+    const client = createDbClient(path);
+    try {
+      await prepareEmbeddedReplicaClient(client, path);
+      return client;
+    } catch (error) {
+      client.close();
+      throw error;
+    }
+  } catch (error) {
+    if (
+      !allowRecovery ||
+      !tursoConfigured() ||
+      !isReplicaConflictError(error)
+    ) {
+      throw error;
+    }
+
+    recoverEmbeddedReplica(path);
+    return openDbClient(path, false);
+  }
+}
+
+async function prepareEmbeddedReplicaClient(
+  client: Client,
+  path: string,
+  syncFirst = true,
+): Promise<void> {
+  if (syncFirst) {
+    await client.sync();
+  }
+  await initializeLocalClient(client, path, { embeddedReplica: true });
+}
+
+async function initializeLocalClient(
+  client: Client,
+  path: string,
+  options: { embeddedReplica?: boolean } = {},
+): Promise<void> {
+  await migrate(client);
+
+  if (!options.embeddedReplica) {
+    await client.execute("PRAGMA journal_mode = WAL");
+  }
+
+  await importPendingLocalBackup(client, path);
+}
+
+export async function migrate(client: Client): Promise<void> {
+  const schema = readSchemaSql();
+  const schemaChecksum = checksum(schema);
+
+  await client.execute(
+    "CREATE TABLE IF NOT EXISTS schema_migrations (version INTEGER PRIMARY KEY, name TEXT NOT NULL, checksum TEXT NOT NULL, applied_at INTEGER NOT NULL)",
+  );
+
+  const existing = await client.execute({
+    sql: "SELECT version, checksum FROM schema_migrations WHERE version = ?",
+    args: [MIGRATION_VERSION],
+  });
+
+  if (existing.rows.length > 0) {
+    const row = existing.rows[0] as { checksum?: string };
+    if (row.checksum && row.checksum !== schemaChecksum) {
+      throw new Error(
+        `schema migration ${MIGRATION_VERSION} checksum mismatch`,
+      );
+    }
+    return;
+  }
+
+  const statements = splitSqlStatements(schema);
+
+  for (const statement of statements) {
+    await client.execute(statement);
+  }
+
+  await client.execute({
+    sql: "INSERT INTO schema_migrations(version, name, checksum, applied_at) VALUES (?, ?, ?, ?)",
+    args: [MIGRATION_VERSION, MIGRATION_NAME, schemaChecksum, Date.now()],
+  });
+}
+
+export async function doctorDb(): Promise<{
+  ok: boolean;
+  path: string;
+  tables: string[];
+  replica: {
+    mode: "local-only" | "embedded-replica" | "missing";
+    pendingBackup: string | null;
+    importedBackup: string | null;
+  };
+}> {
+  const path = dbPath();
+  return withDb(async (client) => {
+    const tables = await client.execute(
+      "SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name",
+    );
+    const replicaInfoExists = existsSync(`${path}-info`);
+    return {
+      ok: existsSync(path),
+      path,
+      tables: tables.rows.map((row) => String(row.name)),
+      replica: {
+        mode:
+          !existsSync(path) ? "missing"
+          : replicaInfoExists ? "embedded-replica"
+          : "local-only",
+        pendingBackup: findPendingBackup(path),
+        importedBackup:
+          existsSync(importedBackupDir(path)) ? importedBackupDir(path) : null,
+      },
+    };
+  });
+}
+
+export function resetDbCache(): void {
+  cachedClient = null;
+}
+
+function splitSqlStatements(sql: string): string[] {
+  const cleaned = sql
+    .split("\n")
+    .filter((line) => !line.trim().startsWith("--"))
+    .join("\n");
+
+  const statements: string[] = [];
+  let current = "";
+  let depth = 0;
+
+  for (let index = 0; index < cleaned.length; index += 1) {
+    const rest = cleaned.slice(index);
+    const beginMatch = rest.match(/^BEGIN\b/i);
+    if (beginMatch) {
+      depth += 1;
+      current += beginMatch[0];
+      index += beginMatch[0].length - 1;
+      continue;
+    }
+
+    const endMatch = rest.match(/^END\b/i);
+    if (endMatch) {
+      depth = Math.max(0, depth - 1);
+      current += endMatch[0];
+      index += endMatch[0].length - 1;
+      continue;
+    }
+
+    const char = cleaned[index];
+    if (char === ";" && depth === 0) {
+      const statement = current.trim();
+      if (statement.length > 0) {
+        statements.push(statement);
+      }
+      current = "";
+      continue;
+    }
+
+    current += char;
+  }
+
+  const tail = current.trim();
+  if (tail.length > 0) {
+    statements.push(tail);
+  }
+
+  return statements;
+}