@nothumanwork/nn 0.1.0

package/src/providers/grok/index.ts

@@ -0,0 +1,146 @@
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { homeDir, projectDir } from "../../config/paths.ts";
+import { lineKeyFor, parseCompleteJsonlLines } from "../../ingest/jsonl.ts";
+import type {
+  EventBoundary,
+  NormalizedEvent,
+  SourceImportBatch,
+  SourceSessionRef,
+} from "../../ir/types.ts";
+import { defaultRole } from "../../ir/types.ts";
+import type { ProviderAdapter } from "../types.ts";
+import { stringField, textFromUnknown, truncate } from "../types.ts";
+
+export const PARSER_VERSION = "grok-updates-jsonl-v1";
+
+function grokRoot(): string {
+  return join(homeDir(), ".grok", "sessions");
+}
+
+function discoverGrokSessions(): SourceSessionRef[] {
+  if (!existsSync(grokRoot())) {
+    return [];
+  }
+  const sessions: SourceSessionRef[] = [];
+  for (const cwdEntry of readdirSync(grokRoot(), { withFileTypes: true })) {
+    if (!cwdEntry.isDirectory()) {
+      continue;
+    }
+    const cwdPath = join(grokRoot(), cwdEntry.name);
+    for (const sessionEntry of readdirSync(cwdPath, { withFileTypes: true })) {
+      if (!sessionEntry.isDirectory()) {
+        continue;
+      }
+      const updatesPath = join(cwdPath, sessionEntry.name, "updates.jsonl");
+      if (!existsSync(updatesPath)) {
+        continue;
+      }
+      sessions.push({
+        source: "grok",
+        sessionId: sessionEntry.name,
+        path: updatesPath,
+        workspaceRoot: projectDir(),
+      });
+    }
+  }
+  return sessions;
+}
+
+function boundaryForGrokUpdate(updateType: string): EventBoundary {
+  if (updateType === "user_message_chunk") {
+    return "user_turn";
+  }
+  if (updateType === "agent_message_chunk") {
+    return "assistant_turn";
+  }
+  return "unknown_raw_event";
+}
+
+function parseGrokFile(
+  sourcePath: string,
+  sessionId: string,
+  offset: number,
+): SourceImportBatch {
+  const buffer = readFileSync(sourcePath);
+  const slice = buffer.subarray(offset);
+  const { lines, consumedBytes, partialTail } = parseCompleteJsonlLines(
+    slice,
+    0,
+  );
+  const events: NormalizedEvent[] = [];
+  const warnings: string[] = [];
+  let emittedLines = 0;
+
+  for (const line of lines) {
+    emittedLines = line.lineIndex;
+    const absoluteStart = offset + line.lineStart;
+    const lineKey = lineKeyFor(sourcePath, absoluteStart);
+    let value: Record<string, unknown>;
+    try {
+      value = JSON.parse(line.raw) as Record<string, unknown>;
+    } catch (error) {
+      warnings.push(
+        `${sourcePath}:${line.lineIndex} malformed row: ${String(error)}`,
+      );
+      continue;
+    }
+
+    const params = value.params as Record<string, unknown> | undefined;
+    const update = params?.update as Record<string, unknown> | undefined;
+    const updateType = stringField(update, "sessionUpdate") ?? "unknown";
+    const text = textFromUnknown(update?.content ?? update) ?? "";
+    if (!text.trim()) {
+      continue;
+    }
+    const boundary = boundaryForGrokUpdate(updateType);
+    events.push({
+      source: "grok",
+      sourceSessionId: stringField(params, "sessionId") ?? sessionId,
+      sourceEventId: `line:${line.lineIndex}`,
+      lineKey,
+      boundary,
+      role:
+        boundary === "user_turn" ? "user"
+        : boundary === "assistant_turn" ? "assistant"
+        : defaultRole(boundary),
+      timestamp: new Date(
+        Number(value.timestamp ?? Date.now()) * 1000,
+      ).toISOString(),
+      contentText: truncate(text),
+      parserVersion: PARSER_VERSION,
+      rawLocalRef: `${sourcePath}:${line.lineIndex}`,
+      rawJson: line.raw,
+    });
+  }
+
+  const absoluteConsumed =
+    partialTail ?
+      offset + (lines.at(-1)?.lineEnd ?? 0)
+    : offset + consumedBytes;
+
+  return {
+    source: "grok",
+    parserVersion: PARSER_VERSION,
+    session: {
+      source: "grok",
+      sessionId,
+      path: sourcePath,
+      workspaceRoot: projectDir(),
+    },
+    events,
+    cursorKey: sourcePath,
+    consumedBytes: absoluteConsumed,
+    emittedLines,
+    warnings,
+  };
+}
+
+export const grokProvider: ProviderAdapter = {
+  name: "grok",
+  parserVersion: PARSER_VERSION,
+  discover: discoverGrokSessions,
+  parseIncremental: parseGrokFile,
+  parseFull: (sourcePath, sessionId) => parseGrokFile(sourcePath, sessionId, 0),
+};

package/src/providers/pi/index.ts

@@ -0,0 +1,197 @@
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { basename, join } from "node:path";
+
+import { homeDir, projectDir } from "../../config/paths.ts";
+import { lineKeyFor, parseCompleteJsonlLines } from "../../ingest/jsonl.ts";
+import type {
+  EventBoundary,
+  NormalizedEvent,
+  SourceImportBatch,
+  SourceSessionRef,
+} from "../../ir/types.ts";
+import type { ProviderAdapter } from "../types.ts";
+import { stringField, textFromUnknown, truncate } from "../types.ts";
+
+export const PARSER_VERSION = "pi-session-jsonl-v3";
+
+function piRoot(): string {
+  return join(homeDir(), ".pi", "agent", "sessions");
+}
+
+function discoverPiSessions(): SourceSessionRef[] {
+  if (!existsSync(piRoot())) {
+    return [];
+  }
+  const sessions: SourceSessionRef[] = [];
+  const walk = (dir: string): void => {
+    for (const entry of readdirSync(dir, { withFileTypes: true })) {
+      const path = join(dir, entry.name);
+      if (entry.isDirectory()) {
+        walk(path);
+        continue;
+      }
+      if (entry.isFile() && entry.name.endsWith(".jsonl")) {
+        const parts = entry.name.replace(".jsonl", "").split("_");
+        const sessionId = parts.at(-1) ?? basename(entry.name, ".jsonl");
+        sessions.push({
+          source: "pi",
+          sessionId,
+          path,
+          workspaceRoot: projectDir(),
+        });
+      }
+    }
+  };
+  walk(piRoot());
+  return sessions;
+}
+
+function boundaryForPiMessage(role: string, blockType?: string): EventBoundary {
+  if (role === "user") {
+    return "user_turn";
+  }
+  if (role === "assistant") {
+    return blockType === "toolCall" ? "tool_call" : "assistant_turn";
+  }
+  if (role === "toolResult") {
+    return "tool_result";
+  }
+  return "unknown_raw_event";
+}
+
+function parsePiFile(
+  sourcePath: string,
+  sessionId: string,
+  offset: number,
+): SourceImportBatch {
+  const buffer = readFileSync(sourcePath);
+  const slice = buffer.subarray(offset);
+  const { lines, consumedBytes, partialTail } = parseCompleteJsonlLines(
+    slice,
+    0,
+  );
+  const events: NormalizedEvent[] = [];
+  const warnings: string[] = [];
+  let currentSessionId = sessionId;
+  let emittedLines = 0;
+
+  for (const line of lines) {
+    emittedLines = line.lineIndex;
+    const absoluteStart = offset + line.lineStart;
+    const lineKey = lineKeyFor(sourcePath, absoluteStart);
+    let value: Record<string, unknown>;
+    try {
+      value = JSON.parse(line.raw) as Record<string, unknown>;
+    } catch (error) {
+      warnings.push(
+        `${sourcePath}:${line.lineIndex} malformed row: ${String(error)}`,
+      );
+      continue;
+    }
+
+    const rowType = stringField(value, "type") ?? "unknown";
+    if (rowType === "session") {
+      currentSessionId = stringField(value, "id") ?? currentSessionId;
+      events.push({
+        source: "pi",
+        sourceSessionId: currentSessionId,
+        sourceEventId: currentSessionId,
+        lineKey,
+        boundary: "session_start",
+        role: "system",
+        timestamp: stringField(value, "timestamp") ?? new Date().toISOString(),
+        contentText: "Pi session started",
+        parserVersion: PARSER_VERSION,
+        rawLocalRef: `${sourcePath}:${line.lineIndex}`,
+        rawJson: line.raw,
+      });
+      continue;
+    }
+
+    if (rowType !== "message") {
+      continue;
+    }
+
+    const message = value.message as Record<string, unknown> | undefined;
+    const role = stringField(message, "role") ?? "unknown";
+    const timestamp =
+      stringField(value, "timestamp") ?? new Date().toISOString();
+    const rowEventId = stringField(value, "id") ?? `line:${line.lineIndex}`;
+    const content = message?.content;
+
+    if (Array.isArray(content)) {
+      for (const [index, block] of content.entries()) {
+        const item = block as Record<string, unknown>;
+        const blockType = stringField(item, "type");
+        const text = textFromUnknown(item) ?? "";
+        events.push({
+          source: "pi",
+          sourceSessionId: currentSessionId,
+          sourceEventId: `${rowEventId}:block:${index}`,
+          lineKey: `${lineKey}:block:${index}`,
+          boundary: boundaryForPiMessage(role, blockType),
+          role: role === "toolResult" ? "tool" : role,
+          timestamp,
+          contentText: truncate(text || JSON.stringify(item)),
+          toolName: stringField(item, "name"),
+          toolCallId:
+            stringField(item, "id") ?? stringField(message, "toolCallId"),
+          toolInputJson:
+            blockType === "toolCall" ?
+              JSON.stringify(item.arguments ?? {})
+            : undefined,
+          toolOutputJson:
+            role === "toolResult" ? JSON.stringify(item) : undefined,
+          parserVersion: PARSER_VERSION,
+          rawLocalRef: `${sourcePath}:${line.lineIndex}`,
+          rawJson: line.raw,
+        });
+      }
+      continue;
+    }
+
+    const text = textFromUnknown(message) ?? truncate(JSON.stringify(value));
+    events.push({
+      source: "pi",
+      sourceSessionId: currentSessionId,
+      sourceEventId: rowEventId,
+      lineKey,
+      boundary: boundaryForPiMessage(role),
+      role: role === "toolResult" ? "tool" : role,
+      timestamp,
+      contentText: text,
+      parserVersion: PARSER_VERSION,
+      rawLocalRef: `${sourcePath}:${line.lineIndex}`,
+      rawJson: line.raw,
+    });
+  }
+
+  const absoluteConsumed =
+    partialTail ?
+      offset + (lines.at(-1)?.lineEnd ?? 0)
+    : offset + consumedBytes;
+
+  return {
+    source: "pi",
+    parserVersion: PARSER_VERSION,
+    session: {
+      source: "pi",
+      sessionId: currentSessionId,
+      path: sourcePath,
+      workspaceRoot: projectDir(),
+    },
+    events,
+    cursorKey: sourcePath,
+    consumedBytes: absoluteConsumed,
+    emittedLines,
+    warnings,
+  };
+}
+
+export const piProvider: ProviderAdapter = {
+  name: "pi",
+  parserVersion: PARSER_VERSION,
+  discover: discoverPiSessions,
+  parseIncremental: parsePiFile,
+  parseFull: (sourcePath, sessionId) => parsePiFile(sourcePath, sessionId, 0),
+};

package/src/providers/registry.ts

@@ -0,0 +1,31 @@
+import { cursorProvider } from "./cursor/index.ts";
+import { claudeProvider } from "./claude/index.ts";
+import { codexProvider } from "./codex/index.ts";
+import { copilotProvider } from "./copilot/index.ts";
+import { antigravityProvider } from "./antigravity/index.ts";
+import { piProvider } from "./pi/index.ts";
+import { grokProvider } from "./grok/index.ts";
+import type { ProviderAdapter } from "./types.ts";
+import type { ProviderId } from "../ir/types.ts";
+
+const providers: Record<ProviderId, ProviderAdapter> = {
+  cursor: cursorProvider,
+  claude: claudeProvider,
+  codex: codexProvider,
+  copilot: copilotProvider,
+  antigravity: antigravityProvider,
+  pi: piProvider,
+  grok: grokProvider,
+};
+
+export function getProvider(id: ProviderId): ProviderAdapter {
+  const provider = providers[id];
+  if (!provider) {
+    throw new Error(`Unknown provider: ${id}`);
+  }
+  return provider;
+}
+
+export function listProviders(): ProviderId[] {
+  return Object.keys(providers) as ProviderId[];
+}

package/src/providers/types.ts

@@ -0,0 +1,53 @@
+import type { SourceImportBatch, SourceSessionRef } from "../ir/types.ts";
+
+export interface ProviderAdapter {
+  name: string;
+  parserVersion: string;
+  discover(): SourceSessionRef[];
+  parseIncremental(
+    sourcePath: string,
+    sessionId: string,
+    offset: number,
+  ): SourceImportBatch;
+  parseFull(sourcePath: string, sessionId: string): SourceImportBatch;
+}
+
+export function stringField(value: unknown, key: string): string | undefined {
+  if (typeof value !== "object" || value === null) {
+    return undefined;
+  }
+  const field = (value as Record<string, unknown>)[key];
+  return typeof field === "string" ? field : undefined;
+}
+
+export function textFromUnknown(value: unknown): string | undefined {
+  if (typeof value === "string") {
+    return value;
+  }
+  if (Array.isArray(value)) {
+    const parts = value
+      .map((item) => textFromUnknown(item))
+      .filter((part): part is string => Boolean(part));
+    return parts.length > 0 ? parts.join("\n") : undefined;
+  }
+  if (typeof value === "object" && value !== null) {
+    const record = value as Record<string, unknown>;
+    if (typeof record.text === "string") {
+      return record.text;
+    }
+    if (typeof record.content === "string") {
+      return record.content;
+    }
+    if (Array.isArray(record.content)) {
+      return textFromUnknown(record.content);
+    }
+  }
+  return undefined;
+}
+
+export function truncate(text: string, max = 2000): string {
+  if (text.length <= max) {
+    return text;
+  }
+  return `${text.slice(0, max)}… [truncated ${text.length - max} chars]`;
+}

package/src/sync/coordinator.ts

@@ -0,0 +1,186 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+
+import { coordinatorStatePath, syncConfig } from "../config/sync.ts";
+import { coordinatorSpawn } from "../config/runtime.ts";
+import { withDb } from "../db/migrate.ts";
+
+export interface CoordinatorState {
+  dirty: boolean;
+  lastIngestAt: number;
+  flushRequestedAt: number | null;
+  lastSyncAt: number | null;
+  workerPid: number | null;
+}
+
+const DEFAULT_STATE: CoordinatorState = {
+  dirty: false,
+  lastIngestAt: 0,
+  flushRequestedAt: null,
+  lastSyncAt: null,
+  workerPid: null,
+};
+
+function readState(): CoordinatorState {
+  const path = coordinatorStatePath();
+  if (!existsSync(path)) {
+    return { ...DEFAULT_STATE };
+  }
+  try {
+    return { ...DEFAULT_STATE, ...JSON.parse(readFileSync(path, "utf-8")) };
+  } catch {
+    return { ...DEFAULT_STATE };
+  }
+}
+
+function writeState(state: CoordinatorState): void {
+  const path = coordinatorStatePath();
+  mkdirSync(dirname(path), { recursive: true });
+  writeFileSync(path, `${JSON.stringify(state, null, 2)}\n`, "utf-8");
+}
+
+async function runSync(): Promise<boolean> {
+  const config = syncConfig();
+  if (!config.enabled) {
+    return false;
+  }
+  try {
+    await withDb(async (client) => {
+      await client.sync();
+    });
+    const state = readState();
+    state.dirty = false;
+    state.lastSyncAt = Date.now();
+    state.flushRequestedAt = null;
+    state.workerPid = null;
+    writeState(state);
+    return true;
+  } catch (error) {
+    console.error("[nn-sync] sync failed", error);
+    return false;
+  }
+}
+
+function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      reject(new Error(`sync timed out after ${timeoutMs}ms`));
+    }, timeoutMs);
+    promise
+      .then((value) => {
+        clearTimeout(timer);
+        resolve(value);
+      })
+      .catch((error) => {
+        clearTimeout(timer);
+        reject(error);
+      });
+  });
+}
+
+export async function flushSync(): Promise<void> {
+  const config = syncConfig();
+  if (!config.enabled) {
+    return;
+  }
+
+  const state = readState();
+  state.flushRequestedAt = Date.now();
+  state.dirty = true;
+  writeState(state);
+
+  try {
+    await withTimeout(runSync(), config.flushTimeoutMs);
+  } catch (error) {
+    console.error("[nn-sync] flush failed", error);
+  }
+}
+
+export async function pullSync(): Promise<void> {
+  await flushSync();
+}
+
+export function scheduleIncrementalSync(): void {
+  const config = syncConfig();
+  if (!config.enabled) {
+    return;
+  }
+
+  const state = readState();
+  state.dirty = true;
+  state.lastIngestAt = Date.now();
+  writeState(state);
+
+  spawnCoordinatorWorker();
+}
+
+function spawnCoordinatorWorker(): void {
+  const state = readState();
+  if (state.workerPid) {
+    try {
+      process.kill(state.workerPid, 0);
+      return;
+    } catch {
+      state.workerPid = null;
+    }
+  }
+
+  const { cmd } = coordinatorSpawn();
+
+  const proc = Bun.spawn(cmd, {
+    env: { ...process.env },
+    stdout: "ignore",
+    stderr: "ignore",
+    stdin: "ignore",
+  });
+
+  state.workerPid = proc.pid;
+  writeState(state);
+  void proc.exited.then(() => {
+    const current = readState();
+    if (current.workerPid === proc.pid) {
+      current.workerPid = null;
+      writeState(current);
+    }
+  });
+}
+
+export function getCoordinatorState(): CoordinatorState {
+  return readState();
+}
+
+export async function runCoordinatorOnce(): Promise<void> {
+  const config = syncConfig();
+  if (!config.enabled) {
+    return;
+  }
+
+  const state = readState();
+  if (!state.dirty) {
+    return;
+  }
+
+  if (state.flushRequestedAt) {
+    await runSync();
+    return;
+  }
+
+  const elapsed = Date.now() - state.lastIngestAt;
+  if (elapsed < config.debounceMs) {
+    await Bun.sleep(config.debounceMs - elapsed);
+    const latest = readState();
+    if (!latest.dirty) {
+      return;
+    }
+    if (latest.flushRequestedAt) {
+      await runSync();
+      return;
+    }
+    const quietFor = Date.now() - latest.lastIngestAt;
+    if (quietFor < config.debounceMs) {
+      return;
+    }
+  }
+
+  await runSync();
+}

package/src/sync/turso.ts

@@ -0,0 +1,64 @@
+import { dbPath } from "../config/paths.ts";
+import { withDb } from "../db/migrate.ts";
+import { syncConfig, tursoConfigured } from "../config/sync.ts";
+import { getCoordinatorState } from "./coordinator.ts";
+
+export interface SyncStatus {
+  configured: boolean;
+  enabled: boolean;
+  localPath: string;
+  syncUrl: string | null;
+  message: string;
+  coordinator: ReturnType<typeof getCoordinatorState>;
+}
+
+export async function syncStatus(): Promise<SyncStatus> {
+  const localPath = dbPath();
+  const configured = tursoConfigured();
+  const config = syncConfig();
+
+  if (!configured) {
+    return {
+      configured: false,
+      enabled: false,
+      localPath,
+      syncUrl: null,
+      message:
+        "Turso sync not configured. Set TURSO_DATABASE_URL and TURSO_AUTH_TOKEN.",
+      coordinator: getCoordinatorState(),
+    };
+  }
+
+  return {
+    configured: true,
+    enabled: config.enabled,
+    localPath,
+    syncUrl: process.env.TURSO_DATABASE_URL ?? null,
+    message: config.enabled ? "Turso sync configured." : "Turso sync disabled.",
+    coordinator: getCoordinatorState(),
+  };
+}
+
+export async function syncPush(): Promise<string> {
+  if (!syncConfig().enabled) {
+    throw new Error(
+      "Turso sync not configured. Set TURSO_DATABASE_URL and TURSO_AUTH_TOKEN.",
+    );
+  }
+  await withDb(async (client) => {
+    await client.sync();
+  });
+  return "Synced local changes with Turso.";
+}
+
+export async function syncPull(): Promise<string> {
+  if (!syncConfig().enabled) {
+    throw new Error(
+      "Turso sync not configured. Set TURSO_DATABASE_URL and TURSO_AUTH_TOKEN.",
+    );
+  }
+  await withDb(async (client) => {
+    await client.sync();
+  });
+  return "Synced remote changes with local database.";
+}

package/src/types/assets.d.ts

@@ -0,0 +1,4 @@
+declare module "*.sql" {
+  const content: string;
+  export default content;
+}