@nothumanwork/nn 0.1.0

package/src/providers/codex/index.ts

@@ -0,0 +1,264 @@
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { basename, join } from "node:path";
+
+import { homeDir, projectDir } from "../../config/paths.ts";
+import { lineKeyFor, parseCompleteJsonlLines } from "../../ingest/jsonl.ts";
+import type {
+  NormalizedEvent,
+  SourceImportBatch,
+  SourceSessionRef,
+} from "../../ir/types.ts";
+import { defaultRole } from "../../ir/types.ts";
+import type { ProviderAdapter } from "../types.ts";
+import { stringField, textFromUnknown, truncate } from "../types.ts";
+
+export const PARSER_VERSION = "codex-rollout-v1";
+
+function codexRoot(): string {
+  return join(homeDir(), ".codex", "sessions");
+}
+
+function discoverCodexSessions(): SourceSessionRef[] {
+  if (!existsSync(codexRoot())) {
+    return [];
+  }
+  const sessions: SourceSessionRef[] = [];
+  const walk = (dir: string): void => {
+    for (const entry of readdirSync(dir, { withFileTypes: true })) {
+      const path = join(dir, entry.name);
+      if (entry.isDirectory()) {
+        walk(path);
+        continue;
+      }
+      if (
+        entry.isFile() &&
+        entry.name.startsWith("rollout-") &&
+        entry.name.endsWith(".jsonl")
+      ) {
+        sessions.push({
+          source: "codex",
+          sessionId: basename(entry.name, ".jsonl").replace(/^rollout-/, ""),
+          path,
+          workspaceRoot: projectDir(),
+        });
+      }
+    }
+  };
+  walk(codexRoot());
+  return sessions;
+}
+
+function parseCodexEnvelope(
+  value: Record<string, unknown>,
+  sessionId: string,
+  timestamp: string,
+  lineKey: string,
+  rawJson: string,
+  rawLocalRef: string,
+  rowEventId: string,
+): NormalizedEvent[] {
+  const type = stringField(value, "type") ?? "unknown";
+  const payload = (value.payload ?? value) as Record<string, unknown>;
+
+  if (type === "session_meta") {
+    return [
+      {
+        source: "codex",
+        sourceSessionId: stringField(payload, "id") ?? sessionId,
+        sourceEventId: rowEventId,
+        lineKey,
+        boundary: "session_start",
+        role: "system",
+        timestamp,
+        contentText: "Codex session started",
+        parserVersion: PARSER_VERSION,
+        rawLocalRef,
+        rawJson,
+      },
+    ];
+  }
+
+  if (type === "event_msg" && stringField(payload, "type") === "user_message") {
+    const text =
+      stringField(payload, "message") ?? textFromUnknown(payload) ?? "";
+    return [
+      {
+        source: "codex",
+        sourceSessionId: sessionId,
+        sourceEventId: rowEventId,
+        lineKey,
+        boundary: "user_turn",
+        role: "user",
+        timestamp,
+        contentText: text,
+        parserVersion: PARSER_VERSION,
+        rawLocalRef,
+        rawJson,
+      },
+    ];
+  }
+
+  if (type === "response_item") {
+    const itemType = stringField(payload, "type") ?? "";
+    if (itemType === "function_call") {
+      const name = stringField(payload, "name") ?? "tool";
+      return [
+        {
+          source: "codex",
+          sourceSessionId: sessionId,
+          sourceEventId: rowEventId,
+          lineKey,
+          boundary: "tool_call",
+          role: "assistant",
+          timestamp,
+          contentText: truncate(
+            `${name} ${stringField(payload, "arguments") ?? "{}"}`,
+          ),
+          toolName: name,
+          toolCallId: stringField(payload, "call_id"),
+          toolInputJson: stringField(payload, "arguments") ?? "{}",
+          parserVersion: PARSER_VERSION,
+          rawLocalRef,
+          rawJson,
+        },
+      ];
+    }
+    if (itemType === "function_call_output") {
+      return [
+        {
+          source: "codex",
+          sourceSessionId: sessionId,
+          sourceEventId: rowEventId,
+          lineKey,
+          boundary: "tool_result",
+          role: "tool",
+          timestamp,
+          contentText: truncate(
+            textFromUnknown(payload.output ?? payload) ?? "",
+          ),
+          toolCallId: stringField(payload, "call_id"),
+          toolOutputJson: JSON.stringify(payload.output ?? payload),
+          parserVersion: PARSER_VERSION,
+          rawLocalRef,
+          rawJson,
+        },
+      ];
+    }
+    if (stringField(payload, "role") === "assistant") {
+      const text = textFromUnknown(payload.content) ?? "";
+      if (!text.trim()) {
+        return [];
+      }
+      return [
+        {
+          source: "codex",
+          sourceSessionId: sessionId,
+          sourceEventId: rowEventId,
+          lineKey,
+          boundary: "assistant_turn",
+          role: "assistant",
+          timestamp,
+          contentText: text,
+          parserVersion: PARSER_VERSION,
+          rawLocalRef,
+          rawJson,
+        },
+      ];
+    }
+  }
+
+  return [
+    {
+      source: "codex",
+      sourceSessionId: sessionId,
+      sourceEventId: rowEventId,
+      lineKey,
+      boundary: "unknown_raw_event",
+      role: defaultRole("unknown_raw_event"),
+      timestamp,
+      contentText: truncate(JSON.stringify(value)),
+      parserVersion: PARSER_VERSION,
+      rawLocalRef,
+      rawJson,
+    },
+  ];
+}
+
+function parseCodexFile(
+  sourcePath: string,
+  sessionId: string,
+  offset: number,
+): SourceImportBatch {
+  const buffer = readFileSync(sourcePath);
+  const slice = buffer.subarray(offset);
+  const { lines, consumedBytes, partialTail } = parseCompleteJsonlLines(
+    slice,
+    0,
+  );
+  const events: NormalizedEvent[] = [];
+  const warnings: string[] = [];
+  let currentSessionId = sessionId;
+  let emittedLines = 0;
+
+  for (const line of lines) {
+    emittedLines = line.lineIndex;
+    const absoluteStart = offset + line.lineStart;
+    const lineKey = lineKeyFor(sourcePath, absoluteStart);
+    let value: Record<string, unknown>;
+    try {
+      value = JSON.parse(line.raw) as Record<string, unknown>;
+    } catch (error) {
+      warnings.push(
+        `${sourcePath}:${line.lineIndex} malformed row: ${String(error)}`,
+      );
+      continue;
+    }
+
+    const timestamp =
+      stringField(value, "timestamp") ?? new Date().toISOString();
+    const rowEventId = `line:${line.lineIndex}`;
+    const parsed = parseCodexEnvelope(
+      value,
+      currentSessionId,
+      timestamp,
+      lineKey,
+      line.raw,
+      `${sourcePath}:${line.lineIndex}`,
+      rowEventId,
+    );
+    for (const event of parsed) {
+      currentSessionId = event.sourceSessionId;
+      events.push(event);
+    }
+  }
+
+  const absoluteConsumed =
+    partialTail ?
+      offset + (lines.at(-1)?.lineEnd ?? 0)
+    : offset + consumedBytes;
+
+  return {
+    source: "codex",
+    parserVersion: PARSER_VERSION,
+    session: {
+      source: "codex",
+      sessionId: currentSessionId,
+      path: sourcePath,
+      workspaceRoot: projectDir(),
+    },
+    events,
+    cursorKey: sourcePath,
+    consumedBytes: absoluteConsumed,
+    emittedLines,
+    warnings,
+  };
+}
+
+export const codexProvider: ProviderAdapter = {
+  name: "codex",
+  parserVersion: PARSER_VERSION,
+  discover: discoverCodexSessions,
+  parseIncremental: parseCodexFile,
+  parseFull: (sourcePath, sessionId) =>
+    parseCodexFile(sourcePath, sessionId, 0),
+};

package/src/providers/copilot/index.ts

@@ -0,0 +1,24 @@
+import type { SourceImportBatch, SourceSessionRef } from "../../ir/types.ts";
+import type { ProviderAdapter } from "../types.ts";
+
+export const PARSER_VERSION = "copilot-unimplemented-v0";
+
+/**
+ * GitHub Copilot Chat stores history in IDE extension globalStorage (often SQLite/state.vscdb),
+ * not plain JSONL. No stable cross-harness export path was found on this machine.
+ *
+ * See docs/providers/copilot.md for the spike outcome and future integration options.
+ */
+function notImplemented(): never {
+  throw new Error(
+    "GitHub Copilot transcript ingest is not implemented yet. Copilot stores chat history in IDE globalStorage, not JSONL. See docs/providers/copilot.md.",
+  );
+}
+
+export const copilotProvider: ProviderAdapter = {
+  name: "copilot",
+  parserVersion: PARSER_VERSION,
+  discover: (): SourceSessionRef[] => [],
+  parseIncremental: (): SourceImportBatch => notImplemented(),
+  parseFull: (): SourceImportBatch => notImplemented(),
+};

package/src/providers/cursor/index.ts

@@ -0,0 +1,340 @@
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { basename, join } from "node:path";
+
+import { homeDir, encodeProjectSlug, projectDir } from "../../config/paths.ts";
+import { lineKeyFor, parseCompleteJsonlLines } from "../../ingest/jsonl.ts";
+import type {
+  EventBoundary,
+  NormalizedEvent,
+  SourceImportBatch,
+  SourceSessionRef,
+} from "../../ir/types.ts";
+import { defaultRole } from "../../ir/types.ts";
+import type { ProviderAdapter } from "../types.ts";
+import { stringField, textFromUnknown, truncate } from "../types.ts";
+
+export const PARSER_VERSION = "cursor-agent-jsonl-v1";
+
+function cursorRoot(): string {
+  return join(homeDir(), ".cursor");
+}
+
+function decodeCursorProject(alias: string): string | null {
+  if (!alias) {
+    return "/";
+  }
+  return `/${alias.replace(/-/g, "/")}`;
+}
+
+function sessionMatchesProject(
+  sessionPath: string,
+  workspaceRoot?: string,
+): boolean {
+  const project = projectDir();
+  const slug = encodeProjectSlug(project);
+  if (workspaceRoot && workspaceRoot === project) {
+    return true;
+  }
+  if (sessionPath.includes(`/projects/${slug}/`)) {
+    return true;
+  }
+  return false;
+}
+
+function discoverCursorSessions(): SourceSessionRef[] {
+  const projectsDir = join(cursorRoot(), "projects");
+  if (!existsSync(projectsDir)) {
+    return [];
+  }
+
+  const sessions: SourceSessionRef[] = [];
+  for (const projectEntry of readdirSync(projectsDir, {
+    withFileTypes: true,
+  })) {
+    if (!projectEntry.isDirectory()) {
+      continue;
+    }
+    const transcriptsDir = join(
+      projectsDir,
+      projectEntry.name,
+      "agent-transcripts",
+    );
+    if (!existsSync(transcriptsDir)) {
+      continue;
+    }
+    for (const sessionEntry of readdirSync(transcriptsDir, {
+      withFileTypes: true,
+    })) {
+      if (!sessionEntry.isDirectory()) {
+        continue;
+      }
+      const jsonlPath = join(
+        transcriptsDir,
+        sessionEntry.name,
+        `${sessionEntry.name}.jsonl`,
+      );
+      if (!existsSync(jsonlPath)) {
+        continue;
+      }
+      const workspaceRoot = decodeCursorProject(projectEntry.name) ?? undefined;
+      if (!sessionMatchesProject(jsonlPath, workspaceRoot)) {
+        continue;
+      }
+      sessions.push({
+        source: "cursor",
+        sessionId: sessionEntry.name,
+        path: jsonlPath,
+        workspaceRoot,
+      });
+    }
+  }
+  return sessions;
+}
+
+interface CursorPart {
+  boundary: EventBoundary;
+  role: string;
+  contentText: string;
+  toolName?: string;
+  toolCallId?: string;
+  toolInputJson?: string;
+  toolOutputJson?: string;
+  suffix?: string;
+}
+
+function cursorRoleBoundary(role: string): EventBoundary {
+  if (role === "user") {
+    return "user_turn";
+  }
+  if (role === "assistant") {
+    return "assistant_turn";
+  }
+  return "unknown_raw_event";
+}
+
+function cursorContentPart(
+  item: Record<string, unknown>,
+  rowRole: string,
+  index: number,
+): CursorPart {
+  const blockType = stringField(item, "type") ?? "text";
+  const suffix = `block:${index}`;
+  if (
+    blockType === "tool_use" ||
+    blockType === "tool_call" ||
+    blockType === "function_call"
+  ) {
+    const name = stringField(item, "name") ?? "tool";
+    const input = item.input ?? item.arguments ?? item;
+    return {
+      boundary: "tool_call",
+      role: "assistant",
+      contentText: truncate(`${name} ${JSON.stringify(input)}`),
+      toolName: name,
+      toolCallId: stringField(item, "id"),
+      toolInputJson: JSON.stringify(input),
+      suffix,
+    };
+  }
+  if (blockType === "tool_result") {
+    const output =
+      textFromUnknown(item.content ?? item.output ?? item) ??
+      JSON.stringify(item);
+    return {
+      boundary: "tool_result",
+      role: "tool",
+      contentText: truncate(output),
+      toolCallId:
+        stringField(item, "tool_use_id") ?? stringField(item, "call_id"),
+      toolOutputJson: JSON.stringify(item),
+      suffix,
+    };
+  }
+  const text = textFromUnknown(item) ?? "";
+  return {
+    boundary: cursorRoleBoundary(rowRole),
+    role: rowRole,
+    contentText: text,
+    suffix,
+  };
+}
+
+function cursorEventParts(value: Record<string, unknown>): CursorPart[] {
+  const rowType = stringField(value, "type");
+  if (rowType === "session_start") {
+    return [
+      {
+        boundary: "session_start",
+        role: "system",
+        contentText: "Cursor session started",
+      },
+    ];
+  }
+  if (rowType === "session_end") {
+    return [
+      {
+        boundary: "session_end",
+        role: "system",
+        contentText: "Cursor session ended",
+      },
+    ];
+  }
+  if (rowType === "compaction" || rowType === "summary") {
+    return [
+      {
+        boundary: "compaction",
+        role: "system",
+        contentText:
+          textFromUnknown(value.content ?? value.message) ??
+          `Cursor ${rowType}`,
+      },
+    ];
+  }
+
+  const role =
+    stringField(value, "role") ?? stringField(value, "type") ?? "unknown";
+  const message = value.message;
+  const content =
+    typeof message === "object" && message !== null ?
+      (message as Record<string, unknown>).content
+    : value.content;
+
+  if (Array.isArray(content)) {
+    return content.map((item, index) =>
+      cursorContentPart(item as Record<string, unknown>, role, index),
+    );
+  }
+
+  const text = textFromUnknown(content);
+  if (text) {
+    return [{ boundary: cursorRoleBoundary(role), role, contentText: text }];
+  }
+
+  return [
+    {
+      boundary: "unknown_raw_event",
+      role: "system",
+      contentText: truncate(JSON.stringify(value)),
+    },
+  ];
+}
+
+function parseCursorBuffer(
+  sourcePath: string,
+  sessionId: string,
+  buffer: Buffer,
+  startOffset: number,
+): SourceImportBatch {
+  const { lines, consumedBytes, partialTail } = parseCompleteJsonlLines(
+    buffer,
+    0,
+  );
+  const events: NormalizedEvent[] = [];
+  const warnings: string[] = [];
+  let currentSessionId = sessionId;
+  let emittedLines = 0;
+
+  for (const line of lines) {
+    emittedLines = line.lineIndex;
+    const absoluteStart = startOffset + line.lineStart;
+    const lineKey = lineKeyFor(sourcePath, absoluteStart);
+    let value: Record<string, unknown>;
+    try {
+      value = JSON.parse(line.raw) as Record<string, unknown>;
+    } catch (error) {
+      warnings.push(
+        `${sourcePath}:${line.lineIndex} malformed row: ${String(error)}`,
+      );
+      continue;
+    }
+
+    const discoveredSession =
+      stringField(value, "sessionId") ?? stringField(value, "session_id");
+    if (discoveredSession) {
+      currentSessionId = discoveredSession;
+    }
+    const timestamp =
+      stringField(value, "timestamp") ?? new Date().toISOString();
+    const rowEventId =
+      stringField(value, "uuid") ??
+      stringField(value, "id") ??
+      `line:${line.lineIndex}`;
+
+    for (const part of cursorEventParts(value)) {
+      if (!part.contentText.trim()) {
+        continue;
+      }
+      events.push({
+        source: "cursor",
+        sourceSessionId: currentSessionId,
+        sourceEventId:
+          part.suffix ? `${rowEventId}:${part.suffix}` : rowEventId,
+        lineKey: part.suffix ? `${lineKey}:${part.suffix}` : lineKey,
+        boundary: part.boundary,
+        role: part.role || defaultRole(part.boundary),
+        timestamp,
+        contentText: part.contentText,
+        toolName: part.toolName,
+        toolCallId: part.toolCallId,
+        toolInputJson: part.toolInputJson,
+        toolOutputJson: part.toolOutputJson,
+        parserVersion: PARSER_VERSION,
+        rawLocalRef: `${sourcePath}:${line.lineIndex}`,
+        rawJson: line.raw,
+      });
+    }
+  }
+
+  const absoluteConsumed =
+    partialTail ?
+      startOffset + (lines.at(-1)?.lineEnd ?? 0)
+    : startOffset + consumedBytes;
+
+  return {
+    source: "cursor",
+    parserVersion: PARSER_VERSION,
+    session: {
+      source: "cursor",
+      sessionId: currentSessionId,
+      path: sourcePath,
+      workspaceRoot: projectDir(),
+    },
+    events,
+    cursorKey: sourcePath,
+    consumedBytes: absoluteConsumed,
+    emittedLines,
+    warnings,
+  };
+}
+
+function parseCursorFile(
+  sourcePath: string,
+  sessionId: string,
+  offset: number,
+): SourceImportBatch {
+  const buffer = readFileSync(sourcePath);
+  return parseCursorBuffer(
+    sourcePath,
+    sessionId,
+    buffer.subarray(offset),
+    offset,
+  );
+}
+
+export const cursorProvider: ProviderAdapter = {
+  name: "cursor",
+  parserVersion: PARSER_VERSION,
+  discover: discoverCursorSessions,
+  parseIncremental: parseCursorFile,
+  parseFull: (sourcePath, sessionId) =>
+    parseCursorFile(sourcePath, sessionId, 0),
+};
+
+export function cursorSessionIdFromPath(sourcePath: string): string {
+  const match = sourcePath.match(/agent-transcripts\/([^/]+)\//);
+  return match?.[1] ?? basename(sourcePath, ".jsonl");
+}
+
+export function isTranscriptFile(path: string): boolean {
+  return existsSync(path) && statSync(path).isFile() && path.endsWith(".jsonl");
+}