@nessielabs/daemon 0.3.2 → 0.4.0

package/dist/sync/loop.js

@@ -1,64 +1,94 @@
 import { setTimeout as sleep } from "node:timers/promises";
 import { AuthRefreshError } from "../auth/tokens.js";
-import { readConfig, readState, writeState } from "../config/store.js";
+import { getDaemonPaths } from "../config/paths.js";
+import { readConfig } from "../config/store.js";
 import { GitRepoResolver } from "../git/repoResolver.js";
-import { PARSER_VERSION } from "../parser/types.js";
-import { planSyncPushBatches } from "./batch.js";
-import { runPreflightV3, pushSyncPayload } from "./client.js";
-import { mapSessionsToPushItems } from "./map.js";
-import { scanSelectedSources } from "./scan.js";
+import { openDaemonGraphStore } from "../localStore/daemonGraphStore.js";
+import { parserVersionForSourceKind } from "../parser/types.js";
+import { EDGE_PUSH_LIMIT, TARGET_SLICES_PER_PUSH } from "./batch.js";
+import { listSyncExclusions, pullCloudSyncPage, pushSyncPayload, runPreflightV4 } from "./client.js";
+import { filterExcludedConversations } from "./exclusions.js";
+import { fetchConversation, listSelectedSourceConversations } from "./scan.js";
+const PULL_PAGE_LIMIT = 500;
+const PULL_NODE_CURSOR_KEY = "daemonPullV4NodeCursor";
+const PULL_EDGE_CURSOR_KEY = "daemonPullV4EdgeCursor";
 export async function runSyncOnce() {
     let config = await readConfig();
-    const state = await readState();
-    config = await runPreflightV3(config);
-    // A parser version bump reparses everything once and re-pushes conversation
-    // headers as field-masked merges so existing conversations pick up newly
-    // extracted metadata. The state only records the new version after a
-    // successful push, so a failed backfill retries on the next run.
-    const backfillParserChanges = state.parserVersion < PARSER_VERSION;
-    const scan = await scanSelectedSources(config.selectedSources, state, {
-        forceReparse: backfillParserChanges,
-    });
+    config = await runPreflightV4(config);
+    const exclusionsResult = await listSyncExclusions(config);
+    config = exclusionsResult.config;
+    const exclusions = exclusionsResult.response.exclusions;
     const repoResolver = new GitRepoResolver();
-    const nodes = [];
-    const edges = [];
-    const previouslyPushedRoots = new Set(state.pushedSourceRootIds);
-    const pushedSourceRootIds = new Set(previouslyPushedRoots);
-    const pushedSessions = { ...state.pushedSessions };
-    for (const source of config.selectedSources) {
-        const sessions = scan.sessionsBySourceId.get(source.localId) ?? [];
-        const includeRoot = !previouslyPushedRoots.has(source.localId);
-        const mapped = mapSessionsToPushItems(source, sessions, {
-            includeRoot,
-            pushedSessions: state.pushedSessions,
-            repushHeaders: backfillParserChanges,
-            resolveRepo: (workspacePath) => repoResolver.resolve(workspacePath),
-        });
-        nodes.push(...mapped.nodes);
-        edges.push(...mapped.edges);
-        if (includeRoot)
-            pushedSourceRootIds.add(source.localId);
-        Object.assign(pushedSessions, mapped.pushedSessions);
+    const store = openDaemonGraphStore(getDaemonPaths().databaseFile);
+    try {
+        let nodeCursor = store.getSyncCursor(PULL_NODE_CURSOR_KEY);
+        let edgeCursor = store.getSyncCursor(PULL_EDGE_CURSOR_KEY);
+        while (true) {
+            const result = await pullCloudSyncPage(config, {
+                nodeCursor,
+                edgeCursor,
+                limit: PULL_PAGE_LIMIT,
+            });
+            config = result.config;
+            const pullResult = store.applyCloudPull(result.response);
+            if (pullResult.skippedEdges > 0) {
+                console.warn(`Skipped ${pullResult.skippedEdges} pulled edge(s) with missing endpoint nodes.`);
+            }
+            if (result.response.nextNodeCursor) {
+                nodeCursor = result.response.nextNodeCursor;
+                store.setSyncCursor(PULL_NODE_CURSOR_KEY, nodeCursor);
+            }
+            if (result.response.nextEdgeCursor) {
+                edgeCursor = result.response.nextEdgeCursor;
+                store.setSyncCursor(PULL_EDGE_CURSOR_KEY, edgeCursor);
+            }
+            if (!result.response.hasMoreNodes && !result.response.hasMoreEdges)
+                break;
+        }
+        const listings = await listSelectedSourceConversations(config.selectedSources);
+        for (const { source, conversations } of listings) {
+            const parserVersion = parserVersionForSourceKind(source.kind);
+            const filteredConversations = filterExcludedConversations(conversations, exclusions);
+            const conversationsToFetch = store.conversationsNeedingFetch(source, filteredConversations, parserVersion);
+            for (const conversation of conversationsToFetch) {
+                const session = await fetchConversation(source, conversation);
+                if (session) {
+                    store.persistSourceSessions(source, [session], {
+                        parserVersion,
+                        resolveRepo: (workspacePath) => repoResolver.resolve(workspacePath),
+                    });
+                }
+            }
+        }
+        let pushedNodes = 0;
+        let pushedEdges = 0;
+        let batchIndex = 0;
+        while (true) {
+            const batch = store.nextPushBatch({
+                targetSliceCount: TARGET_SLICES_PER_PUSH,
+                edgeLimit: EDGE_PUSH_LIMIT,
+            });
+            if (!batch)
+                break;
+            batchIndex += 1;
+            // Per-batch logging keeps a long cold sync diagnosable when it would
+            // otherwise be silent across hundreds of sequential requests.
+            console.log(`Pushing sync batch ${batchIndex}: ${batch.nodes.length} nodes, ${batch.edges.length} edges.`);
+            const result = await pushSyncPayload(config, { nodes: batch.nodes, edges: batch.edges });
+            config = result.config;
+            store.applyPushAcknowledgements(batch, result.response);
+            pushedNodes += batch.nodes.length;
+            pushedEdges += batch.edges.length;
+        }
+        store.setAppSetting("daemonLastSyncAt", new Date().toISOString());
+        for (const source of config.selectedSources) {
+            store.setAppSetting(`daemonParserVersion.${source.kind}`, String(parserVersionForSourceKind(source.kind)));
+        }
+        return { nodes: pushedNodes, edges: pushedEdges };
     }
-    // Push in request-sized batches (all nodes before any edges) so a large cold
-    // sync stays under Cloud Run's body limit. Each batch push is idempotent, and
-    // sync state is only written after every batch succeeds, so a mid-stream
-    // failure simply re-pushes the whole set on the next run.
-    const batches = planSyncPushBatches(nodes, edges);
-    for (const [index, batch] of batches.entries()) {
-        // Per-batch logging keeps a long cold sync diagnosable when it would
-        // otherwise be silent across hundreds of sequential requests.
-        console.log(`Pushing sync batch ${index + 1}/${batches.length}: ${batch.nodes.length} nodes, ${batch.edges.length} edges.`);
-        config = await pushSyncPayload(config, batch);
+    finally {
+        store.close();
     }
-    await writeState({
-        ...scan.nextState,
-        pushedSourceRootIds: [...pushedSourceRootIds].sort(),
-        pushedSessions,
-        parserVersion: PARSER_VERSION,
-        lastSyncAt: new Date().toISOString(),
-    });
-    return { nodes: nodes.length, edges: edges.length };
 }
 export async function runSyncLoop(intervalSeconds) {
     while (true) {

package/dist/sync/scan.js

@@ -1,35 +1,177 @@
-import { readdir, stat } from "node:fs/promises";
-import { join } from "node:path";
+import { createReadStream } from "node:fs";
+import { open, readdir, readFile, stat } from "node:fs/promises";
+import { basename, join, relative } from "node:path";
+import { createInterface } from "node:readline";
+import { asRecord, stringValue } from "../parser/jsonl.js";
 import { parseClaudeCodeSession } from "../parser/claudeCode.js";
 import { parseCodexSession } from "../parser/codex.js";
-export async function scanSelectedSources(sources, state, options = {}) {
-    const nextState = {
-        files: { ...state.files },
-        pushedSourceRootIds: [...state.pushedSourceRootIds],
-        pushedSessions: { ...state.pushedSessions },
-        parserVersion: state.parserVersion,
-        lastSyncAt: state.lastSyncAt,
+const CODEX_ROOTS = [
+    { directoryName: "sessions", priority: 0 },
+    { directoryName: "archived_sessions", priority: 1 },
+];
+export async function listSelectedSourceConversations(sources) {
+    return Promise.all(sources.map(async (source) => ({
+        source,
+        conversations: source.kind === "codex"
+            ? await listCodexConversations(source)
+            : await listClaudeCodeConversations(source),
+    })));
+}
+export async function fetchConversation(source, conversation) {
+    const parsed = source.kind === "codex"
+        ? await parseCodexSession(conversation.path)
+        : await parseClaudeCodeSession(conversation.path);
+    if (!parsed)
+        return null;
+    return {
+        ...parsed,
+        sourceId: conversation.sourceId,
+        updatedAt: conversation.lastMessageTime ?? conversation.updatedAt ?? parsed.updatedAt,
+        cwd: conversation.cwd ?? parsed.cwd,
     };
-    const sessionsBySourceId = new Map();
-    for (const source of sources) {
-        const files = await sessionFilesForSource(source);
-        const sessions = [];
-        for (const file of files) {
-            const cursor = await cursorForFile(file);
-            if (options.forceReparse !== true && isUnchanged(state.files[file], cursor))
+}
+async function listCodexConversations(source) {
+    const importedThreadIds = await externalImportedThreadIds(source.basePath);
+    const refs = [];
+    for (const root of CODEX_ROOTS) {
+        const rootPath = join(source.basePath, root.directoryName);
+        const files = await walkJsonl(rootPath);
+        for (const path of files) {
+            const summary = await summarizeCodexSession(path);
+            if (!summary.isConversation)
                 continue;
-            const session = source.kind === "codex" ? await parseCodexSession(file) : await parseClaudeCodeSession(file);
-            nextState.files[file] = cursor;
-            if (session)
-                sessions.push(session);
+            const relativePath = relative(rootPath, path);
+            const fallbackSessionId = basename(relativePath, ".jsonl");
+            const sessionId = summary.sessionId ?? fallbackSessionId;
+            if (importedThreadIds.has(sessionId))
+                continue;
+            const attrs = await stat(path);
+            refs.push({
+                sourceLocalId: source.localId,
+                sourceKind: "codex",
+                sourceId: sessionId,
+                path,
+                updatedAt: attrs.mtime.toISOString(),
+                lastMessageTime: await peekLastMessageTime(path, isCodexMessageRecord),
+                cwd: summary.cwd,
+                rootPriority: root.priority,
+            });
         }
-        sessionsBySourceId.set(source.localId, sessions);
     }
-    return { sessionsBySourceId, nextState };
+    return deduplicateCodexRefs(refs).sort((left, right) => recencyMillis(right) - recencyMillis(left));
+}
+async function listClaudeCodeConversations(source) {
+    const projectsPath = join(source.basePath, "projects");
+    let projectDirs;
+    try {
+        projectDirs = await readdir(projectsPath, { withFileTypes: true });
+    }
+    catch (error) {
+        if (isNotFound(error))
+            return [];
+        throw error;
+    }
+    const refs = [];
+    for (const projectDir of projectDirs.filter((entry) => entry.isDirectory()).sort((a, b) => a.name.localeCompare(b.name))) {
+        const projectPath = join(projectsPath, projectDir.name);
+        let entries;
+        try {
+            entries = await readdir(projectPath, { withFileTypes: true });
+        }
+        catch {
+            continue;
+        }
+        for (const entry of entries.filter((item) => item.isFile() && item.name.endsWith(".jsonl"))) {
+            const path = join(projectPath, entry.name);
+            const attrs = await stat(path);
+            refs.push({
+                sourceLocalId: source.localId,
+                sourceKind: "claude_code",
+                sourceId: basename(entry.name, ".jsonl"),
+                path,
+                updatedAt: attrs.mtime.toISOString(),
+                lastMessageTime: await peekLastMessageTime(path, isClaudeCodeMessageRecord),
+            });
+        }
+    }
+    return refs.sort((left, right) => recencyMillis(right) - recencyMillis(left));
+}
+async function summarizeCodexSession(path) {
+    let sessionId = null;
+    let cwd = null;
+    let hasUser = false;
+    let hasAssistant = false;
+    const stream = createReadStream(path, { encoding: "utf8" });
+    const lines = createInterface({ input: stream, crlfDelay: Infinity });
+    try {
+        for await (const line of lines) {
+            const record = parseJsonRecord(line);
+            if (!record)
+                continue;
+            if (stringValue(record.type) === "session_meta") {
+                const payload = asRecord(record.payload);
+                sessionId ??= stringValue(payload?.id) ?? stringValue(record.id) ?? stringValue(record.session_id);
+                cwd ??= stringValue(payload?.cwd) ?? stringValue(record.cwd);
+                continue;
+            }
+            if (!isCodexMessageRecord(record))
+                continue;
+            const role = codexRecordRole(record);
+            hasUser ||= role === "user";
+            hasAssistant ||= role === "assistant";
+            if (hasUser && hasAssistant) {
+                lines.close();
+                stream.destroy();
+                break;
+            }
+        }
+    }
+    catch {
+        return { sessionId, cwd, isConversation: false };
+    }
+    return { sessionId, cwd, isConversation: hasUser && hasAssistant };
 }
-async function sessionFilesForSource(source) {
-    const root = source.kind === "codex" ? join(source.basePath, "sessions") : join(source.basePath, "projects");
-    return walkJsonl(root);
+async function externalImportedThreadIds(basePath) {
+    try {
+        const content = await readFile(join(basePath, "external_agent_session_imports.json"), "utf8");
+        const parsed = JSON.parse(content);
+        const records = asRecord(parsed)?.records;
+        if (!Array.isArray(records))
+            return new Set();
+        return new Set(records.flatMap((record) => {
+            const id = stringValue(asRecord(record)?.imported_thread_id);
+            return id ? [id] : [];
+        }));
+    }
+    catch {
+        return new Set();
+    }
+}
+async function peekLastMessageTime(path, isMessageRecord) {
+    let file;
+    try {
+        file = await open(path, "r");
+        const { size } = await file.stat();
+        const readSize = Math.min(size, 50_000);
+        const buffer = Buffer.alloc(readSize);
+        await file.read(buffer, 0, readSize, size - readSize);
+        const tail = buffer.toString("utf8");
+        for (const line of tail.split(/\r?\n/).reverse()) {
+            const record = parseJsonRecord(line);
+            if (!record || !isMessageRecord(record))
+                continue;
+            const timestamp = stringValue(record.timestamp) ?? stringValue(record.created_at);
+            if (timestamp)
+                return timestamp;
+        }
+        return null;
+    }
+    catch {
+        return null;
+    }
+    finally {
+        await file?.close();
+    }
 }
 async function walkJsonl(root) {
     try {
@@ -43,18 +185,61 @@ async function walkJsonl(root) {
         return nested.flat().sort();
     }
     catch (error) {
-        if (typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT")
+        if (isNotFound(error))
             return [];
         throw error;
     }
 }
-async function cursorForFile(file) {
-    const stats = await stat(file);
-    return {
-        mtimeMs: stats.mtimeMs,
-        size: stats.size,
-    };
+function deduplicateCodexRefs(refs) {
+    const selected = new Map();
+    for (const ref of refs) {
+        const existing = selected.get(ref.sourceId);
+        if (!existing || shouldPreferCodexRef(ref, existing))
+            selected.set(ref.sourceId, ref);
+    }
+    return [...selected.values()].map(({ rootPriority: _rootPriority, ...ref }) => ref);
+}
+function shouldPreferCodexRef(candidate, existing) {
+    if (candidate.rootPriority !== existing.rootPriority)
+        return candidate.rootPriority < existing.rootPriority;
+    return recencyMillis(candidate) > recencyMillis(existing);
+}
+function recencyMillis(ref) {
+    return Date.parse(ref.lastMessageTime ?? ref.updatedAt ?? "") || 0;
+}
+function parseJsonRecord(line) {
+    if (!line.trim())
+        return null;
+    try {
+        return asRecord(JSON.parse(line));
+    }
+    catch {
+        return null;
+    }
+}
+function isCodexMessageRecord(record) {
+    return codexRecordRole(record) !== null;
+}
+function codexRecordRole(record) {
+    if (stringValue(record.type) !== "event_msg")
+        return null;
+    const payload = asRecord(record.payload) ?? record;
+    const role = stringValue(payload.role);
+    if (role === "user" || role === "assistant")
+        return role;
+    const messageType = stringValue(payload.type) ?? stringValue(payload.kind);
+    if (messageType?.includes("user"))
+        return "user";
+    if (messageType === "task_complete")
+        return "assistant";
+    if (messageType?.includes("agent") || messageType?.includes("assistant"))
+        return "assistant";
+    return null;
+}
+function isClaudeCodeMessageRecord(record) {
+    const role = stringValue(asRecord(record.message)?.role);
+    return role === "user" || role === "assistant";
 }
-function isUnchanged(previous, next) {
-    return previous?.mtimeMs === next.mtimeMs && previous.size === next.size;
+function isNotFound(error) {
+    return typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT";
 }

package/dist/sync/slices.js

@@ -1,4 +1,4 @@
-import { contentHash, stableUuidFromString } from "./ids.js";
+import { contentHash } from "./hash.js";
 /**
  * Default maximum character count for a content slice.
  *
@@ -58,17 +58,6 @@ export function takeGraphemes(content, count) {
     }
     return result;
 }
-/**
- * Build the content slices for a message node from its already-redacted
- * content. Slice 0 keeps the same stable id the daemon used when every message
- * was a single slice, so re-pushes stay idempotent.
- */
-export function buildContentSlices(nodeId, content) {
-    return splitContent(content).map((piece, index) => ({
-        id: stableUuidFromString(`${nodeId}:slice:${index}`),
-        nodeId,
-        index,
-        content: piece,
-        contentHash: contentHash(piece),
-    }));
+export function contentHashesFor(content) {
+    return splitContent(content).map((piece) => contentHash(piece));
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nessielabs/daemon",
-  "version": "0.3.2",
+  "version": "0.4.0",
   "description": "Headless Linux ingestion daemon for Nessie agent traces.",
   "license": "UNLICENSED",
   "type": "module",
@@ -25,6 +25,8 @@
   },
   "dependencies": {
     "@inquirer/prompts": "^7.8.0",
+    "@types/better-sqlite3": "^7.6.13",
+    "better-sqlite3": "^12.11.1",
     "commander": "^14.0.2"
   },
   "devDependencies": {

package/dist/sync/ids.js

@@ -1,11 +0,0 @@
-import { createHash } from "node:crypto";
-export function stableUuidFromString(input) {
-    const bytes = createHash("sha256").update(input).digest().subarray(0, 16);
-    bytes[6] = (bytes[6] & 0x0f) | 0x80;
-    bytes[8] = (bytes[8] & 0x3f) | 0x80;
-    const hex = bytes.toString("hex");
-    return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
-}
-export function contentHash(content) {
-    return createHash("sha256").update(content).digest("hex");
-}

package/dist/sync/map.js

@@ -1,111 +0,0 @@
-import { redactSecrets } from "../redact/secrets.js";
-import { stableUuidFromString } from "./ids.js";
-import { buildContentSlices, takeGraphemes } from "./slices.js";
-export function mapSessionsToPushItems(source, sessions, options = { includeRoot: true }) {
-    const pushedSessions = options.pushedSessions ?? {};
-    const nextPushedSessions = {};
-    const nodes = options.includeRoot ? [mapRootNode(source)] : [];
-    const edges = [];
-    for (const session of sessions) {
-        const conversationId = stableUuidFromString(`${source.localId}:session:${session.id}`);
-        const stateKey = sessionStateKey(source, session);
-        const sessionCursor = pushedSessions[stateKey] ?? {
-            conversationPushed: false,
-            pushedMessageCount: 0,
-        };
-        if (!sessionCursor.conversationPushed || options.repushHeaders === true) {
-            const repoInfo = session.cwd !== null ? options.resolveRepo?.(session.cwd) ?? null : null;
-            // Headers always push as a field-masked merge: new conversations insert
-            // normally, and re-pushed headers only touch the workspace metadata
-            // columns, never fields other clients own (name, labels, deletion).
-            nodes.push({
-                node: {
-                    id: conversationId,
-                    integrationId: source.localId,
-                    sourceId: session.sourceId,
-                    name: redactSecrets(session.title),
-                    kind: session.sourceKind === "codex" ? "codex_chat" : "claude_code_chat",
-                    workspacePath: session.cwd,
-                    repoRemoteUrl: repoInfo?.remoteUrl ?? null,
-                    repoKey: repoInfo?.repoKey ?? null,
-                    originalCreatedAt: session.createdAt,
-                    originalUpdatedAt: session.updatedAt,
-                },
-                baseCloudUpdatedAt: null,
-                slices: [],
-                aiChatMessage: null,
-                nessieChatMessage: null,
-                mergeFields: ["workspacePath", "repoRemoteUrl", "repoKey"],
-            });
-        }
-        if (!sessionCursor.conversationPushed) {
-            edges.push(mapEdge(source.localId, conversationId, "source_of", 0));
-        }
-        const pushableMessages = session.messages.filter((message) => message.role !== "system");
-        nextPushedSessions[stateKey] = {
-            conversationPushed: true,
-            pushedMessageCount: pushableMessages.length,
-        };
-        pushableMessages.slice(sessionCursor.pushedMessageCount).forEach((message, offset) => {
-            const index = sessionCursor.pushedMessageCount + offset;
-            const messageId = stableUuidFromString(`${conversationId}:message:${index}`);
-            const content = redactSecrets(message.content);
-            const timestamp = message.timestamp ?? session.updatedAt;
-            nodes.push({
-                node: {
-                    id: messageId,
-                    integrationId: source.localId,
-                    sourceId: `${session.sourceId}#${index}`,
-                    name: takeGraphemes(content, 80) || message.role,
-                    kind: session.sourceKind === "codex" ? "codex_chat_message" : "claude_code_chat_message",
-                    originalCreatedAt: timestamp,
-                    originalUpdatedAt: timestamp,
-                },
-                baseCloudUpdatedAt: null,
-                slices: buildContentSlices(messageId, content),
-                aiChatMessage: {
-                    nodeId: messageId,
-                    role: message.role,
-                    author: null,
-                },
-                nessieChatMessage: null,
-            });
-            edges.push(mapEdge(conversationId, messageId, "contains", index));
-        });
-    }
-    return { nodes, edges, pushedSessions: nextPushedSessions };
-}
-export function sessionStateKey(source, session) {
-    return `${source.localId}:session:${session.id}`;
-}
-function mapRootNode(source) {
-    const now = new Date().toISOString();
-    return {
-        node: {
-            id: source.localId,
-            integrationId: source.localId,
-            sourceId: source.basePath,
-            name: redactSecrets(source.displayName),
-            kind: "integration_root",
-            originalCreatedAt: now,
-            originalUpdatedAt: now,
-        },
-        baseCloudUpdatedAt: null,
-        slices: [],
-        aiChatMessage: null,
-        nessieChatMessage: null,
-    };
-}
-function mapEdge(fromNodeId, toNodeId, type, sortIndex) {
-    const id = stableUuidFromString(`${fromNodeId}:${type}:${toNodeId}`);
-    return {
-        edge: {
-            id,
-            fromNodeId,
-            toNodeId,
-            type,
-            sortIndex,
-        },
-        baseCloudUpdatedAt: null,
-    };
-}