@nessielabs/daemon 0.3.0 → 0.3.2

package/dist/cli.js

@@ -7,7 +7,7 @@ const program = new Command();
 program
     .name("nessie-daemon")
     .description("Headless Nessie ingestion daemon for Linux agent machines.")
-    .version("0.2.2");
+    .version("0.3.1");
 program
     .command("setup")
     .description("Authenticate, select local sources, and optionally install the user systemd service.")

package/dist/parser/claudeCode.js

@@ -4,7 +4,11 @@ import { asRecord, readJsonl, stringValue } from "./jsonl.js";
 export async function parseClaudeCodeSession(path) {
     const records = await readJsonl(path);
     const messages = [];
-    let sessionId = basename(path, ".jsonl");
+    // Anchor session identity to the JSONL filename, matching the macOS client.
+    // The in-file `sessionId` is reused across files for worktree/resumed
+    // sessions, so keying on it collapses distinct transcripts into one
+    // conversation id and makes their deterministic message ids collide (409).
+    const sessionId = basename(path, ".jsonl");
     let title = null;
     let createdAt = null;
     let updatedAt = null;
@@ -19,7 +23,6 @@ export async function parseClaudeCodeSession(path) {
         cwd ??= stringValue(record.cwd);
         if (record.isSidechain === true)
             continue;
-        sessionId = stringValue(record.sessionId) ?? stringValue(record.session_id) ?? sessionId;
         const timestamp = stringValue(record.timestamp);
         if (timestamp) {
             createdAt ??= timestamp;

package/dist/sync/batch.js

@@ -0,0 +1,50 @@
+/**
+ * Target number of content slices per node push request.
+ *
+ * Mirrors `CloudSyncPipeline.targetSlicesPerPush` in the macOS app. Batching by
+ * slice count keeps each request well under Cloud Run's 32 MiB body limit, which
+ * a single all-in-one push of a large cold sync would blow past (413).
+ */
+export const TARGET_SLICES_PER_PUSH = 200;
+/**
+ * Maximum edges per push request. Mirrors the macOS app's `edgeLimit`. Edges
+ * carry only ids, so this bound is about request count, not size.
+ */
+export const EDGE_PUSH_LIMIT = 500;
+/**
+ * Split a push into request-sized batches, mirroring the macOS client.
+ *
+ * All node batches are emitted before any edge batch: edges carry foreign keys
+ * to their endpoint nodes, so every node pushed in this sync must land before
+ * the edges that reference it. Within a sync, node ordering is otherwise
+ * irrelevant — `node.integration_id` is not a foreign key — so nodes are batched
+ * purely by accumulated slice count.
+ *
+ * A node whose own slice count exceeds `targetSliceCount` is still emitted as
+ * its own batch rather than dropped, matching the macOS app's "a single large
+ * node is kept intact" behaviour.
+ */
+export function planSyncPushBatches(nodes, edges, options = {}) {
+    const targetSliceCount = options.targetSliceCount ?? TARGET_SLICES_PER_PUSH;
+    const edgeLimit = options.edgeLimit ?? EDGE_PUSH_LIMIT;
+    const batches = [];
+    let currentNodes = [];
+    let sliceCount = 0;
+    for (const node of nodes) {
+        const nodeSliceCount = node.slices.length;
+        if (currentNodes.length > 0 && sliceCount + nodeSliceCount > targetSliceCount) {
+            batches.push({ nodes: currentNodes, edges: [] });
+            currentNodes = [];
+            sliceCount = 0;
+        }
+        currentNodes.push(node);
+        sliceCount += nodeSliceCount;
+    }
+    if (currentNodes.length > 0) {
+        batches.push({ nodes: currentNodes, edges: [] });
+    }
+    for (let offset = 0; offset < edges.length; offset += edgeLimit) {
+        batches.push({ nodes: [], edges: edges.slice(offset, offset + edgeLimit) });
+    }
+    return batches;
+}

package/dist/sync/loop.js

@@ -3,6 +3,7 @@ import { AuthRefreshError } from "../auth/tokens.js";
 import { readConfig, readState, writeState } from "../config/store.js";
 import { GitRepoResolver } from "../git/repoResolver.js";
 import { PARSER_VERSION } from "../parser/types.js";
+import { planSyncPushBatches } from "./batch.js";
 import { runPreflightV3, pushSyncPayload } from "./client.js";
 import { mapSessionsToPushItems } from "./map.js";
 import { scanSelectedSources } from "./scan.js";
@@ -39,8 +40,16 @@ export async function runSyncOnce() {
             pushedSourceRootIds.add(source.localId);
         Object.assign(pushedSessions, mapped.pushedSessions);
     }
-    if (nodes.length > 0 || edges.length > 0) {
-        config = await pushSyncPayload(config, { nodes, edges });
+    // Push in request-sized batches (all nodes before any edges) so a large cold
+    // sync stays under Cloud Run's body limit. Each batch push is idempotent, and
+    // sync state is only written after every batch succeeds, so a mid-stream
+    // failure simply re-pushes the whole set on the next run.
+    const batches = planSyncPushBatches(nodes, edges);
+    for (const [index, batch] of batches.entries()) {
+        // Per-batch logging keeps a long cold sync diagnosable when it would
+        // otherwise be silent across hundreds of sequential requests.
+        console.log(`Pushing sync batch ${index + 1}/${batches.length}: ${batch.nodes.length} nodes, ${batch.edges.length} edges.`);
+        config = await pushSyncPayload(config, batch);
     }
     await writeState({
         ...scan.nextState,

package/dist/sync/map.js

@@ -1,5 +1,6 @@
 import { redactSecrets } from "../redact/secrets.js";
-import { contentHash, stableUuidFromString } from "./ids.js";
+import { stableUuidFromString } from "./ids.js";
+import { buildContentSlices, takeGraphemes } from "./slices.js";
 export function mapSessionsToPushItems(source, sessions, options = { includeRoot: true }) {
     const pushedSessions = options.pushedSessions ?? {};
     const nextPushedSessions = {};
@@ -55,19 +56,13 @@ export function mapSessionsToPushItems(source, sessions, options = { includeRoot
                     id: messageId,
                     integrationId: source.localId,
                     sourceId: `${session.sourceId}#${index}`,
-                    name: content.slice(0, 80) || message.role,
+                    name: takeGraphemes(content, 80) || message.role,
                     kind: session.sourceKind === "codex" ? "codex_chat_message" : "claude_code_chat_message",
                     originalCreatedAt: timestamp,
                     originalUpdatedAt: timestamp,
                 },
                 baseCloudUpdatedAt: null,
-                slices: [{
-                        id: stableUuidFromString(`${messageId}:slice:0`),
-                        nodeId: messageId,
-                        index: 0,
-                        content,
-                        contentHash: contentHash(content),
-                    }],
+                slices: buildContentSlices(messageId, content),
                 aiChatMessage: {
                     nodeId: messageId,
                     role: message.role,

package/dist/sync/slices.js

@@ -0,0 +1,74 @@
+import { contentHash, stableUuidFromString } from "./ids.js";
+/**
+ * Default maximum character count for a content slice.
+ *
+ * Mirrors `NodeContentSlice.defaultContentSliceSize` in NessieCore so the
+ * daemon produces the same slice granularity as the macOS client.
+ */
+export const DEFAULT_CONTENT_SLICE_SIZE = 800;
+// Swift's `String.index(_:offsetBy:)` advances by `Character`, i.e. extended
+// grapheme clusters, so its 800-unit slices never split a grapheme. Segmenting
+// by grapheme here matches that behaviour rather than slicing by UTF-16 code
+// units, which would cut emoji and combining sequences in half. A fixed locale
+// keeps grapheme boundaries deterministic across daemon hosts regardless of the
+// process locale (e.g. LANG=C).
+const graphemeSegmenter = new Intl.Segmenter("en-US", { granularity: "grapheme" });
+/**
+ * Split content into the canonical chunks used for node content slices.
+ *
+ * Ports `NodeContentSlice.splitContent` from NessieCore: empty content yields a
+ * single empty slice, and non-empty content is chunked into `sliceSize`
+ * grapheme runs.
+ */
+export function splitContent(content, sliceSize = DEFAULT_CONTENT_SLICE_SIZE) {
+    if (content === "")
+        return [""];
+    const pieces = [];
+    let current = "";
+    let count = 0;
+    for (const { segment } of graphemeSegmenter.segment(content)) {
+        current += segment;
+        count += 1;
+        if (count === sliceSize) {
+            pieces.push(current);
+            current = "";
+            count = 0;
+        }
+    }
+    if (count > 0)
+        pieces.push(current);
+    return pieces;
+}
+/**
+ * Take the first `count` grapheme clusters of `content`.
+ *
+ * Used for short display names so a truncation boundary never severs an emoji
+ * or combining sequence the way a UTF-16 `content.slice(0, count)` would.
+ */
+export function takeGraphemes(content, count) {
+    if (count <= 0)
+        return "";
+    let result = "";
+    let taken = 0;
+    for (const { segment } of graphemeSegmenter.segment(content)) {
+        result += segment;
+        taken += 1;
+        if (taken === count)
+            break;
+    }
+    return result;
+}
+/**
+ * Build the content slices for a message node from its already-redacted
+ * content. Slice 0 keeps the same stable id the daemon used when every message
+ * was a single slice, so re-pushes stay idempotent.
+ */
+export function buildContentSlices(nodeId, content) {
+    return splitContent(content).map((piece, index) => ({
+        id: stableUuidFromString(`${nodeId}:slice:${index}`),
+        nodeId,
+        index,
+        content: piece,
+        contentHash: contentHash(piece),
+    }));
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nessielabs/daemon",
-  "version": "0.3.0",
+  "version": "0.3.2",
   "description": "Headless Linux ingestion daemon for Nessie agent traces.",
   "license": "UNLICENSED",
   "type": "module",