@vkenliu/adit-cloud 0.2.0

package/dist/transcript/uploader.js

@@ -0,0 +1,235 @@
+/**
+ * Transcript incremental uploader.
+ *
+ * Reads a JSONL transcript file, detects new data since the last upload,
+ * compresses the increment with gzip, and uploads it to the server.
+ *
+ * Key design decisions:
+ * - Increments are split on newline boundaries to preserve JSONL format
+ * - Gzip compression reduces bandwidth for large transcripts
+ * - Server returns a version token for conflict detection
+ * - Server can signal "resync" to request a full re-upload
+ * - Max 3 consecutive failures per file before giving up
+ *
+ * The upload API is generic (/api/sync/upload) and uses `type` + `cli`
+ * fields so the server can handle different file types from different
+ * AI coding tools (Claude Code, Cursor, Windsurf, etc.).
+ */
+import { statSync, existsSync, openSync, readSync, closeSync } from "node:fs";
+import { createGzip } from "node:zlib";
+import { Readable } from "node:stream";
+import { CloudApiError } from "../http/errors.js";
+// ── File reading ─────────────────────────────────────────────────────────
+/**
+ * Read the increment from a transcript file starting at the given
+ * byte offset, snapping to the nearest complete line boundary.
+ *
+ * Returns the raw bytes of complete JSONL lines after the offset.
+ * If no complete new lines exist, returns null.
+ */
+export function readIncrement(filePath, offsetBytes) {
+    if (!existsSync(filePath))
+        return null;
+    const stat = statSync(filePath);
+    if (stat.size <= offsetBytes)
+        return null;
+    // Read from the offset to end of file
+    const fd = openSync(filePath, "r");
+    try {
+        const remainingSize = stat.size - offsetBytes;
+        const buf = Buffer.alloc(remainingSize);
+        readSync(fd, buf, 0, remainingSize, offsetBytes);
+        // Find the last newline to ensure we only send complete lines.
+        // JSONL files have one JSON object per line, terminated by \n.
+        let endPos = remainingSize;
+        if (buf[endPos - 1] !== 0x0a) {
+            // Last byte is not a newline — find the last complete line
+            const lastNewline = buf.lastIndexOf(0x0a);
+            if (lastNewline === -1) {
+                // No complete line in the new data yet
+                return null;
+            }
+            endPos = lastNewline + 1;
+        }
+        const data = buf.subarray(0, endPos);
+        return {
+            data: Buffer.from(data), // Copy to avoid holding the full buffer
+            newOffset: offsetBytes + endPos,
+        };
+    }
+    finally {
+        closeSync(fd);
+    }
+}
+// ── Compression ──────────────────────────────────────────────────────────
+/**
+ * Compress a buffer with gzip.
+ */
+export async function compressGzip(data) {
+    return new Promise((resolve, reject) => {
+        const chunks = [];
+        const gzip = createGzip({ level: 6 });
+        const input = Readable.from(data);
+        input
+            .pipe(gzip)
+            .on("data", (chunk) => chunks.push(chunk))
+            .on("end", () => resolve(Buffer.concat(chunks)))
+            .on("error", reject);
+    });
+}
+// ── Upload ───────────────────────────────────────────────────────────────
+/**
+ * Upload an incremental chunk to the server.
+ *
+ * Posts to `/api/sync/upload` with `type` and `cli` fields so the server
+ * can route to the correct handler regardless of which CLI/file-type
+ * produced the data.
+ *
+ * The chunk is gzip-compressed before sending. The server responds
+ * with the confirmed byte offset and a version token.
+ *
+ * If the server returns a 409 with resyncRequired=true, the caller
+ * should reset the upload state and re-upload from offset 0.
+ */
+export async function uploadChunk(client, params) {
+    const increment = readIncrement(params.filePath, params.offsetBytes);
+    if (!increment) {
+        // No new data — return current state
+        return {
+            uploadId: "",
+            confirmedOffset: params.offsetBytes,
+            serverVersion: params.serverVersion ?? "",
+            resyncRequired: false,
+        };
+    }
+    // Compress the increment
+    const compressed = await compressGzip(increment.data);
+    // Build the upload payload
+    const payload = {
+        type: params.type,
+        cli: params.cli,
+        sessionId: params.sessionId,
+        filePath: params.filePath,
+        offsetBytes: params.offsetBytes,
+        newOffsetBytes: increment.newOffset,
+        fileSizeBytes: params.fileSizeBytes,
+        serverVersion: params.serverVersion,
+        encoding: "gzip",
+        // Base64-encode the compressed data for JSON transport
+        data: compressed.toString("base64"),
+        rawBytes: increment.data.length,
+        compressedBytes: compressed.length,
+    };
+    try {
+        return await client.post("/api/sync/upload", payload);
+    }
+    catch (error) {
+        if (error instanceof CloudApiError && error.status === 409) {
+            // Server detected corruption — needs full re-upload
+            const body = error.body;
+            if (body?.resyncRequired) {
+                return {
+                    uploadId: "",
+                    confirmedOffset: 0,
+                    serverVersion: "",
+                    resyncRequired: true,
+                };
+            }
+        }
+        throw error;
+    }
+}
+/**
+ * Upload the full file from offset 0.
+ * Used for initial upload or when server requests a resync.
+ */
+export async function uploadFull(client, params) {
+    return uploadChunk(client, {
+        ...params,
+        offsetBytes: 0,
+        serverVersion: null, // Reset server version on full upload
+    });
+}
+// ── Status check ─────────────────────────────────────────────────────────
+/**
+ * Check upload status via HEAD /api/sync/upload/{uploadId}.
+ *
+ * All status is returned in standard HTTP headers — no JSON body.
+ * This is the only status check method; there is no GET-by-ID endpoint.
+ *
+ * Header mapping:
+ *   X-Upload-Id          → uploadId
+ *   X-Confirmed-Offset   → confirmedOffset
+ *   ETag                 → serverVersion
+ *   X-Upload-Type        → type
+ *   X-Upload-Cli         → cli
+ *   Content-Length        → totalStoredBytes
+ *   Last-Modified         → lastModified
+ *
+ * Returns null if the upload is not found (404).
+ */
+export async function checkUploadStatus(client, uploadId) {
+    try {
+        const h = await client.head(`/api/sync/upload/${encodeURIComponent(uploadId)}`);
+        return parseStatusHeaders(h);
+    }
+    catch (error) {
+        if (error instanceof CloudApiError && error.status === 404) {
+            return null;
+        }
+        throw error;
+    }
+}
+/**
+ * Look up an upload by its original file path.
+ *
+ * Uses GET /api/sync/upload?lookup=by-path&type=...&cli=...&...
+ * The response has no JSON body — status comes from headers only,
+ * same as HEAD.
+ *
+ * Returns null if not found (404).
+ */
+export async function lookupUploadByPath(client, params) {
+    const qs = new URLSearchParams({
+        lookup: "by-path",
+        type: params.type,
+        cli: params.cli,
+        sessionId: params.sessionId,
+        filePath: params.filePath,
+    });
+    try {
+        // We use head() here because the GET response also has no JSON body —
+        // it returns the same headers. The server treats GET and HEAD
+        // identically for this lookup route (both return headers-only).
+        const h = await client.head(`/api/sync/upload?${qs.toString()}`);
+        return parseStatusHeaders(h);
+    }
+    catch (error) {
+        if (error instanceof CloudApiError && error.status === 404) {
+            return null;
+        }
+        throw error;
+    }
+}
+/** Parse upload status from HTTP response headers. */
+function parseStatusHeaders(h) {
+    return {
+        uploadId: h["x-upload-id"] ?? "",
+        confirmedOffset: parseInt(h["x-confirmed-offset"] ?? "0", 10),
+        serverVersion: h["etag"] ?? "",
+        type: h["x-upload-type"] ?? "",
+        cli: h["x-upload-cli"] ?? "",
+        totalStoredBytes: parseInt(h["content-length"] ?? "0", 10),
+        lastModified: h["last-modified"] ?? null,
+    };
+}
+// ── Utilities ────────────────────────────────────────────────────────────
+/**
+ * Get the current file size, or 0 if the file doesn't exist.
+ */
+export function getFileSize(filePath) {
+    if (!existsSync(filePath))
+        return 0;
+    return statSync(filePath).size;
+}
+//# sourceMappingURL=uploader.js.map

package/dist/transcript/uploader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"uploader.js","sourceRoot":"","sources":["../../src/transcript/uploader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAC9E,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAqDlD,4EAA4E;AAE5E;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAC3B,QAAgB,EAChB,WAAmB;IAEnB,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IAEvC,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAChC,IAAI,IAAI,CAAC,IAAI,IAAI,WAAW;QAAE,OAAO,IAAI,CAAC;IAE1C,sCAAsC;IACtC,MAAM,EAAE,GAAG,QAAQ,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IACnC,IAAI,CAAC;QACH,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,GAAG,WAAW,CAAC;QAC9C,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QACxC,QAAQ,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,aAAa,EAAE,WAAW,CAAC,CAAC;QAEjD,+DAA+D;QAC/D,+DAA+D;QAC/D,IAAI,MAAM,GAAG,aAAa,CAAC;QAC3B,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC7B,2DAA2D;YAC3D,MAAM,WAAW,GAAG,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YAC1C,IAAI,WAAW,KAAK,CAAC,CAAC,EAAE,CAAC;gBACvB,uCAAuC;gBACvC,OAAO,IAAI,CAAC;YACd,CAAC;YACD,MAAM,GAAG,WAAW,GAAG,CAAC,CAAC;QAC3B,CAAC;QAED,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QACrC,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,wCAAwC;YACjE,SAAS,EAAE,WAAW,GAAG,MAAM;SAChC,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,SAAS,CAAC,EAAE,CAAC,CAAC;IAChB,CAAC;AACH,CAAC;AAED,4EAA4E;AAE5E;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY;IAC7C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,UAAU,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAElC,KAAK;aACF,IAAI,CAAC,IAAI,CAAC;aACV,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;aACjD,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;aAC/C,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACzB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,4EAA4E;AAE5E;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,MAAmB,EACnB,MAAyB;IAEzB,MAAM,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;IAErE,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,qCAAqC;QACrC,OAAO;YACL,QAAQ,EAAE,EAAE;YACZ,eAAe,EAAE,MAAM,CAAC,WAAW;YACnC,aAAa,EAAE,MAAM,CAAC,aAAa,IAAI,EAAE;YACzC,cAAc,EAAE,KAAK;SACtB,CAAC;IACJ,CAAC;IAED,yBAAyB;IACzB,MAAM,UAAU,GAAG,MAAM,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAEtD,2BAA2B;IAC3B,MAAM,OAAO,GAAG;QACd,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,cAAc,EAAE,SAAS,CAAC,SAAS;QACnC,aAAa,EAAE,MAAM,CAAC,aAAa;QACnC,aAAa,EAAE,MAAM,CAAC,aAAa;QACnC,QAAQ,EAAE,MAAe;QACzB,uDAAuD;QACvD,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACnC,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,MAAM;QAC/B,eAAe,EAAE,UAAU,CAAC,MAAM;KACnC,CAAC;IAEF,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC,IAAI,CACtB,kBAAkB,EAClB,OAAO,CACR,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,aAAa,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC3D,oDAAoD;YACpD,MAAM,IAAI,GAAG,KAAK,CAAC,IAAgD,CAAC;YACpE,IAAI,IAAI,EAAE,cAAc,EAAE,CAAC;gBACzB,OAAO;oBACL,QAAQ,EAAE,EAAE;oBACZ,eAAe,EAAE,CAAC;oBAClB,aAAa,EAAE,EAAE;oBACjB,cAAc,EAAE,IAAI;iBACrB,CAAC;YACJ,CAAC;QACH,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,MAAmB,EACnB,MAA8C;IAE9C,OAAO,WAAW,CAAC,MAAM,EAAE;QACzB,GAAG,MAAM;QACT,WAAW,EAAE,CAAC;QACd,aAAa,EAAE,IAAI,EAAE,sCAAsC;KAC5D,CAAC,CAAC;AACL,CAAC;AAED,4EAA4E;AAE5E;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,MAAmB,EACnB,QAAgB;IAEhB,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,IAAI,CACzB,oBAAoB,kBAAkB,CAAC,QAAQ,CAAC,EAAE,CACnD,CAAC;QACF,OAAO,kBAAkB,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,aAAa,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC3D,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,MAAmB,EACnB,MAKC;IAED,MAAM,EAAE,GAAG,IAAI,eAAe,CAAC;QAC7B,MAAM,EAAE,SAAS;QACjB,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,QAAQ,EAAE,MAAM,CAAC,QAAQ;KAC1B,CAAC,CAAC;IACH,IAAI,CAAC;QACH,sEAAsE;QACtE,8DAA8D;QAC9D,gEAAgE;QAChE,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACjE,OAAO,kBAAkB,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,aAAa,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC3D,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,sDAAsD;AACtD,SAAS,kBAAkB,CACzB,CAAyB;IAEzB,OAAO;QACL,QAAQ,EAAE,CAAC,CAAC,aAAa,CAAC,IAAI,EAAE;QAChC,eAAe,EAAE,QAAQ,CAAC,CAAC,CAAC,oBAAoB,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC;QAC7D,aAAa,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE;QAC9B,IAAI,EAAE,CAAC,CAAC,eAAe,CAAC,IAAI,EAAE;QAC9B,GAAG,EAAE,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE;QAC5B,gBAAgB,EAAE,QAAQ,CAAC,CAAC,CAAC,gBAAgB,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC;QAC1D,YAAY,EAAE,CAAC,CAAC,eAAe,CAAC,IAAI,IAAI;KACzC,CAAC;AACJ,CAAC;AAED,4EAA4E;AAE5E;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,QAAgB;IAC1C,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,CAAC,CAAC;IACpC,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC;AACjC,CAAC"}

package/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "@vkenliu/adit-cloud",
+  "version": "0.2.0",
+  "description": "Cloud sync client for ADIT — device auth, incremental push, conflict resolution",
+  "type": "module",
+  "files": [
+    "dist"
+  ],
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "dependencies": {
+    "@vkenliu/adit-core": "0.2.0",
+    "@vkenliu/adit-engine": "0.2.0"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.13",
+    "typescript": "^5.7.2"
+  },
+  "scripts": {
+    "build": "tsc",
+    "typecheck": "tsc --noEmit"
+  }
+}