@thingd/cli 0.79.0 → 0.80.0

package/dist/data-movement.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"data-movement.d.ts","sourceRoot":"","sources":["../src/data-movement.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,KAAK,UAAU,EAAwD,MAAM,YAAY,CAAC;AAwEnG,wBAAsB,SAAS,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAyClE;AAED,wBAAsB,SAAS,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAsFlE;AAoND,wBAAsB,WAAW,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAmHpE;AAED,wBAAsB,SAAS,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAyClE"}
+{"version":3,"file":"data-movement.d.ts","sourceRoot":"","sources":["../src/data-movement.ts"],"names":[],"mappings":"AAqBA,OAAO,EACL,KAAK,UAAU,EAOhB,MAAM,YAAY,CAAC;AAwEpB,wBAAsB,SAAS,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAyClE;AAED,wBAAsB,SAAS,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAsFlE;AAoND,wBAAsB,WAAW,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAmMpE;AAoHD,wBAAsB,SAAS,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAyClE"}

package/dist/data-movement.js

@@ -1,6 +1,10 @@
-import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { once } from "node:events";
+import { createReadStream, createWriteStream, existsSync, readFileSync, writeFileSync, } from "node:fs";
 import { resolve } from "node:path";
-import { hasFlag, requiredFlag, stringFlag, withDb, writeJson } from "./index.js";
+import { createInterface } from "node:readline";
+import { Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import { hasFlag, requiredFlag, resolveConnection, stringFlag, withDb, writeJson, } from "./index.js";
 const SIDECAR_DEFAULT_URL = "http://localhost:8757";
 const DEFAULT_REDACT_KEYS = [
     "password",
@@ -336,31 +340,48 @@ async function runImportDb(context, source, explicitType) {
 }
 export async function runSnapshot(context) {
     const subCommand = context.parsed.tokens[1];
+    const connection = resolveConnection(context);
+    if (connection.cloud) {
+        await runRemoteSnapshot(context, subCommand);
+        return;
+    }
     if (subCommand === "create") {
         const outPath = requiredFlag(context.parsed, "out");
         await withDb(context, async (db) => {
-            const collectionsMap = {};
+            const output = createWriteStream(resolve(outPath), "utf8");
+            const writeRecord = async (record) => {
+                if (!output.write(`${JSON.stringify(record)}\n`)) {
+                    await once(output, "drain");
+                }
+            };
+            await writeRecord({
+                type: "thingd.snapshot",
+                version: "2.0.0",
+                timestamp: new Date().toISOString(),
+                format: "jsonl",
+            });
             const cols = await db.listCollections();
             for (const col of cols) {
-                collectionsMap[col] = await db.listObjects(col);
+                const objects = await db.listObjects(col);
+                for (const object of objects) {
+                    await writeRecord({ type: "object", collection: col, object });
+                }
             }
             const eventsList = await db.events.list();
-            const queuesMap = {};
+            for (const event of eventsList) {
+                await writeRecord({ type: "event", event });
+            }
             const queues = await db.listQueues();
             for (const q of queues) {
                 const queue = db.queue(q);
                 const [active, dead] = await Promise.all([queue.list(), queue.dead()]);
-                queuesMap[q] = { active, dead };
+                for (const job of [...active, ...dead]) {
+                    await writeRecord({ type: "queue", job });
+                }
             }
-            const snapshot = {
-                version: "1.0.0",
-                timestamp: new Date().toISOString(),
-                collections: collectionsMap,
-                events: eventsList,
-                queues: queuesMap,
-            };
-            writeFileSync(resolve(outPath), JSON.stringify(snapshot, null, 2), "utf8");
-            writeJson(context.stdout, { success: true, out: outPath }, context.pretty);
+            output.end();
+            await once(output, "close");
+            writeJson(context.stdout, { success: true, out: outPath, format: "thingd-snapshot-jsonl", version: "2.0.0" }, context.pretty);
         });
     }
     else if (subCommand === "restore") {
@@ -369,13 +390,65 @@ export async function runSnapshot(context) {
         if (!existsSync(resolvedPath)) {
             throw new Error(`Snapshot file not found: ${inPath}`);
         }
-        const snapshot = JSON.parse(readFileSync(resolvedPath, "utf8"));
-        if (snapshot.version !== "1.0.0") {
-            throw new Error(`Unsupported snapshot version: ${snapshot.version}`);
+        if (inPath.endsWith(".jsonl") || inPath.endsWith(".ndjson")) {
+            await restoreJsonlStream(context, inPath, resolvedPath);
+            return;
+        }
+        const content = readFileSync(resolvedPath, "utf8");
+        const firstLine = content
+            .split("\n")
+            .find((line) => line.trim().length > 0)
+            ?.trim() ?? "";
+        const isJsonl = firstLine.includes('"type":"thingd.snapshot"');
+        const snapshot = isJsonl ? null : JSON.parse(content);
+        if (!isJsonl && snapshot?.version !== "1.0.0") {
+            throw new Error(`Unsupported snapshot version: ${snapshot?.version ?? "unknown"}`);
         }
         context.stderr.write("Restoring snapshot... (consider 'thingd backup --out pre-restore.db' first)\n");
         await withDb(context, async (db) => {
             try {
+                if (isJsonl) {
+                    const records = content
+                        .split("\n")
+                        .map((line) => line.trim())
+                        .filter(Boolean)
+                        .map((line) => JSON.parse(line));
+                    const header = records.shift();
+                    if (header?.type !== "thingd.snapshot" || header.version !== "2.0.0") {
+                        throw new Error(`Unsupported JSONL snapshot version: ${String(header?.version ?? "unknown")}`);
+                    }
+                    for (const record of records) {
+                        if (record.type === "object") {
+                            const object = record.object;
+                            const cleanObject = { ...object };
+                            delete cleanObject.collection;
+                            delete cleanObject.createdAt;
+                            delete cleanObject.updatedAt;
+                            delete cleanObject.version;
+                            await db.put(String(record.collection), cleanObject);
+                        }
+                        else if (record.type === "event") {
+                            const event = record.event;
+                            const cleanEvent = { ...event };
+                            delete cleanEvent.id;
+                            delete cleanEvent.createdAt;
+                            delete cleanEvent.stream;
+                            await db.events.append(event.stream, cleanEvent);
+                        }
+                        else if (record.type === "queue") {
+                            const job = record.job;
+                            await db.queue(job.queue).push(job.payload, {
+                                idempotencyKey: job.id,
+                                maxAttempts: job.maxAttempts,
+                            });
+                        }
+                        else {
+                            throw new Error(`Unsupported snapshot record type: ${String(record.type)}`);
+                        }
+                    }
+                    writeJson(context.stdout, { success: true, in: inPath, format: "thingd-snapshot-jsonl", records: records.length }, context.pretty);
+                    return;
+                }
                 // 1. Restore Collections (clear existing first for true restore)
                 if (snapshot.collections) {
                     for (const [colName, objects] of Object.entries(snapshot.collections)) {
@@ -438,6 +511,100 @@ export async function runSnapshot(context) {
         throw new Error(`Unknown snapshot command: ${subCommand}. Expected 'create' or 'restore'.`);
     }
 }
+async function restoreJsonlStream(context, inPath, resolvedPath) {
+    context.stderr.write("Restoring snapshot... (consider 'thingd backup --out pre-restore.db' first)\n");
+    await withDb(context, async (db) => {
+        const input = createInterface({ input: createReadStream(resolvedPath), crlfDelay: Infinity });
+        let headerSeen = false;
+        let records = 0;
+        try {
+            for await (const line of input) {
+                if (!line.trim()) {
+                    continue;
+                }
+                const record = JSON.parse(line);
+                if (!headerSeen) {
+                    if (record.type !== "thingd.snapshot" || record.version !== "2.0.0") {
+                        throw new Error(`Unsupported JSONL snapshot version: ${String(record.version ?? "unknown")}`);
+                    }
+                    headerSeen = true;
+                    continue;
+                }
+                if (record.type === "object") {
+                    const object = record.object;
+                    const cleanObject = { ...object };
+                    delete cleanObject.collection;
+                    delete cleanObject.createdAt;
+                    delete cleanObject.updatedAt;
+                    delete cleanObject.version;
+                    await db.put(String(record.collection), cleanObject);
+                }
+                else if (record.type === "event") {
+                    const event = record.event;
+                    const cleanEvent = { ...event };
+                    delete cleanEvent.id;
+                    delete cleanEvent.createdAt;
+                    delete cleanEvent.stream;
+                    await db.events.append(event.stream, cleanEvent);
+                }
+                else if (record.type === "queue") {
+                    const job = record.job;
+                    await db.queue(job.queue).push(job.payload, {
+                        idempotencyKey: job.id,
+                        maxAttempts: job.maxAttempts,
+                    });
+                }
+                else {
+                    throw new Error(`Unsupported snapshot record type: ${String(record.type)}`);
+                }
+                records += 1;
+            }
+        }
+        finally {
+            input.close();
+        }
+        if (!headerSeen) {
+            throw new Error("Snapshot is missing its header");
+        }
+        writeJson(context.stdout, { success: true, in: inPath, format: "thingd-snapshot-jsonl", records }, context.pretty);
+    });
+}
+async function runRemoteSnapshot(context, subCommand) {
+    const inPath = stringFlag(context.parsed, "in");
+    const outPath = stringFlag(context.parsed, "out");
+    const base = resolveSidecarUrl(context);
+    const headers = authHeaders(context);
+    if (subCommand === "create") {
+        if (!outPath) {
+            throw new Error("Expected --out <path> for remote snapshot export.");
+        }
+        const response = await fetch(`${base}/v1/snapshot`, { headers });
+        if (!response.ok || !response.body) {
+            throw new Error(`Remote snapshot export failed: HTTP ${response.status}`);
+        }
+        await pipeline(Readable.fromWeb(response.body), createWriteStream(resolve(outPath)));
+        writeJson(context.stdout, { success: true, out: outPath, format: "thingd-snapshot-jsonl", remote: true }, context.pretty);
+        return;
+    }
+    if (subCommand === "restore") {
+        if (!inPath) {
+            throw new Error("Expected --in <path> for remote snapshot import.");
+        }
+        const response = await fetch(`${base}/v1/snapshot`, {
+            method: "POST",
+            headers: { ...headers, "content-type": "application/x-ndjson" },
+            body: createReadStream(resolve(inPath)),
+            duplex: "half",
+        });
+        const result = (await response.json());
+        if (!response.ok) {
+            throw new Error(`Remote snapshot import failed: ${JSON.stringify(result)}`);
+        }
+        writeJson(context.stdout, result, context.pretty);
+        return;
+    }
+    throw new Error(`Unknown snapshot command: ${subCommand}. Expected 'create' or 'restore'.`);
+}
 export async function runBackup(context) {
     const inPath = stringFlag(context.parsed, "in");
     const outPath = stringFlag(context.parsed, "out");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@thingd/cli",
-  "version": "0.79.0",
+  "version": "0.80.0",
   "description": "CLI, Interactive TUI Dashboard, and MCP server for thingd — a fast object-first data engine for applications and AI agents.",
   "type": "module",
   "homepage": "https://engine.thingd.cloud",
@@ -42,14 +42,15 @@
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.30.0",
-    "@thingd/sdk": "^0.79.0",
+    "@thingd/sdk": "^0.80.0",
     "cli-table3": "^0.6.5",
     "picocolors": "^1.1.1",
     "zod": "^4.4.3"
   },
   "scripts": {
     "build": "pnpm --filter frontend build && tsc -p tsconfig.json && node -e \"fs.mkdirSync('dist/dashboard', { recursive: true }); fs.cpSync('src/dashboard/public', 'dist/dashboard/public', { recursive: true })\"",
-    "test": "pnpm build && node --test test/*.test.mjs"
+    "test": "pnpm build && pnpm test:built",
+    "test:built": "node --test test/*.test.mjs"
   },
   "engines": {
     "node": ">=24.0.0"