@thingd/cli 0.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +238 -0
- package/dist/dashboard/public/assets/index-B-Y-3-0l.js +2 -0
- package/dist/dashboard/public/assets/index-B5YhpIl3.js +2 -0
- package/dist/dashboard/public/assets/index-BnFclxvN.css +1 -0
- package/dist/dashboard/public/assets/index-BtA9rnyI.js +2 -0
- package/dist/dashboard/public/assets/index-BzLTzidY.js +2 -0
- package/dist/dashboard/public/assets/index-C6PkDB7y.css +1 -0
- package/dist/dashboard/public/assets/index-D8yUCdOQ.js +2 -0
- package/dist/dashboard/public/assets/index-fQywB2df.js +2 -0
- package/dist/dashboard/public/assets/index-kZdrdi3K.css +1 -0
- package/dist/dashboard/public/assets/index-kgZrboBN.js +4 -0
- package/dist/dashboard/public/favicon.svg +1 -0
- package/dist/dashboard/public/icons.svg +24 -0
- package/dist/dashboard/public/index.html +16 -0
- package/dist/dashboard/server.d.ts +6 -0
- package/dist/dashboard/server.d.ts.map +1 -0
- package/dist/dashboard/server.js +385 -0
- package/dist/data-movement.d.ts +5 -0
- package/dist/data-movement.d.ts.map +1 -0
- package/dist/data-movement.js +257 -0
- package/dist/doctor.d.ts +3 -0
- package/dist/doctor.d.ts.map +1 -0
- package/dist/doctor.js +109 -0
- package/dist/index.d.ts +42 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +1015 -0
- package/dist/install.d.ts +3 -0
- package/dist/install.d.ts.map +1 -0
- package/dist/install.js +311 -0
- package/dist/interactive.d.ts +2 -0
- package/dist/interactive.d.ts.map +1 -0
- package/dist/interactive.js +1592 -0
- package/dist/logo.d.ts +3 -0
- package/dist/logo.d.ts.map +1 -0
- package/dist/logo.js +8 -0
- package/dist/mcp/audit.d.ts +27 -0
- package/dist/mcp/audit.d.ts.map +1 -0
- package/dist/mcp/audit.js +36 -0
- package/dist/mcp/cluster.d.ts +68 -0
- package/dist/mcp/cluster.d.ts.map +1 -0
- package/dist/mcp/cluster.js +303 -0
- package/dist/mcp/config.d.ts +14 -0
- package/dist/mcp/config.d.ts.map +1 -0
- package/dist/mcp/config.js +67 -0
- package/dist/mcp/http.d.ts +25 -0
- package/dist/mcp/http.d.ts.map +1 -0
- package/dist/mcp/http.js +588 -0
- package/dist/mcp/index.d.ts +5 -0
- package/dist/mcp/index.d.ts.map +1 -0
- package/dist/mcp/index.js +3 -0
- package/dist/mcp/result.d.ts +3 -0
- package/dist/mcp/result.d.ts.map +1 -0
- package/dist/mcp/result.js +10 -0
- package/dist/mcp/server.d.ts +19 -0
- package/dist/mcp/server.d.ts.map +1 -0
- package/dist/mcp/server.js +51 -0
- package/dist/mcp/tools.d.ts +10 -0
- package/dist/mcp/tools.d.ts.map +1 -0
- package/dist/mcp/tools.js +568 -0
- package/dist/mcp-http.d.ts +3 -0
- package/dist/mcp-http.d.ts.map +1 -0
- package/dist/mcp-http.js +42 -0
- package/dist/mcp.d.ts +3 -0
- package/dist/mcp.d.ts.map +1 -0
- package/dist/mcp.js +22 -0
- package/dist/paths.d.ts +4 -0
- package/dist/paths.d.ts.map +1 -0
- package/dist/paths.js +14 -0
- package/dist/rest/helpers.d.ts +17 -0
- package/dist/rest/helpers.d.ts.map +1 -0
- package/dist/rest/helpers.js +55 -0
- package/dist/rest/server.d.ts +4 -0
- package/dist/rest/server.d.ts.map +1 -0
- package/dist/rest/server.js +317 -0
- package/package.json +57 -0
|
@@ -0,0 +1,385 @@
|
|
|
1
|
+
import { existsSync, promises as fs, statSync } from "node:fs";
|
|
2
|
+
import { createServer } from "node:http";
|
|
3
|
+
import { dirname, extname, join } from "node:path";
|
|
4
|
+
import { fileURLToPath } from "node:url";
|
|
5
|
+
import { handleRestRequest, ThingD } from "@thingd/node";
|
|
6
|
+
const __filename = fileURLToPath(import.meta.url);
|
|
7
|
+
const __dirname = dirname(__filename);
|
|
8
|
+
// Candidate public folders to support both tsx dev and compiled dist packaging
|
|
9
|
+
const publicDirCandidates = [
|
|
10
|
+
join(__dirname, "public"),
|
|
11
|
+
join(__dirname, "../public"),
|
|
12
|
+
join(__dirname, "../../../src/dashboard/public"),
|
|
13
|
+
join(__dirname, "../../src/dashboard/public"),
|
|
14
|
+
];
|
|
15
|
+
let publicDir = "";
|
|
16
|
+
for (const cand of publicDirCandidates) {
|
|
17
|
+
if (existsSync(cand)) {
|
|
18
|
+
publicDir = cand;
|
|
19
|
+
break;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
const MIME_TYPES = {
|
|
23
|
+
".html": "text/html",
|
|
24
|
+
".css": "text/css",
|
|
25
|
+
".js": "application/javascript",
|
|
26
|
+
".json": "application/json",
|
|
27
|
+
".png": "image/png",
|
|
28
|
+
".ico": "image/x-icon",
|
|
29
|
+
};
|
|
30
|
+
async function readBody(req) {
|
|
31
|
+
return new Promise((resolvePromise, rejectPromise) => {
|
|
32
|
+
let body = "";
|
|
33
|
+
req.on("data", (chunk) => {
|
|
34
|
+
body += chunk;
|
|
35
|
+
});
|
|
36
|
+
req.on("end", () => {
|
|
37
|
+
resolvePromise(body);
|
|
38
|
+
});
|
|
39
|
+
req.on("error", (err) => {
|
|
40
|
+
rejectPromise(err);
|
|
41
|
+
});
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
function sendError(res, status, message) {
|
|
45
|
+
res.writeHead(status, { "Content-Type": "application/json" });
|
|
46
|
+
res.end(JSON.stringify({ error: message }));
|
|
47
|
+
}
|
|
48
|
+
function isCloudPath(path) {
|
|
49
|
+
return path.startsWith("http://") || path.startsWith("https://") || path.startsWith("thingd://");
|
|
50
|
+
}
|
|
51
|
+
export async function startDashboardServer(connectionOptions, port) {
|
|
52
|
+
// 1. Maintain dynamic active database options
|
|
53
|
+
let activeOptions = { ...connectionOptions };
|
|
54
|
+
let db = await ThingD.open({
|
|
55
|
+
path: activeOptions.path,
|
|
56
|
+
url: activeOptions.cloud ? activeOptions.path : undefined,
|
|
57
|
+
driver: activeOptions.driver,
|
|
58
|
+
authToken: activeOptions.authToken,
|
|
59
|
+
});
|
|
60
|
+
// 2. Create HTTP Server
|
|
61
|
+
const server = createServer(async (req, res) => {
|
|
62
|
+
try {
|
|
63
|
+
const url = new URL(req.url || "", `http://${req.headers.host || "localhost"}`);
|
|
64
|
+
const pathname = url.pathname;
|
|
65
|
+
// Handle CORS for ease of developer integrations
|
|
66
|
+
res.setHeader("Access-Control-Allow-Origin", "*");
|
|
67
|
+
res.setHeader("Access-Control-Allow-Methods", "GET, POST, DELETE, OPTIONS");
|
|
68
|
+
res.setHeader("Access-Control-Allow-Headers", "Content-Type, Authorization");
|
|
69
|
+
if (req.method === "OPTIONS") {
|
|
70
|
+
res.writeHead(204);
|
|
71
|
+
res.end();
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
// Security Gate middleware for API endpoints
|
|
75
|
+
const isApiRoute = pathname.startsWith("/api/");
|
|
76
|
+
const isConnectRoute = pathname === "/api/connect";
|
|
77
|
+
const isRestRoute = pathname.startsWith("/v1/");
|
|
78
|
+
if ((isApiRoute || isRestRoute) && !isConnectRoute && activeOptions.authToken) {
|
|
79
|
+
const authHeader = req.headers.authorization;
|
|
80
|
+
const expectedHeader = `Bearer ${activeOptions.authToken}`;
|
|
81
|
+
if (!authHeader || authHeader !== expectedHeader) {
|
|
82
|
+
res.writeHead(401, { "Content-Type": "application/json" });
|
|
83
|
+
res.end(JSON.stringify({ error: "Unauthorized. Valid auth token is required." }));
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
// REST API Routes
|
|
88
|
+
if (pathname.startsWith("/api/")) {
|
|
89
|
+
// POST /api/connect (Dynamic connection swapping)
|
|
90
|
+
if (pathname === "/api/connect" && req.method === "POST") {
|
|
91
|
+
const bodyStr = await readBody(req);
|
|
92
|
+
const { path, driver, authToken } = JSON.parse(bodyStr);
|
|
93
|
+
if (!path || !driver) {
|
|
94
|
+
sendError(res, 400, "Fields 'path' and 'driver' are required.");
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
// Safely shut down the old db instance
|
|
98
|
+
await db.close();
|
|
99
|
+
// Spawn new db connection dynamically
|
|
100
|
+
db = await ThingD.open({
|
|
101
|
+
path,
|
|
102
|
+
url: isCloudPath(path) ? path : undefined,
|
|
103
|
+
driver,
|
|
104
|
+
authToken,
|
|
105
|
+
});
|
|
106
|
+
activeOptions = {
|
|
107
|
+
path,
|
|
108
|
+
driver,
|
|
109
|
+
authToken,
|
|
110
|
+
cloud: isCloudPath(path),
|
|
111
|
+
};
|
|
112
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
113
|
+
res.end(JSON.stringify({ success: true, path, driver }));
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
// GET /api/status
|
|
117
|
+
if (pathname === "/api/status" && req.method === "GET") {
|
|
118
|
+
const [objects, events, activeJobs, deadJobs] = await Promise.all([
|
|
119
|
+
db.countObjects(),
|
|
120
|
+
db.countEvents(),
|
|
121
|
+
db.countActiveJobs(),
|
|
122
|
+
db.countDeadJobs(),
|
|
123
|
+
]);
|
|
124
|
+
let dbSize = "N/A (in-memory)";
|
|
125
|
+
if (activeOptions.driver === "native" && existsSync(activeOptions.path)) {
|
|
126
|
+
try {
|
|
127
|
+
const stats = statSync(activeOptions.path);
|
|
128
|
+
dbSize = `${(stats.size / 1024).toFixed(1)} KB`;
|
|
129
|
+
}
|
|
130
|
+
catch {
|
|
131
|
+
dbSize = "N/A (error)";
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
135
|
+
res.end(JSON.stringify({
|
|
136
|
+
mode: activeOptions.cloud ? "cloud" : "local",
|
|
137
|
+
driver: activeOptions.driver || "memory",
|
|
138
|
+
path: activeOptions.path,
|
|
139
|
+
metrics: { objects, events, activeJobs, deadJobs, dbSize },
|
|
140
|
+
authRequired: !!activeOptions.authToken,
|
|
141
|
+
}));
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
// GET /api/collections
|
|
145
|
+
if (pathname === "/api/collections" && req.method === "GET") {
|
|
146
|
+
const collections = await db.listCollections();
|
|
147
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
148
|
+
res.end(JSON.stringify(collections));
|
|
149
|
+
return;
|
|
150
|
+
}
|
|
151
|
+
// GET/POST/DELETE /api/objects
|
|
152
|
+
if (pathname === "/api/objects") {
|
|
153
|
+
if (req.method === "GET") {
|
|
154
|
+
const collection = url.searchParams.get("collection");
|
|
155
|
+
if (!collection) {
|
|
156
|
+
sendError(res, 400, "Query parameter 'collection' is required.");
|
|
157
|
+
return;
|
|
158
|
+
}
|
|
159
|
+
const objects = await db.listObjects(collection);
|
|
160
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
161
|
+
res.end(JSON.stringify(objects));
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
if (req.method === "POST") {
|
|
165
|
+
const bodyStr = await readBody(req);
|
|
166
|
+
const { collection, id, text, data } = JSON.parse(bodyStr);
|
|
167
|
+
if (!collection || !id) {
|
|
168
|
+
sendError(res, 400, "Fields 'collection' and 'id' are required.");
|
|
169
|
+
return;
|
|
170
|
+
}
|
|
171
|
+
const result = await db.put(collection, { id, text, ...data });
|
|
172
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
173
|
+
res.end(JSON.stringify(result));
|
|
174
|
+
return;
|
|
175
|
+
}
|
|
176
|
+
if (req.method === "DELETE") {
|
|
177
|
+
const collection = url.searchParams.get("collection");
|
|
178
|
+
const id = url.searchParams.get("id");
|
|
179
|
+
if (!collection || !id) {
|
|
180
|
+
sendError(res, 400, "Query parameters 'collection' and 'id' are required.");
|
|
181
|
+
return;
|
|
182
|
+
}
|
|
183
|
+
const result = await db.delete(collection, id);
|
|
184
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
185
|
+
res.end(JSON.stringify(result));
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
// GET/POST /api/events
|
|
190
|
+
if (pathname === "/api/events") {
|
|
191
|
+
if (req.method === "GET") {
|
|
192
|
+
const stream = url.searchParams.get("stream") || undefined;
|
|
193
|
+
const limitVal = url.searchParams.get("limit");
|
|
194
|
+
const limit = limitVal ? Number.parseInt(limitVal, 10) : undefined;
|
|
195
|
+
const events = await db.events.list(stream);
|
|
196
|
+
const sliced = limit ? events.slice(0, limit) : events;
|
|
197
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
198
|
+
res.end(JSON.stringify(sliced));
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
201
|
+
if (req.method === "POST") {
|
|
202
|
+
const bodyStr = await readBody(req);
|
|
203
|
+
const { stream, type, text, data } = JSON.parse(bodyStr);
|
|
204
|
+
if (!stream || !type) {
|
|
205
|
+
sendError(res, 400, "Fields 'stream' and 'type' are required.");
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
208
|
+
const result = await db.events.append(stream, { type, text, ...data });
|
|
209
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
210
|
+
res.end(JSON.stringify(result));
|
|
211
|
+
return;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
// GET /api/events/streams
|
|
215
|
+
if (pathname === "/api/events/streams" && req.method === "GET") {
|
|
216
|
+
const streams = await db.listStreams();
|
|
217
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
218
|
+
res.end(JSON.stringify(streams));
|
|
219
|
+
return;
|
|
220
|
+
}
|
|
221
|
+
// GET /api/queues
|
|
222
|
+
if (pathname === "/api/queues" && req.method === "GET") {
|
|
223
|
+
const queues = await db.listQueues();
|
|
224
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
225
|
+
res.end(JSON.stringify(queues));
|
|
226
|
+
return;
|
|
227
|
+
}
|
|
228
|
+
// GET /api/queues/jobs
|
|
229
|
+
if (pathname === "/api/queues/jobs" && req.method === "GET") {
|
|
230
|
+
const queue = url.searchParams.get("queue");
|
|
231
|
+
const status = url.searchParams.get("status");
|
|
232
|
+
if (!queue) {
|
|
233
|
+
sendError(res, 400, "Query parameter 'queue' is required.");
|
|
234
|
+
return;
|
|
235
|
+
}
|
|
236
|
+
const q = db.queue(queue);
|
|
237
|
+
const jobs = status === "dead" ? await q.dead() : await q.list();
|
|
238
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
239
|
+
res.end(JSON.stringify(jobs));
|
|
240
|
+
return;
|
|
241
|
+
}
|
|
242
|
+
// GET /api/queues/stats
|
|
243
|
+
if (pathname === "/api/queues/stats" && req.method === "GET") {
|
|
244
|
+
const queue = url.searchParams.get("queue");
|
|
245
|
+
if (!queue) {
|
|
246
|
+
sendError(res, 400, "Query parameter 'queue' is required.");
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
249
|
+
const q = db.queue(queue);
|
|
250
|
+
const [activeJobs, deadJobs] = await Promise.all([q.list(), q.dead()]);
|
|
251
|
+
const leased = activeJobs.filter((j) => j.status === "leased").length;
|
|
252
|
+
const ready = activeJobs.filter((j) => j.status === "ready").length;
|
|
253
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
254
|
+
res.end(JSON.stringify({
|
|
255
|
+
queue,
|
|
256
|
+
totalActive: activeJobs.length,
|
|
257
|
+
ready,
|
|
258
|
+
leased,
|
|
259
|
+
dead: deadJobs.length,
|
|
260
|
+
}));
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
263
|
+
// POST /api/queues/push
|
|
264
|
+
if (pathname === "/api/queues/push" && req.method === "POST") {
|
|
265
|
+
const bodyStr = await readBody(req);
|
|
266
|
+
const { queue, payload, delayMs, maxAttempts, idempotencyKey } = JSON.parse(bodyStr);
|
|
267
|
+
if (!queue || !payload) {
|
|
268
|
+
sendError(res, 400, "Fields 'queue' and 'payload' are required.");
|
|
269
|
+
return;
|
|
270
|
+
}
|
|
271
|
+
const q = db.queue(queue);
|
|
272
|
+
const result = await q.push(payload, { delayMs, maxAttempts, idempotencyKey });
|
|
273
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
274
|
+
res.end(JSON.stringify(result));
|
|
275
|
+
return;
|
|
276
|
+
}
|
|
277
|
+
// POST /api/queues/claim
|
|
278
|
+
if (pathname === "/api/queues/claim" && req.method === "POST") {
|
|
279
|
+
const bodyStr = await readBody(req);
|
|
280
|
+
const { queue, leaseMs } = JSON.parse(bodyStr);
|
|
281
|
+
if (!queue) {
|
|
282
|
+
sendError(res, 400, "Field 'queue' is required.");
|
|
283
|
+
return;
|
|
284
|
+
}
|
|
285
|
+
const q = db.queue(queue);
|
|
286
|
+
const job = await q.claim({ leaseMs });
|
|
287
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
288
|
+
res.end(JSON.stringify(job || null));
|
|
289
|
+
return;
|
|
290
|
+
}
|
|
291
|
+
// POST /api/queues/ack
|
|
292
|
+
if (pathname === "/api/queues/ack" && req.method === "POST") {
|
|
293
|
+
const bodyStr = await readBody(req);
|
|
294
|
+
const { queue, jobId } = JSON.parse(bodyStr);
|
|
295
|
+
if (!queue || !jobId) {
|
|
296
|
+
sendError(res, 400, "Fields 'queue' and 'jobId' are required.");
|
|
297
|
+
return;
|
|
298
|
+
}
|
|
299
|
+
const q = db.queue(queue);
|
|
300
|
+
const result = await q.ack(jobId);
|
|
301
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
302
|
+
res.end(JSON.stringify(result));
|
|
303
|
+
return;
|
|
304
|
+
}
|
|
305
|
+
// POST /api/queues/nack
|
|
306
|
+
if (pathname === "/api/queues/nack" && req.method === "POST") {
|
|
307
|
+
const bodyStr = await readBody(req);
|
|
308
|
+
const { queue, jobId, error, delayMs } = JSON.parse(bodyStr);
|
|
309
|
+
if (!queue || !jobId) {
|
|
310
|
+
sendError(res, 400, "Fields 'queue' and 'jobId' are required.");
|
|
311
|
+
return;
|
|
312
|
+
}
|
|
313
|
+
const q = db.queue(queue);
|
|
314
|
+
const result = await q.nack(jobId, { error, delayMs });
|
|
315
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
316
|
+
res.end(JSON.stringify(result));
|
|
317
|
+
return;
|
|
318
|
+
}
|
|
319
|
+
// GET /api/search
|
|
320
|
+
if (pathname === "/api/search" && req.method === "GET") {
|
|
321
|
+
const query = url.searchParams.get("query");
|
|
322
|
+
const limitVal = url.searchParams.get("limit");
|
|
323
|
+
const collectionsStr = url.searchParams.get("collections");
|
|
324
|
+
const filterStr = url.searchParams.get("filter");
|
|
325
|
+
if (!query) {
|
|
326
|
+
sendError(res, 400, "Query parameter 'query' is required.");
|
|
327
|
+
return;
|
|
328
|
+
}
|
|
329
|
+
const limit = limitVal ? Number.parseInt(limitVal, 10) : undefined;
|
|
330
|
+
const collections = collectionsStr ? collectionsStr.split(",") : undefined;
|
|
331
|
+
const filter = filterStr ? JSON.parse(filterStr) : undefined;
|
|
332
|
+
const results = await db.search(query, { limit, collections, filter });
|
|
333
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
334
|
+
res.end(JSON.stringify(results));
|
|
335
|
+
return;
|
|
336
|
+
}
|
|
337
|
+
sendError(res, 404, `Endpoint ${req.method} ${pathname} not found.`);
|
|
338
|
+
return;
|
|
339
|
+
}
|
|
340
|
+
// REST API Routes (/v1/*)
|
|
341
|
+
if (pathname.startsWith("/v1/")) {
|
|
342
|
+
await handleRestRequest(db, req, res, pathname);
|
|
343
|
+
return;
|
|
344
|
+
}
|
|
345
|
+
// Static File Server
|
|
346
|
+
const targetFilePath = pathname === "/" ? "index.html" : pathname.replace(/^\//, "");
|
|
347
|
+
const fullFilePath = join(publicDir, targetFilePath);
|
|
348
|
+
// Security: ensure the resolved path is inside the public folder
|
|
349
|
+
if (!fullFilePath.startsWith(publicDir)) {
|
|
350
|
+
res.writeHead(403);
|
|
351
|
+
res.end("Forbidden");
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
354
|
+
if (existsSync(fullFilePath)) {
|
|
355
|
+
const fileContent = await fs.readFile(fullFilePath);
|
|
356
|
+
const ext = extname(fullFilePath).toLowerCase();
|
|
357
|
+
const contentType = MIME_TYPES[ext] || "application/octet-stream";
|
|
358
|
+
res.writeHead(200, { "Content-Type": contentType });
|
|
359
|
+
res.end(fileContent);
|
|
360
|
+
}
|
|
361
|
+
else {
|
|
362
|
+
res.writeHead(404);
|
|
363
|
+
res.end("Not Found");
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
catch (err) {
|
|
367
|
+
console.error("Dashboard server exception:", err);
|
|
368
|
+
sendError(res, 500, err instanceof Error ? err.message : String(err));
|
|
369
|
+
}
|
|
370
|
+
});
|
|
371
|
+
return new Promise((resolvePromise, rejectPromise) => {
|
|
372
|
+
server.listen(port, () => {
|
|
373
|
+
resolvePromise({
|
|
374
|
+
server,
|
|
375
|
+
close: async () => {
|
|
376
|
+
await new Promise((closeRes) => server.close(() => closeRes()));
|
|
377
|
+
await db.close();
|
|
378
|
+
},
|
|
379
|
+
});
|
|
380
|
+
});
|
|
381
|
+
server.on("error", (err) => {
|
|
382
|
+
rejectPromise(err);
|
|
383
|
+
});
|
|
384
|
+
});
|
|
385
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import { type CliContext } from "./index.js";
|
|
2
|
+
export declare function runExport(context: CliContext): Promise<void>;
|
|
3
|
+
export declare function runImport(context: CliContext): Promise<void>;
|
|
4
|
+
export declare function runSnapshot(context: CliContext): Promise<void>;
|
|
5
|
+
//# sourceMappingURL=data-movement.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"data-movement.d.ts","sourceRoot":"","sources":["../src/data-movement.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,KAAK,UAAU,EAAwD,MAAM,YAAY,CAAC;AA+DnG,wBAAsB,SAAS,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAyClE;AAED,wBAAsB,SAAS,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAuElE;AAED,wBAAsB,WAAW,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAoGpE"}
|
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { resolve } from "node:path";
|
|
3
|
+
import { hasFlag, requiredFlag, stringFlag, withDb, writeJson } from "./index.js";
|
|
4
|
+
const DEFAULT_REDACT_KEYS = [
|
|
5
|
+
"password",
|
|
6
|
+
"secret",
|
|
7
|
+
"token",
|
|
8
|
+
"key",
|
|
9
|
+
"auth",
|
|
10
|
+
"credential",
|
|
11
|
+
"email",
|
|
12
|
+
"phone",
|
|
13
|
+
"session",
|
|
14
|
+
"cookie",
|
|
15
|
+
"signature",
|
|
16
|
+
"private",
|
|
17
|
+
"cert",
|
|
18
|
+
"api",
|
|
19
|
+
];
|
|
20
|
+
function redactText(text) {
|
|
21
|
+
let res = text;
|
|
22
|
+
// Redact emails
|
|
23
|
+
res = res.replace(/[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g, "[REDACTED_EMAIL]");
|
|
24
|
+
// Redact API keys matching sk-...
|
|
25
|
+
res = res.replace(/sk-[a-zA-Z0-9]{20,}/g, "[REDACTED_KEY]");
|
|
26
|
+
// Redact Bearer tokens
|
|
27
|
+
res = res.replace(/bearer\s+[a-zA-Z0-9\-._~+/]+=*/gi, "Bearer [REDACTED]");
|
|
28
|
+
return res;
|
|
29
|
+
}
|
|
30
|
+
function redactValue(val, redactKeys) {
|
|
31
|
+
if (val === null || val === undefined) {
|
|
32
|
+
return val;
|
|
33
|
+
}
|
|
34
|
+
if (Array.isArray(val)) {
|
|
35
|
+
return val.map((item) => redactValue(item, redactKeys));
|
|
36
|
+
}
|
|
37
|
+
if (typeof val === "object") {
|
|
38
|
+
const obj = val;
|
|
39
|
+
const result = {};
|
|
40
|
+
for (const [k, v] of Object.entries(obj)) {
|
|
41
|
+
const lowerKey = k.toLowerCase();
|
|
42
|
+
const shouldRedact = redactKeys.some((rKey) => lowerKey.includes(rKey.toLowerCase()));
|
|
43
|
+
if (shouldRedact) {
|
|
44
|
+
result[k] = "[REDACTED]";
|
|
45
|
+
}
|
|
46
|
+
else if (v && typeof v === "object") {
|
|
47
|
+
result[k] = redactValue(v, redactKeys);
|
|
48
|
+
}
|
|
49
|
+
else if (typeof v === "string" && k === "text") {
|
|
50
|
+
result[k] = redactText(v);
|
|
51
|
+
}
|
|
52
|
+
else {
|
|
53
|
+
result[k] = v;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
return result;
|
|
57
|
+
}
|
|
58
|
+
return val;
|
|
59
|
+
}
|
|
60
|
+
export async function runExport(context) {
|
|
61
|
+
const isEvents = hasFlag(context.parsed, "events");
|
|
62
|
+
const collection = stringFlag(context.parsed, "collection");
|
|
63
|
+
const outPath = requiredFlag(context.parsed, "out");
|
|
64
|
+
if (!isEvents && !collection) {
|
|
65
|
+
throw new Error("Must specify either --collection <name> or --events for export.");
|
|
66
|
+
}
|
|
67
|
+
if (isEvents && collection) {
|
|
68
|
+
throw new Error("Cannot specify both --collection <name> and --events.");
|
|
69
|
+
}
|
|
70
|
+
await withDb(context, async (db) => {
|
|
71
|
+
const redactFlag = stringFlag(context.parsed, "redact");
|
|
72
|
+
const isRedact = hasFlag(context.parsed, "redact");
|
|
73
|
+
const redactKeys = isRedact
|
|
74
|
+
? redactFlag
|
|
75
|
+
? redactFlag.split(",").map((k) => k.trim())
|
|
76
|
+
: DEFAULT_REDACT_KEYS
|
|
77
|
+
: null;
|
|
78
|
+
let lines = [];
|
|
79
|
+
if (collection) {
|
|
80
|
+
const objects = await db.listObjects(collection);
|
|
81
|
+
lines = objects.map((obj) => {
|
|
82
|
+
const finalObj = redactKeys ? redactValue(obj, redactKeys) : obj;
|
|
83
|
+
return JSON.stringify(finalObj);
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
else {
|
|
87
|
+
const stream = stringFlag(context.parsed, "stream");
|
|
88
|
+
const events = await db.events.list(stream);
|
|
89
|
+
lines = events.map((ev) => {
|
|
90
|
+
const finalEv = redactKeys ? redactValue(ev, redactKeys) : ev;
|
|
91
|
+
return JSON.stringify(finalEv);
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
writeFileSync(resolve(outPath), `${lines.join("\n")}\n`, "utf8");
|
|
95
|
+
writeJson(context.stdout, { success: true, count: lines.length, out: outPath }, context.pretty);
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
export async function runImport(context) {
|
|
99
|
+
const collection = requiredFlag(context.parsed, "collection");
|
|
100
|
+
const inPath = requiredFlag(context.parsed, "in");
|
|
101
|
+
const resolvedPath = resolve(inPath);
|
|
102
|
+
if (!existsSync(resolvedPath)) {
|
|
103
|
+
throw new Error(`Input file not found: ${inPath}`);
|
|
104
|
+
}
|
|
105
|
+
const content = readFileSync(resolvedPath, "utf8");
|
|
106
|
+
const lines = content
|
|
107
|
+
.split("\n")
|
|
108
|
+
.map((l) => l.trim())
|
|
109
|
+
.filter(Boolean);
|
|
110
|
+
// Detect file type
|
|
111
|
+
const isCsv = inPath.endsWith(".csv");
|
|
112
|
+
await withDb(context, async (db) => {
|
|
113
|
+
let count = 0;
|
|
114
|
+
if (isCsv) {
|
|
115
|
+
// Parse CSV
|
|
116
|
+
const headers = lines[0]?.split(",").map((h) => h.trim()) ?? [];
|
|
117
|
+
for (let i = 1; i < lines.length; i++) {
|
|
118
|
+
const values = lines[i]?.split(",") ?? [];
|
|
119
|
+
const obj = {};
|
|
120
|
+
for (let j = 0; j < headers.length; j++) {
|
|
121
|
+
const key = headers[j];
|
|
122
|
+
if (!key) {
|
|
123
|
+
continue;
|
|
124
|
+
}
|
|
125
|
+
const val = values[j]?.trim() ?? "";
|
|
126
|
+
// Try to infer types
|
|
127
|
+
if (val === "" || val === "null") {
|
|
128
|
+
obj[key] = null;
|
|
129
|
+
}
|
|
130
|
+
else if (val === "true") {
|
|
131
|
+
obj[key] = true;
|
|
132
|
+
}
|
|
133
|
+
else if (val === "false") {
|
|
134
|
+
obj[key] = false;
|
|
135
|
+
}
|
|
136
|
+
else if (!Number.isNaN(Number(val))) {
|
|
137
|
+
obj[key] = Number(val);
|
|
138
|
+
}
|
|
139
|
+
else {
|
|
140
|
+
obj[key] = val;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
// Add row index as ID if not present
|
|
144
|
+
if (!obj.id) {
|
|
145
|
+
obj.id = `row-${i}`;
|
|
146
|
+
}
|
|
147
|
+
await db.put(collection, obj);
|
|
148
|
+
count += 1;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
else {
|
|
152
|
+
// Parse JSONL
|
|
153
|
+
for (const line of lines) {
|
|
154
|
+
const parsedObj = JSON.parse(line);
|
|
155
|
+
if (!parsedObj.id || typeof parsedObj.id !== "string") {
|
|
156
|
+
throw new Error("Imported object must contain a string 'id' field.");
|
|
157
|
+
}
|
|
158
|
+
await db.put(collection, parsedObj);
|
|
159
|
+
count += 1;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
writeJson(context.stdout, { success: true, count, collection, format: isCsv ? "csv" : "jsonl" }, context.pretty);
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
export async function runSnapshot(context) {
|
|
166
|
+
const subCommand = context.parsed.tokens[1];
|
|
167
|
+
if (subCommand === "create") {
|
|
168
|
+
const outPath = requiredFlag(context.parsed, "out");
|
|
169
|
+
await withDb(context, async (db) => {
|
|
170
|
+
const collectionsMap = {};
|
|
171
|
+
const cols = await db.listCollections();
|
|
172
|
+
for (const col of cols) {
|
|
173
|
+
collectionsMap[col] = await db.listObjects(col);
|
|
174
|
+
}
|
|
175
|
+
const eventsList = await db.events.list();
|
|
176
|
+
const queuesMap = {};
|
|
177
|
+
const queues = await db.listQueues();
|
|
178
|
+
for (const q of queues) {
|
|
179
|
+
const queue = db.queue(q);
|
|
180
|
+
const [active, dead] = await Promise.all([queue.list(), queue.dead()]);
|
|
181
|
+
queuesMap[q] = { active, dead };
|
|
182
|
+
}
|
|
183
|
+
const snapshot = {
|
|
184
|
+
version: "1.0.0",
|
|
185
|
+
timestamp: new Date().toISOString(),
|
|
186
|
+
collections: collectionsMap,
|
|
187
|
+
events: eventsList,
|
|
188
|
+
queues: queuesMap,
|
|
189
|
+
};
|
|
190
|
+
writeFileSync(resolve(outPath), JSON.stringify(snapshot, null, 2), "utf8");
|
|
191
|
+
writeJson(context.stdout, { success: true, out: outPath }, context.pretty);
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
else if (subCommand === "restore") {
|
|
195
|
+
const inPath = requiredFlag(context.parsed, "in");
|
|
196
|
+
const resolvedPath = resolve(inPath);
|
|
197
|
+
if (!existsSync(resolvedPath)) {
|
|
198
|
+
throw new Error(`Snapshot file not found: ${inPath}`);
|
|
199
|
+
}
|
|
200
|
+
const snapshot = JSON.parse(readFileSync(resolvedPath, "utf8"));
|
|
201
|
+
if (snapshot.version !== "1.0.0") {
|
|
202
|
+
throw new Error(`Unsupported snapshot version: ${snapshot.version}`);
|
|
203
|
+
}
|
|
204
|
+
await withDb(context, async (db) => {
|
|
205
|
+
// 1. Restore Collections (clear existing first for true restore)
|
|
206
|
+
if (snapshot.collections) {
|
|
207
|
+
for (const [colName, objects] of Object.entries(snapshot.collections)) {
|
|
208
|
+
const currentObjs = await db.listObjects(colName);
|
|
209
|
+
for (const obj of currentObjs) {
|
|
210
|
+
await db.delete(colName, obj.id);
|
|
211
|
+
}
|
|
212
|
+
for (const obj of objects) {
|
|
213
|
+
const cleanObj = { ...obj };
|
|
214
|
+
delete cleanObj.collection;
|
|
215
|
+
delete cleanObj.createdAt;
|
|
216
|
+
delete cleanObj.updatedAt;
|
|
217
|
+
delete cleanObj.version;
|
|
218
|
+
await db.put(colName, cleanObj);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
// 2. Restore Events
|
|
223
|
+
if (snapshot.events) {
|
|
224
|
+
for (const ev of snapshot.events) {
|
|
225
|
+
const cleanEv = { ...ev };
|
|
226
|
+
delete cleanEv.id;
|
|
227
|
+
delete cleanEv.createdAt;
|
|
228
|
+
delete cleanEv.stream;
|
|
229
|
+
await db.events.append(ev.stream, cleanEv);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
// 3. Restore Queues
|
|
233
|
+
if (snapshot.queues) {
|
|
234
|
+
for (const [qName, jobsData] of Object.entries(snapshot.queues)) {
|
|
235
|
+
const queue = db.queue(qName);
|
|
236
|
+
const { active, dead } = jobsData;
|
|
237
|
+
for (const job of active) {
|
|
238
|
+
await queue.push(job.payload, {
|
|
239
|
+
idempotencyKey: job.id,
|
|
240
|
+
maxAttempts: job.maxAttempts,
|
|
241
|
+
});
|
|
242
|
+
}
|
|
243
|
+
for (const job of dead) {
|
|
244
|
+
await queue.push(job.payload, {
|
|
245
|
+
idempotencyKey: job.id,
|
|
246
|
+
maxAttempts: job.maxAttempts,
|
|
247
|
+
});
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
writeJson(context.stdout, { success: true, in: inPath }, context.pretty);
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
else {
|
|
255
|
+
throw new Error(`Unknown snapshot command: ${subCommand}. Expected 'create' or 'restore'.`);
|
|
256
|
+
}
|
|
257
|
+
}
|
package/dist/doctor.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"doctor.d.ts","sourceRoot":"","sources":["../src/doctor.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,UAAU,EAAqB,MAAM,YAAY,CAAC;AAEhE,wBAAsB,SAAS,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAwIlE"}
|