@deeplake/hivemind 0.6.47

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,862 @@
1
+ #!/usr/bin/env node
2
+
3
+ // dist/src/commands/auth.js
4
+ import { readFileSync, writeFileSync, existsSync, mkdirSync, unlinkSync } from "node:fs";
5
+ import { join } from "node:path";
6
+ import { homedir } from "node:os";
7
+ import { execSync } from "node:child_process";
8
+ var CONFIG_DIR = join(homedir(), ".deeplake");
9
+ var CREDS_PATH = join(CONFIG_DIR, "credentials.json");
10
+ var DEFAULT_API_URL = "https://api.deeplake.ai";
11
+ function loadCredentials() {
12
+ if (!existsSync(CREDS_PATH))
13
+ return null;
14
+ try {
15
+ return JSON.parse(readFileSync(CREDS_PATH, "utf-8"));
16
+ } catch {
17
+ return null;
18
+ }
19
+ }
20
+ function saveCredentials(creds) {
21
+ if (!existsSync(CONFIG_DIR))
22
+ mkdirSync(CONFIG_DIR, { recursive: true, mode: 448 });
23
+ writeFileSync(CREDS_PATH, JSON.stringify({ ...creds, savedAt: (/* @__PURE__ */ new Date()).toISOString() }, null, 2), { mode: 384 });
24
+ }
25
+ function deleteCredentials() {
26
+ if (existsSync(CREDS_PATH)) {
27
+ unlinkSync(CREDS_PATH);
28
+ return true;
29
+ }
30
+ return false;
31
+ }
32
+ async function apiGet(path, token, apiUrl, orgId) {
33
+ const headers = {
34
+ Authorization: `Bearer ${token}`,
35
+ "Content-Type": "application/json"
36
+ };
37
+ if (orgId)
38
+ headers["X-Activeloop-Org-Id"] = orgId;
39
+ const resp = await fetch(`${apiUrl}${path}`, { headers });
40
+ if (!resp.ok)
41
+ throw new Error(`API ${resp.status}: ${await resp.text().catch(() => "")}`);
42
+ return resp.json();
43
+ }
44
+ async function apiPost(path, body, token, apiUrl, orgId) {
45
+ const headers = {
46
+ Authorization: `Bearer ${token}`,
47
+ "Content-Type": "application/json"
48
+ };
49
+ if (orgId)
50
+ headers["X-Activeloop-Org-Id"] = orgId;
51
+ const resp = await fetch(`${apiUrl}${path}`, { method: "POST", headers, body: JSON.stringify(body) });
52
+ if (!resp.ok)
53
+ throw new Error(`API ${resp.status}: ${await resp.text().catch(() => "")}`);
54
+ return resp.json();
55
+ }
56
+ async function apiDelete(path, token, apiUrl, orgId) {
57
+ const headers = {
58
+ Authorization: `Bearer ${token}`,
59
+ "Content-Type": "application/json"
60
+ };
61
+ if (orgId)
62
+ headers["X-Activeloop-Org-Id"] = orgId;
63
+ const resp = await fetch(`${apiUrl}${path}`, { method: "DELETE", headers });
64
+ if (!resp.ok)
65
+ throw new Error(`API ${resp.status}: ${await resp.text().catch(() => "")}`);
66
+ }
67
+ async function requestDeviceCode(apiUrl = DEFAULT_API_URL) {
68
+ const resp = await fetch(`${apiUrl}/auth/device/code`, {
69
+ method: "POST",
70
+ headers: { "Content-Type": "application/json" }
71
+ });
72
+ if (!resp.ok)
73
+ throw new Error(`Device flow unavailable: HTTP ${resp.status}`);
74
+ return resp.json();
75
+ }
76
+ async function pollForToken(deviceCode, apiUrl = DEFAULT_API_URL) {
77
+ const resp = await fetch(`${apiUrl}/auth/device/token`, {
78
+ method: "POST",
79
+ headers: { "Content-Type": "application/json" },
80
+ body: JSON.stringify({ device_code: deviceCode })
81
+ });
82
+ if (resp.ok)
83
+ return resp.json();
84
+ if (resp.status === 400) {
85
+ const err = await resp.json().catch(() => null);
86
+ if (err?.error === "authorization_pending" || err?.error === "slow_down")
87
+ return null;
88
+ if (err?.error === "expired_token")
89
+ throw new Error("Device code expired. Try again.");
90
+ if (err?.error === "access_denied")
91
+ throw new Error("Authorization denied.");
92
+ }
93
+ throw new Error(`Token polling failed: HTTP ${resp.status}`);
94
+ }
95
+ function openBrowser(url) {
96
+ try {
97
+ const cmd = process.platform === "darwin" ? `open "${url}"` : process.platform === "win32" ? `start "${url}"` : `xdg-open "${url}" 2>/dev/null`;
98
+ execSync(cmd, { stdio: "ignore", timeout: 5e3 });
99
+ return true;
100
+ } catch {
101
+ return false;
102
+ }
103
+ }
104
+ async function deviceFlowLogin(apiUrl = DEFAULT_API_URL) {
105
+ const code = await requestDeviceCode(apiUrl);
106
+ const opened = openBrowser(code.verification_uri_complete);
107
+ const msg = [
108
+ "\nDeeplake Authentication",
109
+ "\u2500".repeat(40),
110
+ `
111
+ Open this URL: ${code.verification_uri_complete}`,
112
+ `Or visit ${code.verification_uri} and enter code: ${code.user_code}`,
113
+ opened ? "\nBrowser opened. Waiting for sign in..." : "\nWaiting for sign in..."
114
+ ].join("\n");
115
+ process.stderr.write(msg + "\n");
116
+ const interval = Math.max(code.interval || 5, 5) * 1e3;
117
+ const deadline = Date.now() + code.expires_in * 1e3;
118
+ while (Date.now() < deadline) {
119
+ await new Promise((r) => setTimeout(r, interval));
120
+ const result = await pollForToken(code.device_code, apiUrl);
121
+ if (result) {
122
+ process.stderr.write("\nAuthentication successful!\n");
123
+ return { token: result.access_token, expiresIn: result.expires_in };
124
+ }
125
+ }
126
+ throw new Error("Device code expired.");
127
+ }
128
+ async function listOrgs(token, apiUrl = DEFAULT_API_URL) {
129
+ const data = await apiGet("/organizations", token, apiUrl);
130
+ return Array.isArray(data) ? data : [];
131
+ }
132
+ async function switchOrg(orgId, orgName) {
133
+ const creds = loadCredentials();
134
+ if (!creds)
135
+ throw new Error("Not logged in. Run deeplake login first.");
136
+ saveCredentials({ ...creds, orgId, orgName });
137
+ }
138
+ async function listWorkspaces(token, apiUrl = DEFAULT_API_URL, orgId) {
139
+ const raw = await apiGet("/workspaces", token, apiUrl, orgId);
140
+ const data = raw.data ?? raw;
141
+ return Array.isArray(data) ? data : [];
142
+ }
143
+ async function switchWorkspace(workspaceId) {
144
+ const creds = loadCredentials();
145
+ if (!creds)
146
+ throw new Error("Not logged in. Run deeplake login first.");
147
+ saveCredentials({ ...creds, workspaceId });
148
+ }
149
+ async function inviteMember(username, accessMode, token, orgId, apiUrl = DEFAULT_API_URL) {
150
+ await apiPost(`/organizations/${orgId}/members/invite`, { username, access_mode: accessMode }, token, apiUrl, orgId);
151
+ }
152
+ async function listMembers(token, orgId, apiUrl = DEFAULT_API_URL) {
153
+ const data = await apiGet(`/organizations/${orgId}/members`, token, apiUrl, orgId);
154
+ return data.members ?? [];
155
+ }
156
+ async function removeMember(userId, token, orgId, apiUrl = DEFAULT_API_URL) {
157
+ await apiDelete(`/organizations/${orgId}/members/${userId}`, token, apiUrl, orgId);
158
+ }
159
+ async function login(apiUrl = DEFAULT_API_URL) {
160
+ const { token: authToken } = await deviceFlowLogin(apiUrl);
161
+ const user = await apiGet("/me", authToken, apiUrl);
162
+ const userName = user.name || (user.email ? user.email.split("@")[0] : "unknown");
163
+ process.stderr.write(`
164
+ Logged in as: ${userName}
165
+ `);
166
+ const orgs = await listOrgs(authToken, apiUrl);
167
+ let orgId;
168
+ let orgName;
169
+ if (orgs.length === 1) {
170
+ orgId = orgs[0].id;
171
+ orgName = orgs[0].name;
172
+ process.stderr.write(`Organization: ${orgName}
173
+ `);
174
+ } else {
175
+ process.stderr.write("\nOrganizations:\n");
176
+ orgs.forEach((org, i) => process.stderr.write(` ${i + 1}. ${org.name}
177
+ `));
178
+ orgId = orgs[0].id;
179
+ orgName = orgs[0].name;
180
+ process.stderr.write(`
181
+ Using: ${orgName}
182
+ `);
183
+ }
184
+ const tokenName = `deeplake-plugin-${(/* @__PURE__ */ new Date()).toISOString().slice(0, 10)}`;
185
+ const tokenData = await apiPost("/users/me/tokens", {
186
+ name: tokenName,
187
+ duration: 365 * 24 * 3600,
188
+ organization_id: orgId
189
+ }, authToken, apiUrl);
190
+ const apiToken = tokenData.token.token;
191
+ const creds = {
192
+ token: apiToken,
193
+ orgId,
194
+ orgName,
195
+ userName,
196
+ workspaceId: "default",
197
+ apiUrl,
198
+ savedAt: (/* @__PURE__ */ new Date()).toISOString()
199
+ };
200
+ saveCredentials(creds);
201
+ return creds;
202
+ }
203
+
204
+ // dist/src/config.js
205
+ import { readFileSync as readFileSync2, existsSync as existsSync2 } from "node:fs";
206
+ import { join as join2 } from "node:path";
207
+ import { homedir as homedir2, userInfo } from "node:os";
208
+ function loadConfig() {
209
+ const home = homedir2();
210
+ const credPath = join2(home, ".deeplake", "credentials.json");
211
+ let creds = null;
212
+ if (existsSync2(credPath)) {
213
+ try {
214
+ creds = JSON.parse(readFileSync2(credPath, "utf-8"));
215
+ } catch {
216
+ return null;
217
+ }
218
+ }
219
+ const token = process.env.HIVEMIND_TOKEN ?? creds?.token;
220
+ const orgId = process.env.HIVEMIND_ORG_ID ?? creds?.orgId;
221
+ if (!token || !orgId)
222
+ return null;
223
+ return {
224
+ token,
225
+ orgId,
226
+ orgName: creds?.orgName ?? orgId,
227
+ userName: creds?.userName || userInfo().username || "unknown",
228
+ workspaceId: process.env.HIVEMIND_WORKSPACE_ID ?? creds?.workspaceId ?? "default",
229
+ apiUrl: process.env.HIVEMIND_API_URL ?? creds?.apiUrl ?? "https://api.deeplake.ai",
230
+ tableName: process.env.HIVEMIND_TABLE ?? "memory",
231
+ sessionsTableName: process.env.HIVEMIND_SESSIONS_TABLE ?? "sessions",
232
+ memoryPath: process.env.HIVEMIND_MEMORY_PATH ?? join2(home, ".deeplake", "memory")
233
+ };
234
+ }
235
+
236
+ // dist/src/deeplake-api.js
237
+ import { randomUUID } from "node:crypto";
238
+ import { existsSync as existsSync3, mkdirSync as mkdirSync2, readFileSync as readFileSync3, writeFileSync as writeFileSync2 } from "node:fs";
239
+ import { join as join4 } from "node:path";
240
+ import { tmpdir } from "node:os";
241
+
242
+ // dist/src/utils/debug.js
243
+ import { appendFileSync } from "node:fs";
244
+ import { join as join3 } from "node:path";
245
+ import { homedir as homedir3 } from "node:os";
246
+ var DEBUG = process.env.HIVEMIND_DEBUG === "1";
247
+ var LOG = join3(homedir3(), ".deeplake", "hook-debug.log");
248
+ function log(tag, msg) {
249
+ if (!DEBUG)
250
+ return;
251
+ appendFileSync(LOG, `${(/* @__PURE__ */ new Date()).toISOString()} [${tag}] ${msg}
252
+ `);
253
+ }
254
+
255
+ // dist/src/utils/sql.js
256
+ function sqlStr(value) {
257
+ return value.replace(/\\/g, "\\\\").replace(/'/g, "''").replace(/\0/g, "").replace(/[\x01-\x08\x0b\x0c\x0e-\x1f\x7f]/g, "");
258
+ }
259
+
260
+ // dist/src/deeplake-api.js
261
+ var log2 = (msg) => log("sdk", msg);
262
+ function summarizeSql(sql, maxLen = 220) {
263
+ const compact = sql.replace(/\s+/g, " ").trim();
264
+ return compact.length > maxLen ? `${compact.slice(0, maxLen)}...` : compact;
265
+ }
266
+ function traceSql(msg) {
267
+ const traceEnabled = process.env.HIVEMIND_TRACE_SQL === "1" || process.env.HIVEMIND_DEBUG === "1";
268
+ if (!traceEnabled)
269
+ return;
270
+ process.stderr.write(`[deeplake-sql] ${msg}
271
+ `);
272
+ if (process.env.HIVEMIND_DEBUG === "1")
273
+ log2(msg);
274
+ }
275
+ var RETRYABLE_CODES = /* @__PURE__ */ new Set([429, 500, 502, 503, 504]);
276
+ var MAX_RETRIES = 3;
277
+ var BASE_DELAY_MS = 500;
278
+ var MAX_CONCURRENCY = 5;
279
+ var QUERY_TIMEOUT_MS = Number(process.env.HIVEMIND_QUERY_TIMEOUT_MS ?? 1e4);
280
+ var INDEX_MARKER_TTL_MS = Number(process.env.HIVEMIND_INDEX_MARKER_TTL_MS ?? 6 * 60 * 6e4);
281
+ function sleep(ms) {
282
+ return new Promise((resolve) => setTimeout(resolve, ms));
283
+ }
284
+ function isTimeoutError(error) {
285
+ const name = error instanceof Error ? error.name.toLowerCase() : "";
286
+ const message = error instanceof Error ? error.message.toLowerCase() : String(error).toLowerCase();
287
+ return name.includes("timeout") || name === "aborterror" || message.includes("timeout") || message.includes("timed out");
288
+ }
289
+ function isDuplicateIndexError(error) {
290
+ const message = error instanceof Error ? error.message.toLowerCase() : String(error).toLowerCase();
291
+ return message.includes("duplicate key value violates unique constraint") || message.includes("pg_class_relname_nsp_index") || message.includes("already exists");
292
+ }
293
+ function isSessionInsertQuery(sql) {
294
+ return /^\s*insert\s+into\s+"[^"]+"\s*\(\s*id\s*,\s*path\s*,\s*filename\s*,\s*message\s*,/i.test(sql);
295
+ }
296
+ function isTransientHtml403(text) {
297
+ const body = text.toLowerCase();
298
+ return body.includes("<html") || body.includes("403 forbidden") || body.includes("cloudflare") || body.includes("nginx");
299
+ }
300
+ function getIndexMarkerDir() {
301
+ return process.env.HIVEMIND_INDEX_MARKER_DIR ?? join4(tmpdir(), "hivemind-deeplake-indexes");
302
+ }
303
+ var Semaphore = class {
304
+ max;
305
+ waiting = [];
306
+ active = 0;
307
+ constructor(max) {
308
+ this.max = max;
309
+ }
310
+ async acquire() {
311
+ if (this.active < this.max) {
312
+ this.active++;
313
+ return;
314
+ }
315
+ await new Promise((resolve) => this.waiting.push(resolve));
316
+ }
317
+ release() {
318
+ this.active--;
319
+ const next = this.waiting.shift();
320
+ if (next) {
321
+ this.active++;
322
+ next();
323
+ }
324
+ }
325
+ };
326
+ var DeeplakeApi = class {
327
+ token;
328
+ apiUrl;
329
+ orgId;
330
+ workspaceId;
331
+ tableName;
332
+ _pendingRows = [];
333
+ _sem = new Semaphore(MAX_CONCURRENCY);
334
+ _tablesCache = null;
335
+ constructor(token, apiUrl, orgId, workspaceId, tableName) {
336
+ this.token = token;
337
+ this.apiUrl = apiUrl;
338
+ this.orgId = orgId;
339
+ this.workspaceId = workspaceId;
340
+ this.tableName = tableName;
341
+ }
342
+ /** Execute SQL with retry on transient errors and bounded concurrency. */
343
+ async query(sql) {
344
+ const startedAt = Date.now();
345
+ const summary = summarizeSql(sql);
346
+ traceSql(`query start: ${summary}`);
347
+ await this._sem.acquire();
348
+ try {
349
+ const rows = await this._queryWithRetry(sql);
350
+ traceSql(`query ok (${Date.now() - startedAt}ms, rows=${rows.length}): ${summary}`);
351
+ return rows;
352
+ } catch (e) {
353
+ const message = e instanceof Error ? e.message : String(e);
354
+ traceSql(`query fail (${Date.now() - startedAt}ms): ${summary} :: ${message}`);
355
+ throw e;
356
+ } finally {
357
+ this._sem.release();
358
+ }
359
+ }
360
+ async _queryWithRetry(sql) {
361
+ let lastError;
362
+ for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
363
+ let resp;
364
+ try {
365
+ const signal = AbortSignal.timeout(QUERY_TIMEOUT_MS);
366
+ resp = await fetch(`${this.apiUrl}/workspaces/${this.workspaceId}/tables/query`, {
367
+ method: "POST",
368
+ headers: {
369
+ Authorization: `Bearer ${this.token}`,
370
+ "Content-Type": "application/json",
371
+ "X-Activeloop-Org-Id": this.orgId
372
+ },
373
+ signal,
374
+ body: JSON.stringify({ query: sql })
375
+ });
376
+ } catch (e) {
377
+ if (isTimeoutError(e)) {
378
+ lastError = new Error(`Query timeout after ${QUERY_TIMEOUT_MS}ms`);
379
+ throw lastError;
380
+ }
381
+ lastError = e instanceof Error ? e : new Error(String(e));
382
+ if (attempt < MAX_RETRIES) {
383
+ const delay = BASE_DELAY_MS * Math.pow(2, attempt) + Math.random() * 200;
384
+ log2(`query retry ${attempt + 1}/${MAX_RETRIES} (fetch error: ${lastError.message}) in ${delay.toFixed(0)}ms`);
385
+ await sleep(delay);
386
+ continue;
387
+ }
388
+ throw lastError;
389
+ }
390
+ if (resp.ok) {
391
+ const raw = await resp.json();
392
+ if (!raw?.rows || !raw?.columns)
393
+ return [];
394
+ return raw.rows.map((row) => Object.fromEntries(raw.columns.map((col, i) => [col, row[i]])));
395
+ }
396
+ const text = await resp.text().catch(() => "");
397
+ const retryable403 = isSessionInsertQuery(sql) && (resp.status === 401 || resp.status === 403 && (text.length === 0 || isTransientHtml403(text)));
398
+ if (attempt < MAX_RETRIES && (RETRYABLE_CODES.has(resp.status) || retryable403)) {
399
+ const delay = BASE_DELAY_MS * Math.pow(2, attempt) + Math.random() * 200;
400
+ log2(`query retry ${attempt + 1}/${MAX_RETRIES} (${resp.status}) in ${delay.toFixed(0)}ms`);
401
+ await sleep(delay);
402
+ continue;
403
+ }
404
+ throw new Error(`Query failed: ${resp.status}: ${text.slice(0, 200)}`);
405
+ }
406
+ throw lastError ?? new Error("Query failed: max retries exceeded");
407
+ }
408
+ // ── Writes ──────────────────────────────────────────────────────────────────
409
+ /** Queue rows for writing. Call commit() to flush. */
410
+ appendRows(rows) {
411
+ this._pendingRows.push(...rows);
412
+ }
413
+ /** Flush pending rows via SQL. */
414
+ async commit() {
415
+ if (this._pendingRows.length === 0)
416
+ return;
417
+ const rows = this._pendingRows;
418
+ this._pendingRows = [];
419
+ const CONCURRENCY = 10;
420
+ for (let i = 0; i < rows.length; i += CONCURRENCY) {
421
+ const chunk = rows.slice(i, i + CONCURRENCY);
422
+ await Promise.allSettled(chunk.map((r) => this.upsertRowSql(r)));
423
+ }
424
+ log2(`commit: ${rows.length} rows`);
425
+ }
426
+ async upsertRowSql(row) {
427
+ const ts = (/* @__PURE__ */ new Date()).toISOString();
428
+ const cd = row.creationDate ?? ts;
429
+ const lud = row.lastUpdateDate ?? ts;
430
+ const exists = await this.query(`SELECT path FROM "${this.tableName}" WHERE path = '${sqlStr(row.path)}' LIMIT 1`);
431
+ if (exists.length > 0) {
432
+ let setClauses = `summary = E'${sqlStr(row.contentText)}', mime_type = '${sqlStr(row.mimeType)}', size_bytes = ${row.sizeBytes}, last_update_date = '${lud}'`;
433
+ if (row.project !== void 0)
434
+ setClauses += `, project = '${sqlStr(row.project)}'`;
435
+ if (row.description !== void 0)
436
+ setClauses += `, description = '${sqlStr(row.description)}'`;
437
+ await this.query(`UPDATE "${this.tableName}" SET ${setClauses} WHERE path = '${sqlStr(row.path)}'`);
438
+ } else {
439
+ const id = randomUUID();
440
+ let cols = "id, path, filename, summary, mime_type, size_bytes, creation_date, last_update_date";
441
+ let vals = `'${id}', '${sqlStr(row.path)}', '${sqlStr(row.filename)}', E'${sqlStr(row.contentText)}', '${sqlStr(row.mimeType)}', ${row.sizeBytes}, '${cd}', '${lud}'`;
442
+ if (row.project !== void 0) {
443
+ cols += ", project";
444
+ vals += `, '${sqlStr(row.project)}'`;
445
+ }
446
+ if (row.description !== void 0) {
447
+ cols += ", description";
448
+ vals += `, '${sqlStr(row.description)}'`;
449
+ }
450
+ await this.query(`INSERT INTO "${this.tableName}" (${cols}) VALUES (${vals})`);
451
+ }
452
+ }
453
+ /** Update specific columns on a row by path. */
454
+ async updateColumns(path, columns) {
455
+ const setClauses = Object.entries(columns).map(([col, val]) => typeof val === "number" ? `${col} = ${val}` : `${col} = '${sqlStr(String(val))}'`).join(", ");
456
+ await this.query(`UPDATE "${this.tableName}" SET ${setClauses} WHERE path = '${sqlStr(path)}'`);
457
+ }
458
+ // ── Convenience ─────────────────────────────────────────────────────────────
459
+ /** Create a BM25 search index on a column. */
460
+ async createIndex(column) {
461
+ await this.query(`CREATE INDEX IF NOT EXISTS idx_${sqlStr(column)}_bm25 ON "${this.tableName}" USING deeplake_index ("${column}")`);
462
+ }
463
+ buildLookupIndexName(table, suffix) {
464
+ return `idx_${table}_${suffix}`.replace(/[^a-zA-Z0-9_]/g, "_");
465
+ }
466
+ getLookupIndexMarkerPath(table, suffix) {
467
+ const markerKey = [
468
+ this.workspaceId,
469
+ this.orgId,
470
+ table,
471
+ suffix
472
+ ].join("__").replace(/[^a-zA-Z0-9_.-]/g, "_");
473
+ return join4(getIndexMarkerDir(), `${markerKey}.json`);
474
+ }
475
+ hasFreshLookupIndexMarker(table, suffix) {
476
+ const markerPath = this.getLookupIndexMarkerPath(table, suffix);
477
+ if (!existsSync3(markerPath))
478
+ return false;
479
+ try {
480
+ const raw = JSON.parse(readFileSync3(markerPath, "utf-8"));
481
+ const updatedAt = raw.updatedAt ? new Date(raw.updatedAt).getTime() : NaN;
482
+ if (!Number.isFinite(updatedAt) || Date.now() - updatedAt > INDEX_MARKER_TTL_MS)
483
+ return false;
484
+ return true;
485
+ } catch {
486
+ return false;
487
+ }
488
+ }
489
+ markLookupIndexReady(table, suffix) {
490
+ mkdirSync2(getIndexMarkerDir(), { recursive: true });
491
+ writeFileSync2(this.getLookupIndexMarkerPath(table, suffix), JSON.stringify({ updatedAt: (/* @__PURE__ */ new Date()).toISOString() }), "utf-8");
492
+ }
493
+ async ensureLookupIndex(table, suffix, columnsSql) {
494
+ if (this.hasFreshLookupIndexMarker(table, suffix))
495
+ return;
496
+ const indexName = this.buildLookupIndexName(table, suffix);
497
+ try {
498
+ await this.query(`CREATE INDEX IF NOT EXISTS "${indexName}" ON "${table}" ${columnsSql}`);
499
+ this.markLookupIndexReady(table, suffix);
500
+ } catch (e) {
501
+ if (isDuplicateIndexError(e)) {
502
+ this.markLookupIndexReady(table, suffix);
503
+ return;
504
+ }
505
+ log2(`index "${indexName}" skipped: ${e.message}`);
506
+ }
507
+ }
508
+ /** List all tables in the workspace (with retry). */
509
+ async listTables(forceRefresh = false) {
510
+ if (!forceRefresh && this._tablesCache)
511
+ return [...this._tablesCache];
512
+ const { tables, cacheable } = await this._fetchTables();
513
+ if (cacheable)
514
+ this._tablesCache = [...tables];
515
+ return tables;
516
+ }
517
+ async _fetchTables() {
518
+ for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
519
+ try {
520
+ const resp = await fetch(`${this.apiUrl}/workspaces/${this.workspaceId}/tables`, {
521
+ headers: {
522
+ Authorization: `Bearer ${this.token}`,
523
+ "X-Activeloop-Org-Id": this.orgId
524
+ }
525
+ });
526
+ if (resp.ok) {
527
+ const data = await resp.json();
528
+ return {
529
+ tables: (data.tables ?? []).map((t) => t.table_name),
530
+ cacheable: true
531
+ };
532
+ }
533
+ if (attempt < MAX_RETRIES && RETRYABLE_CODES.has(resp.status)) {
534
+ await sleep(BASE_DELAY_MS * Math.pow(2, attempt) + Math.random() * 200);
535
+ continue;
536
+ }
537
+ return { tables: [], cacheable: false };
538
+ } catch {
539
+ if (attempt < MAX_RETRIES) {
540
+ await sleep(BASE_DELAY_MS * Math.pow(2, attempt));
541
+ continue;
542
+ }
543
+ return { tables: [], cacheable: false };
544
+ }
545
+ }
546
+ return { tables: [], cacheable: false };
547
+ }
548
+ /** Create the memory table if it doesn't already exist. Migrate columns on existing tables. */
549
+ async ensureTable(name) {
550
+ const tbl = name ?? this.tableName;
551
+ const tables = await this.listTables();
552
+ if (!tables.includes(tbl)) {
553
+ log2(`table "${tbl}" not found, creating`);
554
+ await this.query(`CREATE TABLE IF NOT EXISTS "${tbl}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'text/plain', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`);
555
+ log2(`table "${tbl}" created`);
556
+ if (!tables.includes(tbl))
557
+ this._tablesCache = [...tables, tbl];
558
+ }
559
+ }
560
+ /** Create the sessions table (uses JSONB for message since every row is a JSON event). */
561
+ async ensureSessionsTable(name) {
562
+ const tables = await this.listTables();
563
+ if (!tables.includes(name)) {
564
+ log2(`table "${name}" not found, creating`);
565
+ await this.query(`CREATE TABLE IF NOT EXISTS "${name}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', message JSONB, author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'application/json', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`);
566
+ log2(`table "${name}" created`);
567
+ if (!tables.includes(name))
568
+ this._tablesCache = [...tables, name];
569
+ }
570
+ await this.ensureLookupIndex(name, "path_creation_date", `("path", "creation_date")`);
571
+ }
572
+ };
573
+
574
+ // dist/src/commands/session-prune.js
575
+ import { createInterface } from "node:readline";
576
+ function parseArgs(argv) {
577
+ let before;
578
+ let sessionId;
579
+ let all = false;
580
+ let yes = false;
581
+ for (let i = 0; i < argv.length; i++) {
582
+ const arg = argv[i];
583
+ if (arg === "--before" && argv[i + 1]) {
584
+ before = argv[++i];
585
+ } else if (arg === "--session-id" && argv[i + 1]) {
586
+ sessionId = argv[++i];
587
+ } else if (arg === "--all") {
588
+ all = true;
589
+ } else if (arg === "--yes" || arg === "-y") {
590
+ yes = true;
591
+ }
592
+ }
593
+ return { before, sessionId, all, yes };
594
+ }
595
+ function confirm(message) {
596
+ const rl = createInterface({ input: process.stdin, output: process.stderr });
597
+ return new Promise((resolve) => {
598
+ rl.question(`${message} [y/N] `, (answer) => {
599
+ rl.close();
600
+ resolve(answer.trim().toLowerCase() === "y");
601
+ });
602
+ });
603
+ }
604
+ function extractSessionId(path) {
605
+ const m = path.match(/\/sessions\/[^/]+\/[^/]+_([^.]+)\.jsonl$/);
606
+ return m ? m[1] : path.split("/").pop()?.replace(/\.jsonl$/, "") ?? path;
607
+ }
608
+ async function listSessions(api, sessionsTable, author) {
609
+ const rows = await api.query(`SELECT path, COUNT(*) as cnt, MIN(creation_date) as first_event, MAX(creation_date) as last_event, MAX(project) as project FROM "${sessionsTable}" WHERE author = '${sqlStr(author)}' GROUP BY path ORDER BY first_event DESC`);
610
+ return rows.map((r) => ({
611
+ path: String(r.path),
612
+ rowCount: Number(r.cnt),
613
+ firstEvent: String(r.first_event),
614
+ lastEvent: String(r.last_event),
615
+ project: String(r.project ?? "")
616
+ }));
617
+ }
618
+ async function deleteSessions(config, sessionPaths) {
619
+ if (sessionPaths.length === 0)
620
+ return { sessionsDeleted: 0, summariesDeleted: 0 };
621
+ const sessionsApi = new DeeplakeApi(config.token, config.apiUrl, config.orgId, config.workspaceId, config.sessionsTableName);
622
+ const memoryApi = new DeeplakeApi(config.token, config.apiUrl, config.orgId, config.workspaceId, config.tableName);
623
+ let sessionsDeleted = 0;
624
+ let summariesDeleted = 0;
625
+ for (const sessionPath of sessionPaths) {
626
+ await sessionsApi.query(`DELETE FROM "${config.sessionsTableName}" WHERE path = '${sqlStr(sessionPath)}'`);
627
+ sessionsDeleted++;
628
+ const sessionId = extractSessionId(sessionPath);
629
+ const summaryPath = `/summaries/${config.userName}/${sessionId}.md`;
630
+ const existing = await memoryApi.query(`SELECT path FROM "${config.tableName}" WHERE path = '${sqlStr(summaryPath)}' LIMIT 1`);
631
+ if (existing.length > 0) {
632
+ await memoryApi.query(`DELETE FROM "${config.tableName}" WHERE path = '${sqlStr(summaryPath)}'`);
633
+ summariesDeleted++;
634
+ }
635
+ }
636
+ return { sessionsDeleted, summariesDeleted };
637
+ }
638
+ async function sessionPrune(argv) {
639
+ const config = loadConfig();
640
+ if (!config) {
641
+ console.error("Not logged in. Run: deeplake login");
642
+ process.exit(1);
643
+ }
644
+ const { before, sessionId, all, yes } = parseArgs(argv);
645
+ const author = config.userName;
646
+ const sessionsApi = new DeeplakeApi(config.token, config.apiUrl, config.orgId, config.workspaceId, config.sessionsTableName);
647
+ const sessions = await listSessions(sessionsApi, config.sessionsTableName, author);
648
+ if (sessions.length === 0) {
649
+ console.log(`No sessions found for author "${author}".`);
650
+ return;
651
+ }
652
+ let targets;
653
+ if (sessionId) {
654
+ targets = sessions.filter((s) => extractSessionId(s.path) === sessionId);
655
+ if (targets.length === 0) {
656
+ console.error(`Session not found: ${sessionId}`);
657
+ console.error(`
658
+ Your sessions:`);
659
+ for (const s of sessions.slice(0, 10)) {
660
+ console.error(` ${extractSessionId(s.path)} ${s.firstEvent.slice(0, 10)} ${s.project}`);
661
+ }
662
+ process.exit(1);
663
+ }
664
+ } else if (before) {
665
+ const cutoff = new Date(before);
666
+ if (isNaN(cutoff.getTime())) {
667
+ console.error(`Invalid date: ${before}`);
668
+ process.exit(1);
669
+ }
670
+ targets = sessions.filter((s) => new Date(s.lastEvent) < cutoff);
671
+ } else if (all) {
672
+ targets = sessions;
673
+ } else {
674
+ console.log(`Sessions for "${author}" (${sessions.length} total):
675
+ `);
676
+ console.log(" Session ID".padEnd(42) + "Date".padEnd(14) + "Events".padEnd(10) + "Project");
677
+ console.log(" " + "\u2500".repeat(80));
678
+ for (const s of sessions) {
679
+ const id = extractSessionId(s.path);
680
+ const date = s.firstEvent.slice(0, 10);
681
+ console.log(` ${id.padEnd(40)}${date.padEnd(14)}${String(s.rowCount).padEnd(10)}${s.project}`);
682
+ }
683
+ console.log(`
684
+ To delete, use: --all, --before <date>, or --session-id <id>`);
685
+ return;
686
+ }
687
+ if (targets.length === 0) {
688
+ console.log("No sessions match the given criteria.");
689
+ return;
690
+ }
691
+ console.log(`Will delete ${targets.length} session(s) for "${author}":
692
+ `);
693
+ for (const s of targets) {
694
+ const id = extractSessionId(s.path);
695
+ console.log(` ${id} ${s.firstEvent.slice(0, 10)} ${s.rowCount} events ${s.project}`);
696
+ }
697
+ console.log();
698
+ if (!yes) {
699
+ const ok = await confirm("Proceed with deletion?");
700
+ if (!ok) {
701
+ console.log("Aborted.");
702
+ return;
703
+ }
704
+ }
705
+ const { sessionsDeleted, summariesDeleted } = await deleteSessions(config, targets.map((t) => t.path));
706
+ console.log(`Deleted ${sessionsDeleted} session(s) and ${summariesDeleted} summary file(s).`);
707
+ }
708
+
709
+ // dist/src/commands/auth-login.js
710
+ async function main() {
711
+ const args = process.argv.slice(2);
712
+ const cmd = args[0] ?? "whoami";
713
+ const creds = loadCredentials();
714
+ const apiUrl = creds?.apiUrl ?? "https://api.deeplake.ai";
715
+ switch (cmd) {
716
+ case "login": {
717
+ await login(apiUrl);
718
+ break;
719
+ }
720
+ case "whoami": {
721
+ if (!creds) {
722
+ console.log("Not logged in. Run: node auth-login.js login");
723
+ break;
724
+ }
725
+ console.log(`User org: ${creds.orgName ?? creds.orgId}`);
726
+ console.log(`Workspace: ${creds.workspaceId ?? "default"}`);
727
+ console.log(`API: ${creds.apiUrl ?? "https://api.deeplake.ai"}`);
728
+ break;
729
+ }
730
+ case "org": {
731
+ if (!creds) {
732
+ console.log("Not logged in.");
733
+ process.exit(1);
734
+ }
735
+ const sub = args[1];
736
+ if (sub === "list") {
737
+ const orgs = await listOrgs(creds.token, apiUrl);
738
+ orgs.forEach((o) => console.log(`${o.id} ${o.name}`));
739
+ } else if (sub === "switch") {
740
+ const target = args[2];
741
+ if (!target) {
742
+ console.log("Usage: org switch <org-name-or-id>");
743
+ process.exit(1);
744
+ }
745
+ const orgs = await listOrgs(creds.token, apiUrl);
746
+ const match = orgs.find((o) => o.id === target || o.name.toLowerCase() === target.toLowerCase());
747
+ if (!match) {
748
+ console.log(`Org not found: ${target}`);
749
+ process.exit(1);
750
+ }
751
+ await switchOrg(match.id, match.name);
752
+ console.log(`Switched to org: ${match.name}`);
753
+ } else {
754
+ console.log("Usage: org list | org switch <name-or-id>");
755
+ }
756
+ break;
757
+ }
758
+ case "workspaces": {
759
+ if (!creds) {
760
+ console.log("Not logged in.");
761
+ process.exit(1);
762
+ }
763
+ const ws = await listWorkspaces(creds.token, apiUrl, creds.orgId);
764
+ ws.forEach((w) => console.log(`${w.id} ${w.name}`));
765
+ break;
766
+ }
767
+ case "workspace": {
768
+ if (!creds) {
769
+ console.log("Not logged in.");
770
+ process.exit(1);
771
+ }
772
+ const wsId = args[1];
773
+ if (!wsId) {
774
+ console.log("Usage: workspace <id>");
775
+ process.exit(1);
776
+ }
777
+ await switchWorkspace(wsId);
778
+ console.log(`Switched to workspace: ${wsId}`);
779
+ break;
780
+ }
781
+ case "invite": {
782
+ if (!creds) {
783
+ console.log("Not logged in.");
784
+ process.exit(1);
785
+ }
786
+ const email = args[1];
787
+ const mode = args[2]?.toUpperCase() ?? "WRITE";
788
+ if (!email) {
789
+ console.log("Usage: invite <email> [ADMIN|WRITE|READ]");
790
+ process.exit(1);
791
+ }
792
+ await inviteMember(email, mode, creds.token, creds.orgId, apiUrl);
793
+ console.log(`Invited ${email} with ${mode} access`);
794
+ break;
795
+ }
796
+ case "members": {
797
+ if (!creds) {
798
+ console.log("Not logged in.");
799
+ process.exit(1);
800
+ }
801
+ const members = await listMembers(creds.token, creds.orgId, apiUrl);
802
+ members.forEach((m) => console.log(`${m.role.padEnd(8)} ${m.email ?? m.name}`));
803
+ break;
804
+ }
805
+ case "remove": {
806
+ if (!creds) {
807
+ console.log("Not logged in.");
808
+ process.exit(1);
809
+ }
810
+ const userId = args[1];
811
+ if (!userId) {
812
+ console.log("Usage: remove <user-id>");
813
+ process.exit(1);
814
+ }
815
+ await removeMember(userId, creds.token, creds.orgId, apiUrl);
816
+ console.log(`Removed user ${userId}`);
817
+ break;
818
+ }
819
+ case "sessions": {
820
+ const sub = args[1];
821
+ if (sub === "prune") {
822
+ await sessionPrune(args.slice(2));
823
+ } else {
824
+ console.log("Usage: sessions prune [--all | --before <date> | --session-id <id>] [--yes]");
825
+ }
826
+ break;
827
+ }
828
+ case "autoupdate": {
829
+ if (!creds) {
830
+ console.log("Not logged in.");
831
+ process.exit(1);
832
+ }
833
+ const val = args[1]?.toLowerCase();
834
+ if (val === "on" || val === "true") {
835
+ saveCredentials({ ...creds, autoupdate: true });
836
+ console.log("Autoupdate enabled. Plugin will update automatically on session start.");
837
+ } else if (val === "off" || val === "false") {
838
+ saveCredentials({ ...creds, autoupdate: false });
839
+ console.log("Autoupdate disabled. You'll see a notice when updates are available.");
840
+ } else {
841
+ const current = creds.autoupdate !== false ? "on" : "off";
842
+ console.log(`Autoupdate is currently: ${current}`);
843
+ console.log("Usage: autoupdate [on|off]");
844
+ }
845
+ break;
846
+ }
847
+ case "logout": {
848
+ if (deleteCredentials()) {
849
+ console.log("Logged out. Credentials removed.");
850
+ } else {
851
+ console.log("Not logged in.");
852
+ }
853
+ break;
854
+ }
855
+ default:
856
+ console.log("Commands: login, logout, whoami, org list, org switch, workspaces, workspace, sessions prune, invite, members, remove, autoupdate");
857
+ }
858
+ }
859
+ main().catch((e) => {
860
+ console.error(e.message);
861
+ process.exit(1);
862
+ });