@deeplake/hivemind 0.6.48 → 0.7.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +2 -2
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +244 -20
- package/bundle/cli.js +1369 -112
- package/codex/bundle/capture.js +546 -96
- package/codex/bundle/commands/auth-login.js +290 -81
- package/codex/bundle/embeddings/embed-daemon.js +243 -0
- package/codex/bundle/pre-tool-use.js +666 -111
- package/codex/bundle/session-start-setup.js +231 -64
- package/codex/bundle/session-start.js +52 -13
- package/codex/bundle/shell/deeplake-shell.js +716 -119
- package/codex/bundle/skilify-worker.js +907 -0
- package/codex/bundle/stop.js +819 -79
- package/codex/bundle/wiki-worker.js +312 -11
- package/cursor/bundle/capture.js +1116 -64
- package/cursor/bundle/commands/auth-login.js +290 -81
- package/cursor/bundle/embeddings/embed-daemon.js +243 -0
- package/cursor/bundle/pre-tool-use.js +598 -77
- package/cursor/bundle/session-end.js +520 -2
- package/cursor/bundle/session-start.js +257 -65
- package/cursor/bundle/shell/deeplake-shell.js +716 -119
- package/cursor/bundle/skilify-worker.js +907 -0
- package/cursor/bundle/wiki-worker.js +571 -0
- package/hermes/bundle/capture.js +1119 -65
- package/hermes/bundle/commands/auth-login.js +290 -81
- package/hermes/bundle/embeddings/embed-daemon.js +243 -0
- package/hermes/bundle/pre-tool-use.js +597 -76
- package/hermes/bundle/session-end.js +522 -1
- package/hermes/bundle/session-start.js +260 -65
- package/hermes/bundle/shell/deeplake-shell.js +716 -119
- package/hermes/bundle/skilify-worker.js +907 -0
- package/hermes/bundle/wiki-worker.js +572 -0
- package/mcp/bundle/server.js +290 -75
- package/openclaw/dist/chunks/auth-creds-AEKS6D3P.js +14 -0
- package/openclaw/dist/chunks/chunk-SRCBBT4H.js +37 -0
- package/openclaw/dist/chunks/config-ZLH6JFJS.js +34 -0
- package/openclaw/dist/chunks/index-marker-store-PGT5CW6T.js +33 -0
- package/openclaw/dist/chunks/setup-config-C35UK4LP.js +114 -0
- package/openclaw/dist/index.js +929 -710
- package/openclaw/dist/skilify-worker.js +907 -0
- package/openclaw/openclaw.plugin.json +1 -1
- package/openclaw/package.json +1 -1
- package/openclaw/skills/SKILL.md +19 -0
- package/package.json +7 -1
- package/pi/extension-source/hivemind.ts +603 -22
|
@@ -1,38 +1,106 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
4
|
+
var __esm = (fn, res) => function __init() {
|
|
5
|
+
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
|
|
6
|
+
};
|
|
7
|
+
var __export = (target, all) => {
|
|
8
|
+
for (var name in all)
|
|
9
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
// dist/src/index-marker-store.js
|
|
13
|
+
var index_marker_store_exports = {};
|
|
14
|
+
__export(index_marker_store_exports, {
|
|
15
|
+
buildIndexMarkerPath: () => buildIndexMarkerPath,
|
|
16
|
+
getIndexMarkerDir: () => getIndexMarkerDir,
|
|
17
|
+
hasFreshIndexMarker: () => hasFreshIndexMarker,
|
|
18
|
+
writeIndexMarker: () => writeIndexMarker
|
|
19
|
+
});
|
|
20
|
+
import { existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync as readFileSync3, writeFileSync as writeFileSync2 } from "node:fs";
|
|
21
|
+
import { join as join4 } from "node:path";
|
|
22
|
+
import { tmpdir } from "node:os";
|
|
23
|
+
function getIndexMarkerDir() {
|
|
24
|
+
return process.env.HIVEMIND_INDEX_MARKER_DIR ?? join4(tmpdir(), "hivemind-deeplake-indexes");
|
|
25
|
+
}
|
|
26
|
+
function buildIndexMarkerPath(workspaceId, orgId, table, suffix) {
|
|
27
|
+
const markerKey = [workspaceId, orgId, table, suffix].join("__").replace(/[^a-zA-Z0-9_.-]/g, "_");
|
|
28
|
+
return join4(getIndexMarkerDir(), `${markerKey}.json`);
|
|
29
|
+
}
|
|
30
|
+
function hasFreshIndexMarker(markerPath) {
|
|
31
|
+
if (!existsSync2(markerPath))
|
|
32
|
+
return false;
|
|
33
|
+
try {
|
|
34
|
+
const raw = JSON.parse(readFileSync3(markerPath, "utf-8"));
|
|
35
|
+
const updatedAt = raw.updatedAt ? new Date(raw.updatedAt).getTime() : NaN;
|
|
36
|
+
if (!Number.isFinite(updatedAt) || Date.now() - updatedAt > INDEX_MARKER_TTL_MS)
|
|
37
|
+
return false;
|
|
38
|
+
return true;
|
|
39
|
+
} catch {
|
|
40
|
+
return false;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
function writeIndexMarker(markerPath) {
|
|
44
|
+
mkdirSync2(getIndexMarkerDir(), { recursive: true });
|
|
45
|
+
writeFileSync2(markerPath, JSON.stringify({ updatedAt: (/* @__PURE__ */ new Date()).toISOString() }), "utf-8");
|
|
46
|
+
}
|
|
47
|
+
var INDEX_MARKER_TTL_MS;
|
|
48
|
+
var init_index_marker_store = __esm({
|
|
49
|
+
"dist/src/index-marker-store.js"() {
|
|
50
|
+
"use strict";
|
|
51
|
+
INDEX_MARKER_TTL_MS = Number(process.env.HIVEMIND_INDEX_MARKER_TTL_MS ?? 6 * 60 * 6e4);
|
|
52
|
+
}
|
|
53
|
+
});
|
|
2
54
|
|
|
3
55
|
// dist/src/commands/auth.js
|
|
4
|
-
import {
|
|
56
|
+
import { execSync } from "node:child_process";
|
|
57
|
+
|
|
58
|
+
// dist/src/utils/client-header.js
|
|
59
|
+
var DEEPLAKE_CLIENT_HEADER = "X-Deeplake-Client";
|
|
60
|
+
function deeplakeClientValue() {
|
|
61
|
+
return "hivemind";
|
|
62
|
+
}
|
|
63
|
+
function deeplakeClientHeader() {
|
|
64
|
+
return { [DEEPLAKE_CLIENT_HEADER]: deeplakeClientValue() };
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// dist/src/commands/auth-creds.js
|
|
68
|
+
import { readFileSync, writeFileSync, mkdirSync, unlinkSync } from "node:fs";
|
|
5
69
|
import { join } from "node:path";
|
|
6
70
|
import { homedir } from "node:os";
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
71
|
+
function configDir() {
|
|
72
|
+
return join(homedir(), ".deeplake");
|
|
73
|
+
}
|
|
74
|
+
function credsPath() {
|
|
75
|
+
return join(configDir(), "credentials.json");
|
|
76
|
+
}
|
|
11
77
|
function loadCredentials() {
|
|
12
|
-
if (!existsSync(CREDS_PATH))
|
|
13
|
-
return null;
|
|
14
78
|
try {
|
|
15
|
-
return JSON.parse(readFileSync(
|
|
79
|
+
return JSON.parse(readFileSync(credsPath(), "utf-8"));
|
|
16
80
|
} catch {
|
|
17
81
|
return null;
|
|
18
82
|
}
|
|
19
83
|
}
|
|
20
84
|
function saveCredentials(creds) {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
writeFileSync(CREDS_PATH, JSON.stringify({ ...creds, savedAt: (/* @__PURE__ */ new Date()).toISOString() }, null, 2), { mode: 384 });
|
|
85
|
+
mkdirSync(configDir(), { recursive: true, mode: 448 });
|
|
86
|
+
writeFileSync(credsPath(), JSON.stringify({ ...creds, savedAt: (/* @__PURE__ */ new Date()).toISOString() }, null, 2), { mode: 384 });
|
|
24
87
|
}
|
|
25
88
|
function deleteCredentials() {
|
|
26
|
-
|
|
27
|
-
unlinkSync(
|
|
89
|
+
try {
|
|
90
|
+
unlinkSync(credsPath());
|
|
28
91
|
return true;
|
|
92
|
+
} catch {
|
|
93
|
+
return false;
|
|
29
94
|
}
|
|
30
|
-
return false;
|
|
31
95
|
}
|
|
96
|
+
|
|
97
|
+
// dist/src/commands/auth.js
|
|
98
|
+
var DEFAULT_API_URL = "https://api.deeplake.ai";
|
|
32
99
|
async function apiGet(path, token, apiUrl, orgId) {
|
|
33
100
|
const headers = {
|
|
34
101
|
Authorization: `Bearer ${token}`,
|
|
35
|
-
"Content-Type": "application/json"
|
|
102
|
+
"Content-Type": "application/json",
|
|
103
|
+
...deeplakeClientHeader()
|
|
36
104
|
};
|
|
37
105
|
if (orgId)
|
|
38
106
|
headers["X-Activeloop-Org-Id"] = orgId;
|
|
@@ -44,7 +112,8 @@ async function apiGet(path, token, apiUrl, orgId) {
|
|
|
44
112
|
async function apiPost(path, body, token, apiUrl, orgId) {
|
|
45
113
|
const headers = {
|
|
46
114
|
Authorization: `Bearer ${token}`,
|
|
47
|
-
"Content-Type": "application/json"
|
|
115
|
+
"Content-Type": "application/json",
|
|
116
|
+
...deeplakeClientHeader()
|
|
48
117
|
};
|
|
49
118
|
if (orgId)
|
|
50
119
|
headers["X-Activeloop-Org-Id"] = orgId;
|
|
@@ -56,7 +125,8 @@ async function apiPost(path, body, token, apiUrl, orgId) {
|
|
|
56
125
|
async function apiDelete(path, token, apiUrl, orgId) {
|
|
57
126
|
const headers = {
|
|
58
127
|
Authorization: `Bearer ${token}`,
|
|
59
|
-
"Content-Type": "application/json"
|
|
128
|
+
"Content-Type": "application/json",
|
|
129
|
+
...deeplakeClientHeader()
|
|
60
130
|
};
|
|
61
131
|
if (orgId)
|
|
62
132
|
headers["X-Activeloop-Org-Id"] = orgId;
|
|
@@ -67,7 +137,7 @@ async function apiDelete(path, token, apiUrl, orgId) {
|
|
|
67
137
|
async function requestDeviceCode(apiUrl = DEFAULT_API_URL) {
|
|
68
138
|
const resp = await fetch(`${apiUrl}/auth/device/code`, {
|
|
69
139
|
method: "POST",
|
|
70
|
-
headers: { "Content-Type": "application/json" }
|
|
140
|
+
headers: { "Content-Type": "application/json", ...deeplakeClientHeader() }
|
|
71
141
|
});
|
|
72
142
|
if (!resp.ok)
|
|
73
143
|
throw new Error(`Device flow unavailable: HTTP ${resp.status}`);
|
|
@@ -76,7 +146,7 @@ async function requestDeviceCode(apiUrl = DEFAULT_API_URL) {
|
|
|
76
146
|
async function pollForToken(deviceCode, apiUrl = DEFAULT_API_URL) {
|
|
77
147
|
const resp = await fetch(`${apiUrl}/auth/device/token`, {
|
|
78
148
|
method: "POST",
|
|
79
|
-
headers: { "Content-Type": "application/json" },
|
|
149
|
+
headers: { "Content-Type": "application/json", ...deeplakeClientHeader() },
|
|
80
150
|
body: JSON.stringify({ device_code: deviceCode })
|
|
81
151
|
});
|
|
82
152
|
if (resp.ok)
|
|
@@ -202,14 +272,14 @@ Using: ${orgName}
|
|
|
202
272
|
}
|
|
203
273
|
|
|
204
274
|
// dist/src/config.js
|
|
205
|
-
import { readFileSync as readFileSync2, existsSync
|
|
275
|
+
import { readFileSync as readFileSync2, existsSync } from "node:fs";
|
|
206
276
|
import { join as join2 } from "node:path";
|
|
207
277
|
import { homedir as homedir2, userInfo } from "node:os";
|
|
208
278
|
function loadConfig() {
|
|
209
279
|
const home = homedir2();
|
|
210
280
|
const credPath = join2(home, ".deeplake", "credentials.json");
|
|
211
281
|
let creds = null;
|
|
212
|
-
if (
|
|
282
|
+
if (existsSync(credPath)) {
|
|
213
283
|
try {
|
|
214
284
|
creds = JSON.parse(readFileSync2(credPath, "utf-8"));
|
|
215
285
|
} catch {
|
|
@@ -229,15 +299,13 @@ function loadConfig() {
|
|
|
229
299
|
apiUrl: process.env.HIVEMIND_API_URL ?? creds?.apiUrl ?? "https://api.deeplake.ai",
|
|
230
300
|
tableName: process.env.HIVEMIND_TABLE ?? "memory",
|
|
231
301
|
sessionsTableName: process.env.HIVEMIND_SESSIONS_TABLE ?? "sessions",
|
|
302
|
+
skillsTableName: process.env.HIVEMIND_SKILLS_TABLE ?? "skills",
|
|
232
303
|
memoryPath: process.env.HIVEMIND_MEMORY_PATH ?? join2(home, ".deeplake", "memory")
|
|
233
304
|
};
|
|
234
305
|
}
|
|
235
306
|
|
|
236
307
|
// dist/src/deeplake-api.js
|
|
237
308
|
import { randomUUID } from "node:crypto";
|
|
238
|
-
import { existsSync as existsSync3, mkdirSync as mkdirSync2, readFileSync as readFileSync3, writeFileSync as writeFileSync2 } from "node:fs";
|
|
239
|
-
import { join as join4 } from "node:path";
|
|
240
|
-
import { tmpdir } from "node:os";
|
|
241
309
|
|
|
242
310
|
// dist/src/utils/debug.js
|
|
243
311
|
import { appendFileSync } from "node:fs";
|
|
@@ -256,8 +324,24 @@ function log(tag, msg) {
|
|
|
256
324
|
function sqlStr(value) {
|
|
257
325
|
return value.replace(/\\/g, "\\\\").replace(/'/g, "''").replace(/\0/g, "").replace(/[\x01-\x08\x0b\x0c\x0e-\x1f\x7f]/g, "");
|
|
258
326
|
}
|
|
327
|
+
function sqlIdent(name) {
|
|
328
|
+
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) {
|
|
329
|
+
throw new Error(`Invalid SQL identifier: ${JSON.stringify(name)}`);
|
|
330
|
+
}
|
|
331
|
+
return name;
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
// dist/src/embeddings/columns.js
|
|
335
|
+
var SUMMARY_EMBEDDING_COL = "summary_embedding";
|
|
336
|
+
var MESSAGE_EMBEDDING_COL = "message_embedding";
|
|
259
337
|
|
|
260
338
|
// dist/src/deeplake-api.js
|
|
339
|
+
var indexMarkerStorePromise = null;
|
|
340
|
+
function getIndexMarkerStore() {
|
|
341
|
+
if (!indexMarkerStorePromise)
|
|
342
|
+
indexMarkerStorePromise = Promise.resolve().then(() => (init_index_marker_store(), index_marker_store_exports));
|
|
343
|
+
return indexMarkerStorePromise;
|
|
344
|
+
}
|
|
261
345
|
var log2 = (msg) => log("sdk", msg);
|
|
262
346
|
function summarizeSql(sql, maxLen = 220) {
|
|
263
347
|
const compact = sql.replace(/\s+/g, " ").trim();
|
|
@@ -277,7 +361,6 @@ var MAX_RETRIES = 3;
|
|
|
277
361
|
var BASE_DELAY_MS = 500;
|
|
278
362
|
var MAX_CONCURRENCY = 5;
|
|
279
363
|
var QUERY_TIMEOUT_MS = Number(process.env.HIVEMIND_QUERY_TIMEOUT_MS ?? 1e4);
|
|
280
|
-
var INDEX_MARKER_TTL_MS = Number(process.env.HIVEMIND_INDEX_MARKER_TTL_MS ?? 6 * 60 * 6e4);
|
|
281
364
|
function sleep(ms) {
|
|
282
365
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
283
366
|
}
|
|
@@ -297,9 +380,6 @@ function isTransientHtml403(text) {
|
|
|
297
380
|
const body = text.toLowerCase();
|
|
298
381
|
return body.includes("<html") || body.includes("403 forbidden") || body.includes("cloudflare") || body.includes("nginx");
|
|
299
382
|
}
|
|
300
|
-
function getIndexMarkerDir() {
|
|
301
|
-
return process.env.HIVEMIND_INDEX_MARKER_DIR ?? join4(tmpdir(), "hivemind-deeplake-indexes");
|
|
302
|
-
}
|
|
303
383
|
var Semaphore = class {
|
|
304
384
|
max;
|
|
305
385
|
waiting = [];
|
|
@@ -368,7 +448,8 @@ var DeeplakeApi = class {
|
|
|
368
448
|
headers: {
|
|
369
449
|
Authorization: `Bearer ${this.token}`,
|
|
370
450
|
"Content-Type": "application/json",
|
|
371
|
-
"X-Activeloop-Org-Id": this.orgId
|
|
451
|
+
"X-Activeloop-Org-Id": this.orgId,
|
|
452
|
+
...deeplakeClientHeader()
|
|
372
453
|
},
|
|
373
454
|
signal,
|
|
374
455
|
body: JSON.stringify({ query: sql })
|
|
@@ -395,7 +476,8 @@ var DeeplakeApi = class {
|
|
|
395
476
|
}
|
|
396
477
|
const text = await resp.text().catch(() => "");
|
|
397
478
|
const retryable403 = isSessionInsertQuery(sql) && (resp.status === 401 || resp.status === 403 && (text.length === 0 || isTransientHtml403(text)));
|
|
398
|
-
|
|
479
|
+
const alreadyExists = resp.status === 500 && isDuplicateIndexError(text);
|
|
480
|
+
if (!alreadyExists && attempt < MAX_RETRIES && (RETRYABLE_CODES.has(resp.status) || retryable403)) {
|
|
399
481
|
const delay = BASE_DELAY_MS * Math.pow(2, attempt) + Math.random() * 200;
|
|
400
482
|
log2(`query retry ${attempt + 1}/${MAX_RETRIES} (${resp.status}) in ${delay.toFixed(0)}ms`);
|
|
401
483
|
await sleep(delay);
|
|
@@ -429,7 +511,7 @@ var DeeplakeApi = class {
|
|
|
429
511
|
const lud = row.lastUpdateDate ?? ts;
|
|
430
512
|
const exists = await this.query(`SELECT path FROM "${this.tableName}" WHERE path = '${sqlStr(row.path)}' LIMIT 1`);
|
|
431
513
|
if (exists.length > 0) {
|
|
432
|
-
let setClauses = `summary = E'${sqlStr(row.contentText)}', mime_type = '${sqlStr(row.mimeType)}', size_bytes = ${row.sizeBytes}, last_update_date = '${lud}'`;
|
|
514
|
+
let setClauses = `summary = E'${sqlStr(row.contentText)}', ${SUMMARY_EMBEDDING_COL} = NULL, mime_type = '${sqlStr(row.mimeType)}', size_bytes = ${row.sizeBytes}, last_update_date = '${lud}'`;
|
|
433
515
|
if (row.project !== void 0)
|
|
434
516
|
setClauses += `, project = '${sqlStr(row.project)}'`;
|
|
435
517
|
if (row.description !== void 0)
|
|
@@ -437,8 +519,8 @@ var DeeplakeApi = class {
|
|
|
437
519
|
await this.query(`UPDATE "${this.tableName}" SET ${setClauses} WHERE path = '${sqlStr(row.path)}'`);
|
|
438
520
|
} else {
|
|
439
521
|
const id = randomUUID();
|
|
440
|
-
let cols =
|
|
441
|
-
let vals = `'${id}', '${sqlStr(row.path)}', '${sqlStr(row.filename)}', E'${sqlStr(row.contentText)}', '${sqlStr(row.mimeType)}', ${row.sizeBytes}, '${cd}', '${lud}'`;
|
|
522
|
+
let cols = `id, path, filename, summary, ${SUMMARY_EMBEDDING_COL}, mime_type, size_bytes, creation_date, last_update_date`;
|
|
523
|
+
let vals = `'${id}', '${sqlStr(row.path)}', '${sqlStr(row.filename)}', E'${sqlStr(row.contentText)}', NULL, '${sqlStr(row.mimeType)}', ${row.sizeBytes}, '${cd}', '${lud}'`;
|
|
442
524
|
if (row.project !== void 0) {
|
|
443
525
|
cols += ", project";
|
|
444
526
|
vals += `, '${sqlStr(row.project)}'`;
|
|
@@ -463,48 +545,83 @@ var DeeplakeApi = class {
|
|
|
463
545
|
buildLookupIndexName(table, suffix) {
|
|
464
546
|
return `idx_${table}_${suffix}`.replace(/[^a-zA-Z0-9_]/g, "_");
|
|
465
547
|
}
|
|
466
|
-
getLookupIndexMarkerPath(table, suffix) {
|
|
467
|
-
const markerKey = [
|
|
468
|
-
this.workspaceId,
|
|
469
|
-
this.orgId,
|
|
470
|
-
table,
|
|
471
|
-
suffix
|
|
472
|
-
].join("__").replace(/[^a-zA-Z0-9_.-]/g, "_");
|
|
473
|
-
return join4(getIndexMarkerDir(), `${markerKey}.json`);
|
|
474
|
-
}
|
|
475
|
-
hasFreshLookupIndexMarker(table, suffix) {
|
|
476
|
-
const markerPath = this.getLookupIndexMarkerPath(table, suffix);
|
|
477
|
-
if (!existsSync3(markerPath))
|
|
478
|
-
return false;
|
|
479
|
-
try {
|
|
480
|
-
const raw = JSON.parse(readFileSync3(markerPath, "utf-8"));
|
|
481
|
-
const updatedAt = raw.updatedAt ? new Date(raw.updatedAt).getTime() : NaN;
|
|
482
|
-
if (!Number.isFinite(updatedAt) || Date.now() - updatedAt > INDEX_MARKER_TTL_MS)
|
|
483
|
-
return false;
|
|
484
|
-
return true;
|
|
485
|
-
} catch {
|
|
486
|
-
return false;
|
|
487
|
-
}
|
|
488
|
-
}
|
|
489
|
-
markLookupIndexReady(table, suffix) {
|
|
490
|
-
mkdirSync2(getIndexMarkerDir(), { recursive: true });
|
|
491
|
-
writeFileSync2(this.getLookupIndexMarkerPath(table, suffix), JSON.stringify({ updatedAt: (/* @__PURE__ */ new Date()).toISOString() }), "utf-8");
|
|
492
|
-
}
|
|
493
548
|
async ensureLookupIndex(table, suffix, columnsSql) {
|
|
494
|
-
|
|
549
|
+
const markers = await getIndexMarkerStore();
|
|
550
|
+
const markerPath = markers.buildIndexMarkerPath(this.workspaceId, this.orgId, table, suffix);
|
|
551
|
+
if (markers.hasFreshIndexMarker(markerPath))
|
|
495
552
|
return;
|
|
496
553
|
const indexName = this.buildLookupIndexName(table, suffix);
|
|
497
554
|
try {
|
|
498
555
|
await this.query(`CREATE INDEX IF NOT EXISTS "${indexName}" ON "${table}" ${columnsSql}`);
|
|
499
|
-
|
|
556
|
+
markers.writeIndexMarker(markerPath);
|
|
500
557
|
} catch (e) {
|
|
501
558
|
if (isDuplicateIndexError(e)) {
|
|
502
|
-
|
|
559
|
+
markers.writeIndexMarker(markerPath);
|
|
503
560
|
return;
|
|
504
561
|
}
|
|
505
562
|
log2(`index "${indexName}" skipped: ${e.message}`);
|
|
506
563
|
}
|
|
507
564
|
}
|
|
565
|
+
/**
|
|
566
|
+
* Ensure a vector column exists on the given table.
|
|
567
|
+
*
|
|
568
|
+
* The previous implementation always issued `ALTER TABLE ADD COLUMN IF NOT
|
|
569
|
+
* EXISTS …` on every SessionStart. On a long-running workspace that's
|
|
570
|
+
* already migrated, every call returns 500 "Column already exists" — noisy
|
|
571
|
+
* in the log and a wasted round-trip. Worse, the very first call after the
|
|
572
|
+
* column is genuinely added triggers Deeplake's post-ALTER `vector::at`
|
|
573
|
+
* window (~30s) during which subsequent INSERTs fail; minimising the
|
|
574
|
+
* number of ALTER calls minimises exposure to that window.
|
|
575
|
+
*
|
|
576
|
+
* New flow:
|
|
577
|
+
* 1. Check the local marker file (mirrors ensureLookupIndex). If fresh,
|
|
578
|
+
* return — zero network calls.
|
|
579
|
+
* 2. SELECT 1 FROM information_schema.columns WHERE table_name = T AND
|
|
580
|
+
* column_name = C. Read-only, idempotent, can't tickle the post-ALTER
|
|
581
|
+
* bug. If the column is present → mark + return.
|
|
582
|
+
* 3. Only if step 2 says the column is missing, fall back to ALTER ADD
|
|
583
|
+
* COLUMN IF NOT EXISTS. Mark on success, also mark if Deeplake reports
|
|
584
|
+
* "already exists" (race: another client added it between our SELECT
|
|
585
|
+
* and ALTER).
|
|
586
|
+
*
|
|
587
|
+
* Marker uses the same dir / TTL as ensureLookupIndex so both schema
|
|
588
|
+
* caches share an opt-out (HIVEMIND_INDEX_MARKER_DIR) and a TTL knob.
|
|
589
|
+
*/
|
|
590
|
+
async ensureEmbeddingColumn(table, column) {
|
|
591
|
+
await this.ensureColumn(table, column, "FLOAT4[]");
|
|
592
|
+
}
|
|
593
|
+
/**
|
|
594
|
+
* Generic marker-gated column migration. Same SELECT-then-ALTER flow as
|
|
595
|
+
* ensureEmbeddingColumn, parameterized by SQL type so it can patch up any
|
|
596
|
+
* column that was added to the schema after the table was originally
|
|
597
|
+
* created. Used today for `summary_embedding`, `message_embedding`, and
|
|
598
|
+
* the `agent` column (added 2026-04-11) — the latter has no fallback if
|
|
599
|
+
* a user upgraded over a pre-2026-04-11 table, so every INSERT fails
|
|
600
|
+
* with `column "agent" does not exist`.
|
|
601
|
+
*/
|
|
602
|
+
async ensureColumn(table, column, sqlType) {
|
|
603
|
+
const markers = await getIndexMarkerStore();
|
|
604
|
+
const markerPath = markers.buildIndexMarkerPath(this.workspaceId, this.orgId, table, `col_${column}`);
|
|
605
|
+
if (markers.hasFreshIndexMarker(markerPath))
|
|
606
|
+
return;
|
|
607
|
+
const colCheck = `SELECT 1 FROM information_schema.columns WHERE table_name = '${sqlStr(table)}' AND column_name = '${sqlStr(column)}' AND table_schema = '${sqlStr(this.workspaceId)}' LIMIT 1`;
|
|
608
|
+
const rows = await this.query(colCheck);
|
|
609
|
+
if (rows.length > 0) {
|
|
610
|
+
markers.writeIndexMarker(markerPath);
|
|
611
|
+
return;
|
|
612
|
+
}
|
|
613
|
+
try {
|
|
614
|
+
await this.query(`ALTER TABLE "${table}" ADD COLUMN ${column} ${sqlType}`);
|
|
615
|
+
} catch (e) {
|
|
616
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
617
|
+
if (!/already exists/i.test(msg))
|
|
618
|
+
throw e;
|
|
619
|
+
const recheck = await this.query(colCheck);
|
|
620
|
+
if (recheck.length === 0)
|
|
621
|
+
throw e;
|
|
622
|
+
}
|
|
623
|
+
markers.writeIndexMarker(markerPath);
|
|
624
|
+
}
|
|
508
625
|
/** List all tables in the workspace (with retry). */
|
|
509
626
|
async listTables(forceRefresh = false) {
|
|
510
627
|
if (!forceRefresh && this._tablesCache)
|
|
@@ -520,7 +637,8 @@ var DeeplakeApi = class {
|
|
|
520
637
|
const resp = await fetch(`${this.apiUrl}/workspaces/${this.workspaceId}/tables`, {
|
|
521
638
|
headers: {
|
|
522
639
|
Authorization: `Bearer ${this.token}`,
|
|
523
|
-
"X-Activeloop-Org-Id": this.orgId
|
|
640
|
+
"X-Activeloop-Org-Id": this.orgId,
|
|
641
|
+
...deeplakeClientHeader()
|
|
524
642
|
}
|
|
525
643
|
});
|
|
526
644
|
if (resp.ok) {
|
|
@@ -545,29 +663,84 @@ var DeeplakeApi = class {
|
|
|
545
663
|
}
|
|
546
664
|
return { tables: [], cacheable: false };
|
|
547
665
|
}
|
|
666
|
+
/**
|
|
667
|
+
* Run a `CREATE TABLE` with an extra outer retry budget. The base
|
|
668
|
+
* `query()` already retries 3 times on fetch errors (~3.5s total), but a
|
|
669
|
+
* failed CREATE is permanent corruption — every subsequent SELECT against
|
|
670
|
+
* the missing table fails. Wrapping in an outer loop with longer backoff
|
|
671
|
+
* (2s, 5s, then 10s) gives us ~17s of reach across transient network
|
|
672
|
+
* blips before giving up. Failures still propagate; getApi() resets its
|
|
673
|
+
* cache on init failure (openclaw plugin) so the next call retries the
|
|
674
|
+
* whole init flow.
|
|
675
|
+
*/
|
|
676
|
+
async createTableWithRetry(sql, label) {
|
|
677
|
+
const OUTER_BACKOFFS_MS = [2e3, 5e3, 1e4];
|
|
678
|
+
let lastErr = null;
|
|
679
|
+
for (let attempt = 0; attempt <= OUTER_BACKOFFS_MS.length; attempt++) {
|
|
680
|
+
try {
|
|
681
|
+
await this.query(sql);
|
|
682
|
+
return;
|
|
683
|
+
} catch (err) {
|
|
684
|
+
lastErr = err;
|
|
685
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
686
|
+
log2(`CREATE TABLE "${label}" attempt ${attempt + 1}/${OUTER_BACKOFFS_MS.length + 1} failed: ${msg}`);
|
|
687
|
+
if (attempt < OUTER_BACKOFFS_MS.length) {
|
|
688
|
+
await sleep(OUTER_BACKOFFS_MS[attempt]);
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
}
|
|
692
|
+
throw lastErr;
|
|
693
|
+
}
|
|
548
694
|
/** Create the memory table if it doesn't already exist. Migrate columns on existing tables. */
|
|
549
695
|
async ensureTable(name) {
|
|
550
|
-
const tbl = name ?? this.tableName;
|
|
696
|
+
const tbl = sqlIdent(name ?? this.tableName);
|
|
551
697
|
const tables = await this.listTables();
|
|
552
698
|
if (!tables.includes(tbl)) {
|
|
553
699
|
log2(`table "${tbl}" not found, creating`);
|
|
554
|
-
await this.
|
|
700
|
+
await this.createTableWithRetry(`CREATE TABLE IF NOT EXISTS "${tbl}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', summary_embedding FLOAT4[], author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'text/plain', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`, tbl);
|
|
555
701
|
log2(`table "${tbl}" created`);
|
|
556
702
|
if (!tables.includes(tbl))
|
|
557
703
|
this._tablesCache = [...tables, tbl];
|
|
558
704
|
}
|
|
705
|
+
await this.ensureEmbeddingColumn(tbl, SUMMARY_EMBEDDING_COL);
|
|
706
|
+
await this.ensureColumn(tbl, "agent", "TEXT NOT NULL DEFAULT ''");
|
|
559
707
|
}
|
|
560
708
|
/** Create the sessions table (uses JSONB for message since every row is a JSON event). */
|
|
561
709
|
async ensureSessionsTable(name) {
|
|
710
|
+
const safe = sqlIdent(name);
|
|
562
711
|
const tables = await this.listTables();
|
|
563
|
-
if (!tables.includes(
|
|
564
|
-
log2(`table "${
|
|
565
|
-
await this.
|
|
566
|
-
log2(`table "${
|
|
567
|
-
if (!tables.includes(
|
|
568
|
-
this._tablesCache = [...tables,
|
|
712
|
+
if (!tables.includes(safe)) {
|
|
713
|
+
log2(`table "${safe}" not found, creating`);
|
|
714
|
+
await this.createTableWithRetry(`CREATE TABLE IF NOT EXISTS "${safe}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', message JSONB, message_embedding FLOAT4[], author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'application/json', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`, safe);
|
|
715
|
+
log2(`table "${safe}" created`);
|
|
716
|
+
if (!tables.includes(safe))
|
|
717
|
+
this._tablesCache = [...tables, safe];
|
|
569
718
|
}
|
|
570
|
-
await this.
|
|
719
|
+
await this.ensureEmbeddingColumn(safe, MESSAGE_EMBEDDING_COL);
|
|
720
|
+
await this.ensureColumn(safe, "agent", "TEXT NOT NULL DEFAULT ''");
|
|
721
|
+
await this.ensureLookupIndex(safe, "path_creation_date", `("path", "creation_date")`);
|
|
722
|
+
}
|
|
723
|
+
/**
|
|
724
|
+
* Create the skills table.
|
|
725
|
+
*
|
|
726
|
+
* One row per skill version. Workers INSERT a fresh row on every KEEP /
|
|
727
|
+
* MERGE rather than UPDATE-ing in place, so the full version history is
|
|
728
|
+
* recoverable. Uniqueness in the *current* state is by (project_key, name)
|
|
729
|
+
* — newer rows shadow older ones at read time (ORDER BY version DESC).
|
|
730
|
+
* This sidesteps the Deeplake UPDATE-coalescing quirk that bit the wiki
|
|
731
|
+
* worker.
|
|
732
|
+
*/
|
|
733
|
+
async ensureSkillsTable(name) {
|
|
734
|
+
const safe = sqlIdent(name);
|
|
735
|
+
const tables = await this.listTables();
|
|
736
|
+
if (!tables.includes(safe)) {
|
|
737
|
+
log2(`table "${safe}" not found, creating`);
|
|
738
|
+
await this.createTableWithRetry(`CREATE TABLE IF NOT EXISTS "${safe}" (id TEXT NOT NULL DEFAULT '', name TEXT NOT NULL DEFAULT '', project TEXT NOT NULL DEFAULT '', project_key TEXT NOT NULL DEFAULT '', local_path TEXT NOT NULL DEFAULT '', install TEXT NOT NULL DEFAULT 'project', source_sessions TEXT NOT NULL DEFAULT '[]', source_agent TEXT NOT NULL DEFAULT '', scope TEXT NOT NULL DEFAULT 'me', author TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', trigger_text TEXT NOT NULL DEFAULT '', body TEXT NOT NULL DEFAULT '', version BIGINT NOT NULL DEFAULT 1, created_at TEXT NOT NULL DEFAULT '', updated_at TEXT NOT NULL DEFAULT '') USING deeplake`, safe);
|
|
739
|
+
log2(`table "${safe}" created`);
|
|
740
|
+
if (!tables.includes(safe))
|
|
741
|
+
this._tablesCache = [...tables, safe];
|
|
742
|
+
}
|
|
743
|
+
await this.ensureLookupIndex(safe, "project_key_name", `("project_key", "name")`);
|
|
571
744
|
}
|
|
572
745
|
};
|
|
573
746
|
|
|
@@ -747,8 +920,24 @@ async function runAuthCommand(args) {
|
|
|
747
920
|
console.log(`Org not found: ${target}`);
|
|
748
921
|
process.exit(1);
|
|
749
922
|
}
|
|
923
|
+
const prevWs = creds.workspaceId ?? "default";
|
|
924
|
+
const lcPrev = prevWs.toLowerCase();
|
|
925
|
+
const wsList = await listWorkspaces(creds.token, apiUrl, match.id);
|
|
926
|
+
const matchedWs = wsList.find((w) => w.id === prevWs || w.name && w.name.toLowerCase() === lcPrev);
|
|
750
927
|
await switchOrg(match.id, match.name);
|
|
751
928
|
console.log(`Switched to org: ${match.name}`);
|
|
929
|
+
if (!matchedWs) {
|
|
930
|
+
if (prevWs !== "default") {
|
|
931
|
+
await switchWorkspace("default");
|
|
932
|
+
console.log(`Workspace '${prevWs}' is not in org '${match.name}'. Reset workspace to 'default'.`);
|
|
933
|
+
if (wsList.length > 0) {
|
|
934
|
+
console.log(`Available workspaces: ${wsList.map((w) => w.name || w.id).join(", ")}`);
|
|
935
|
+
}
|
|
936
|
+
}
|
|
937
|
+
} else if (matchedWs.id !== prevWs) {
|
|
938
|
+
await switchWorkspace(matchedWs.id);
|
|
939
|
+
console.log(`Workspace name '${prevWs}' resolved to id '${matchedWs.id}' in org '${match.name}'.`);
|
|
940
|
+
}
|
|
752
941
|
} else {
|
|
753
942
|
console.log("Usage: org list | org switch <name-or-id>");
|
|
754
943
|
}
|
|
@@ -760,7 +949,7 @@ async function runAuthCommand(args) {
|
|
|
760
949
|
process.exit(1);
|
|
761
950
|
}
|
|
762
951
|
const ws = await listWorkspaces(creds.token, apiUrl, creds.orgId);
|
|
763
|
-
ws.forEach((w) => console.log(
|
|
952
|
+
ws.forEach((w) => console.log(w.name || w.id));
|
|
764
953
|
break;
|
|
765
954
|
}
|
|
766
955
|
case "workspace": {
|
|
@@ -768,14 +957,34 @@ async function runAuthCommand(args) {
|
|
|
768
957
|
console.log("Not logged in.");
|
|
769
958
|
process.exit(1);
|
|
770
959
|
}
|
|
771
|
-
const
|
|
772
|
-
if (
|
|
773
|
-
|
|
774
|
-
|
|
960
|
+
const sub = args[1];
|
|
961
|
+
if (sub === "list") {
|
|
962
|
+
const wsList = await listWorkspaces(creds.token, apiUrl, creds.orgId);
|
|
963
|
+
wsList.forEach((w) => console.log(w.name || w.id));
|
|
964
|
+
break;
|
|
775
965
|
}
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
966
|
+
if (sub === "switch") {
|
|
967
|
+
const target = args[2];
|
|
968
|
+
if (!target) {
|
|
969
|
+
console.log("Usage: workspace switch <name-or-id>");
|
|
970
|
+
process.exit(1);
|
|
971
|
+
}
|
|
972
|
+
const wsList = await listWorkspaces(creds.token, apiUrl, creds.orgId);
|
|
973
|
+
const lcTarget = target.toLowerCase();
|
|
974
|
+
const match = wsList.find((w) => w.id === target || w.name && w.name.toLowerCase() === lcTarget);
|
|
975
|
+
if (!match) {
|
|
976
|
+
console.log(`Workspace not found: ${target}`);
|
|
977
|
+
if (wsList.length > 0) {
|
|
978
|
+
console.log(`Available workspaces: ${wsList.map((w) => w.name || w.id).join(", ")}`);
|
|
979
|
+
}
|
|
980
|
+
process.exit(1);
|
|
981
|
+
}
|
|
982
|
+
await switchWorkspace(match.id);
|
|
983
|
+
console.log(`Switched to workspace: ${match.name || match.id}`);
|
|
984
|
+
break;
|
|
985
|
+
}
|
|
986
|
+
console.log("Usage: workspace list | workspace switch <name-or-id>");
|
|
987
|
+
process.exit(1);
|
|
779
988
|
}
|
|
780
989
|
case "invite": {
|
|
781
990
|
if (!creds) {
|