@deeplake/hivemind 0.7.34 → 0.7.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -6,13 +6,13 @@
|
|
|
6
6
|
},
|
|
7
7
|
"metadata": {
|
|
8
8
|
"description": "Cloud-backed persistent shared memory for AI agents powered by Deeplake",
|
|
9
|
-
"version": "0.7.
|
|
9
|
+
"version": "0.7.35"
|
|
10
10
|
},
|
|
11
11
|
"plugins": [
|
|
12
12
|
{
|
|
13
13
|
"name": "hivemind",
|
|
14
14
|
"description": "Persistent shared memory powered by Deeplake — captures all session activity and provides cross-session, cross-agent memory search",
|
|
15
|
-
"version": "0.7.
|
|
15
|
+
"version": "0.7.35",
|
|
16
16
|
"source": "./claude-code",
|
|
17
17
|
"homepage": "https://github.com/activeloopai/hivemind"
|
|
18
18
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "hivemind",
|
|
3
3
|
"description": "Cloud-backed persistent memory powered by Deeplake — read, write, and share memory across Claude Code sessions and agents",
|
|
4
|
-
"version": "0.7.
|
|
4
|
+
"version": "0.7.35",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Activeloop",
|
|
7
7
|
"url": "https://deeplake.ai"
|
package/openclaw/dist/index.js
CHANGED
|
@@ -4,6 +4,10 @@ import {
|
|
|
4
4
|
saveCredentials
|
|
5
5
|
} from "./chunks/chunk-OSD5GJJ5.js";
|
|
6
6
|
|
|
7
|
+
// stub:node:child_process
|
|
8
|
+
var spawn = () => {
|
|
9
|
+
};
|
|
10
|
+
|
|
7
11
|
// src/utils/client-header.ts
|
|
8
12
|
var DEEPLAKE_CLIENT_HEADER = "X-Deeplake-Client";
|
|
9
13
|
function deeplakeClientValue() {
|
|
@@ -1153,10 +1157,232 @@ async function readVirtualPathContent(api2, memoryTable2, sessionsTable2, virtua
|
|
|
1153
1157
|
return (await readVirtualPathContents(api2, memoryTable2, sessionsTable2, [virtualPath])).get(virtualPath) ?? null;
|
|
1154
1158
|
}
|
|
1155
1159
|
|
|
1160
|
+
// src/embeddings/standalone-embed-client.ts
|
|
1161
|
+
import { connect } from "node:net";
|
|
1162
|
+
import {
|
|
1163
|
+
openSync as openSync2,
|
|
1164
|
+
closeSync as closeSync2,
|
|
1165
|
+
writeSync,
|
|
1166
|
+
unlinkSync as unlinkSync2,
|
|
1167
|
+
existsSync,
|
|
1168
|
+
readFileSync as readFileSync2
|
|
1169
|
+
} from "node:fs";
|
|
1170
|
+
import { homedir as homedir3 } from "node:os";
|
|
1171
|
+
import { join as join3 } from "node:path";
|
|
1172
|
+
|
|
1173
|
+
// src/embeddings/protocol.ts
|
|
1174
|
+
var DEFAULT_SOCKET_DIR = "/tmp";
|
|
1175
|
+
var DEFAULT_IDLE_TIMEOUT_MS = 10 * 60 * 1e3;
|
|
1176
|
+
var DEFAULT_CLIENT_TIMEOUT_MS = 2e3;
|
|
1177
|
+
function socketPathFor(uid, dir = DEFAULT_SOCKET_DIR) {
|
|
1178
|
+
return `${dir}/hivemind-embed-${uid}.sock`;
|
|
1179
|
+
}
|
|
1180
|
+
function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) {
|
|
1181
|
+
return `${dir}/hivemind-embed-${uid}.pid`;
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
// src/embeddings/standalone-embed-client.ts
|
|
1185
|
+
var SHARED_DAEMON_PATH = join3(homedir3(), ".hivemind", "embed-deps", "embed-daemon.js");
|
|
1186
|
+
var _spawn = spawn;
|
|
1187
|
+
function getUid() {
|
|
1188
|
+
const uid = typeof process.getuid === "function" ? process.getuid() : void 0;
|
|
1189
|
+
return uid !== void 0 ? String(uid) : "default";
|
|
1190
|
+
}
|
|
1191
|
+
function isPidAlive(pid) {
|
|
1192
|
+
if (!Number.isFinite(pid) || pid <= 0) return false;
|
|
1193
|
+
try {
|
|
1194
|
+
process.kill(pid, 0);
|
|
1195
|
+
return true;
|
|
1196
|
+
} catch {
|
|
1197
|
+
return false;
|
|
1198
|
+
}
|
|
1199
|
+
}
|
|
1200
|
+
function readPidFile(path) {
|
|
1201
|
+
let raw;
|
|
1202
|
+
try {
|
|
1203
|
+
raw = readFileSync2(path, "utf-8").trim();
|
|
1204
|
+
} catch {
|
|
1205
|
+
return null;
|
|
1206
|
+
}
|
|
1207
|
+
if (raw === "") return "empty";
|
|
1208
|
+
const pid = Number(raw);
|
|
1209
|
+
if (!pid || Number.isNaN(pid)) return null;
|
|
1210
|
+
return pid;
|
|
1211
|
+
}
|
|
1212
|
+
function connectOnce(socketPath, timeoutMs) {
|
|
1213
|
+
return new Promise((resolve2, reject) => {
|
|
1214
|
+
const sock = connect(socketPath);
|
|
1215
|
+
const to = setTimeout(() => {
|
|
1216
|
+
sock.destroy();
|
|
1217
|
+
reject(new Error("connect timeout"));
|
|
1218
|
+
}, timeoutMs);
|
|
1219
|
+
sock.once("connect", () => {
|
|
1220
|
+
clearTimeout(to);
|
|
1221
|
+
resolve2(sock);
|
|
1222
|
+
});
|
|
1223
|
+
sock.once("error", (e) => {
|
|
1224
|
+
clearTimeout(to);
|
|
1225
|
+
reject(e);
|
|
1226
|
+
});
|
|
1227
|
+
});
|
|
1228
|
+
}
|
|
1229
|
+
function sendAndWait(sock, req, timeoutMs) {
|
|
1230
|
+
return new Promise((resolve2, reject) => {
|
|
1231
|
+
let buf = "";
|
|
1232
|
+
const to = setTimeout(() => {
|
|
1233
|
+
sock.destroy();
|
|
1234
|
+
reject(new Error("request timeout"));
|
|
1235
|
+
}, timeoutMs);
|
|
1236
|
+
sock.setEncoding("utf-8");
|
|
1237
|
+
sock.on("data", (chunk) => {
|
|
1238
|
+
buf += chunk;
|
|
1239
|
+
const nl = buf.indexOf("\n");
|
|
1240
|
+
if (nl === -1) return;
|
|
1241
|
+
clearTimeout(to);
|
|
1242
|
+
try {
|
|
1243
|
+
resolve2(JSON.parse(buf.slice(0, nl)));
|
|
1244
|
+
} catch (e) {
|
|
1245
|
+
reject(e);
|
|
1246
|
+
}
|
|
1247
|
+
});
|
|
1248
|
+
sock.on("error", (e) => {
|
|
1249
|
+
clearTimeout(to);
|
|
1250
|
+
reject(e);
|
|
1251
|
+
});
|
|
1252
|
+
sock.on("end", () => {
|
|
1253
|
+
clearTimeout(to);
|
|
1254
|
+
reject(new Error("connection closed without response"));
|
|
1255
|
+
});
|
|
1256
|
+
sock.write(JSON.stringify(req) + "\n");
|
|
1257
|
+
});
|
|
1258
|
+
}
|
|
1259
|
+
function trySpawnDaemon(daemonEntry, pidPath) {
|
|
1260
|
+
let fd;
|
|
1261
|
+
try {
|
|
1262
|
+
fd = openSync2(pidPath, "wx", 384);
|
|
1263
|
+
writeSync(fd, String(process.pid));
|
|
1264
|
+
} catch {
|
|
1265
|
+
const existing = readPidFile(pidPath);
|
|
1266
|
+
if (existing === "empty") return false;
|
|
1267
|
+
if (existing !== null && isPidAlive(existing)) {
|
|
1268
|
+
return false;
|
|
1269
|
+
}
|
|
1270
|
+
try {
|
|
1271
|
+
unlinkSync2(pidPath);
|
|
1272
|
+
} catch {
|
|
1273
|
+
}
|
|
1274
|
+
try {
|
|
1275
|
+
fd = openSync2(pidPath, "wx", 384);
|
|
1276
|
+
writeSync(fd, String(process.pid));
|
|
1277
|
+
} catch {
|
|
1278
|
+
return false;
|
|
1279
|
+
}
|
|
1280
|
+
}
|
|
1281
|
+
try {
|
|
1282
|
+
const child = _spawn(process.execPath, [daemonEntry], {
|
|
1283
|
+
detached: true,
|
|
1284
|
+
stdio: "ignore"
|
|
1285
|
+
});
|
|
1286
|
+
child.unref();
|
|
1287
|
+
return true;
|
|
1288
|
+
} catch {
|
|
1289
|
+
try {
|
|
1290
|
+
unlinkSync2(pidPath);
|
|
1291
|
+
} catch {
|
|
1292
|
+
}
|
|
1293
|
+
return false;
|
|
1294
|
+
} finally {
|
|
1295
|
+
try {
|
|
1296
|
+
closeSync2(fd);
|
|
1297
|
+
} catch {
|
|
1298
|
+
}
|
|
1299
|
+
}
|
|
1300
|
+
}
|
|
1301
|
+
function maybeCleanupOwnPlaceholder(pidPath) {
|
|
1302
|
+
const existing = readPidFile(pidPath);
|
|
1303
|
+
if (existing === process.pid || existing === "empty") {
|
|
1304
|
+
try {
|
|
1305
|
+
unlinkSync2(pidPath);
|
|
1306
|
+
} catch {
|
|
1307
|
+
}
|
|
1308
|
+
}
|
|
1309
|
+
}
|
|
1310
|
+
async function waitForSocket(socketPath, deadline, connectTimeoutMs) {
|
|
1311
|
+
let delay = 30;
|
|
1312
|
+
while (Date.now() < deadline) {
|
|
1313
|
+
await new Promise((r) => setTimeout(r, delay));
|
|
1314
|
+
delay = Math.min(delay * 1.5, 300);
|
|
1315
|
+
if (!existsSync(socketPath)) continue;
|
|
1316
|
+
try {
|
|
1317
|
+
return await connectOnce(socketPath, connectTimeoutMs);
|
|
1318
|
+
} catch {
|
|
1319
|
+
}
|
|
1320
|
+
}
|
|
1321
|
+
return null;
|
|
1322
|
+
}
|
|
1323
|
+
async function tryEmbedStandalone(text, kind, opts = {}) {
|
|
1324
|
+
const socketDir = opts.socketDir ?? "/tmp";
|
|
1325
|
+
const daemonEntry = opts.daemonEntry ?? SHARED_DAEMON_PATH;
|
|
1326
|
+
const requestTimeoutMs = opts.requestTimeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS;
|
|
1327
|
+
const spawnWaitMs = opts.spawnWaitMs ?? 5e3;
|
|
1328
|
+
const uid = getUid();
|
|
1329
|
+
const socketPath = socketPathFor(uid, socketDir);
|
|
1330
|
+
const pidPath = pidPathFor(uid, socketDir);
|
|
1331
|
+
let sock = null;
|
|
1332
|
+
try {
|
|
1333
|
+
sock = await connectOnce(socketPath, requestTimeoutMs);
|
|
1334
|
+
} catch {
|
|
1335
|
+
}
|
|
1336
|
+
if (!sock) {
|
|
1337
|
+
if (!existsSync(daemonEntry)) {
|
|
1338
|
+
return null;
|
|
1339
|
+
}
|
|
1340
|
+
trySpawnDaemon(daemonEntry, pidPath);
|
|
1341
|
+
const deadline = Date.now() + spawnWaitMs;
|
|
1342
|
+
sock = await waitForSocket(socketPath, deadline, requestTimeoutMs);
|
|
1343
|
+
if (!sock) {
|
|
1344
|
+
maybeCleanupOwnPlaceholder(pidPath);
|
|
1345
|
+
return null;
|
|
1346
|
+
}
|
|
1347
|
+
}
|
|
1348
|
+
try {
|
|
1349
|
+
const req = { op: "embed", id: "1", kind, text };
|
|
1350
|
+
const resp = await sendAndWait(sock, req, requestTimeoutMs);
|
|
1351
|
+
if (resp.error || !resp.embedding || !Array.isArray(resp.embedding)) {
|
|
1352
|
+
return null;
|
|
1353
|
+
}
|
|
1354
|
+
for (const v of resp.embedding) {
|
|
1355
|
+
if (typeof v !== "number" || !Number.isFinite(v)) return null;
|
|
1356
|
+
}
|
|
1357
|
+
return resp.embedding;
|
|
1358
|
+
} catch {
|
|
1359
|
+
return null;
|
|
1360
|
+
} finally {
|
|
1361
|
+
try {
|
|
1362
|
+
sock.end();
|
|
1363
|
+
} catch {
|
|
1364
|
+
}
|
|
1365
|
+
}
|
|
1366
|
+
}
|
|
1367
|
+
function _setSpawnImpl(fn) {
|
|
1368
|
+
_spawn = fn ?? spawn;
|
|
1369
|
+
}
|
|
1370
|
+
|
|
1371
|
+
// src/embeddings/sql.ts
|
|
1372
|
+
function embeddingSqlLiteral(vec) {
|
|
1373
|
+
if (!vec || vec.length === 0) return "NULL";
|
|
1374
|
+
const parts = [];
|
|
1375
|
+
for (const v of vec) {
|
|
1376
|
+
if (!Number.isFinite(v)) return "NULL";
|
|
1377
|
+
parts.push(String(v));
|
|
1378
|
+
}
|
|
1379
|
+
return `ARRAY[${parts.join(",")}]::float4[]`;
|
|
1380
|
+
}
|
|
1381
|
+
|
|
1156
1382
|
// openclaw/src/index.ts
|
|
1157
1383
|
import { fileURLToPath } from "node:url";
|
|
1158
1384
|
import { join as joinPath, dirname as dirnamePath } from "node:path";
|
|
1159
|
-
import { homedir as
|
|
1385
|
+
import { homedir as homedir4, tmpdir } from "node:os";
|
|
1160
1386
|
import {
|
|
1161
1387
|
existsSync as fsExists,
|
|
1162
1388
|
mkdirSync as fsMkdir,
|
|
@@ -1202,6 +1428,7 @@ async function loadConfig() {
|
|
|
1202
1428
|
}
|
|
1203
1429
|
var requireFromOpenclaw = createRequire(import.meta.url);
|
|
1204
1430
|
var { spawn: realSpawn, execFileSync: realExecFileSync } = requireFromOpenclaw("node:child_process");
|
|
1431
|
+
_setSpawnImpl(realSpawn);
|
|
1205
1432
|
var inheritedEnv = process;
|
|
1206
1433
|
function applyOpenclawTuning(pluginConfig) {
|
|
1207
1434
|
const cfg = pluginConfig ?? {};
|
|
@@ -1239,7 +1466,7 @@ function extractLatestVersion(body) {
|
|
|
1239
1466
|
return typeof v === "string" && v.length > 0 ? v : null;
|
|
1240
1467
|
}
|
|
1241
1468
|
function getInstalledVersion() {
|
|
1242
|
-
return "0.7.
|
|
1469
|
+
return "0.7.35".length > 0 ? "0.7.35" : null;
|
|
1243
1470
|
}
|
|
1244
1471
|
function isNewer(latest, current) {
|
|
1245
1472
|
const parse = (v) => v.replace(/-.*$/, "").split(".").map(Number);
|
|
@@ -1346,8 +1573,8 @@ var skillifySpawnedFor = /* @__PURE__ */ new Set();
|
|
|
1346
1573
|
var __openclaw_filename = fileURLToPath(import.meta.url);
|
|
1347
1574
|
var __openclaw_dirname = dirnamePath(__openclaw_filename);
|
|
1348
1575
|
var OPENCLAW_SKILLIFY_WORKER_PATH = joinPath(__openclaw_dirname, "skillify-worker.js");
|
|
1349
|
-
var OPENCLAW_SKILLIFY_STATE_DIR = joinPath(
|
|
1350
|
-
var OPENCLAW_SKILLIFY_LEGACY_STATE_DIR = joinPath(
|
|
1576
|
+
var OPENCLAW_SKILLIFY_STATE_DIR = joinPath(homedir4(), ".deeplake", "state", "skillify");
|
|
1577
|
+
var OPENCLAW_SKILLIFY_LEGACY_STATE_DIR = joinPath(homedir4(), ".deeplake", "state", "skilify");
|
|
1351
1578
|
var openclawSkillifyMigrationAttempted = false;
|
|
1352
1579
|
function migrateOpenclawSkillifyLegacyStateDir() {
|
|
1353
1580
|
if (openclawSkillifyMigrationAttempted) return;
|
|
@@ -1463,7 +1690,7 @@ function spawnOpenclawSkillifyWorker(a) {
|
|
|
1463
1690
|
sessionsTable,
|
|
1464
1691
|
skillsTable,
|
|
1465
1692
|
userName: a.userName,
|
|
1466
|
-
cwd:
|
|
1693
|
+
cwd: homedir4(),
|
|
1467
1694
|
// sentinel — only used by worker if install=project
|
|
1468
1695
|
projectKey,
|
|
1469
1696
|
project,
|
|
@@ -1479,7 +1706,7 @@ function spawnOpenclawSkillifyWorker(a) {
|
|
|
1479
1706
|
cursorModel: void 0,
|
|
1480
1707
|
hermesProvider: void 0,
|
|
1481
1708
|
hermesModel: void 0,
|
|
1482
|
-
skillifyLog: joinPath(
|
|
1709
|
+
skillifyLog: joinPath(homedir4(), ".deeplake", "hivemind-openclaw-skillify.log"),
|
|
1483
1710
|
currentSessionId: a.sessionId,
|
|
1484
1711
|
// Pass the tuning dispatch through so the worker can repopulate its
|
|
1485
1712
|
// own globalThis (each process has its own globalThis). The worker
|
|
@@ -2018,7 +2245,9 @@ One brain for every agent on your team.
|
|
|
2018
2245
|
};
|
|
2019
2246
|
const line = JSON.stringify(entry);
|
|
2020
2247
|
const jsonForSql = line.replace(/'/g, "''");
|
|
2021
|
-
const
|
|
2248
|
+
const embedding = await tryEmbedStandalone(line, "document");
|
|
2249
|
+
const embeddingSql = embeddingSqlLiteral(embedding);
|
|
2250
|
+
const insertSql = `INSERT INTO "${sessionsTable}" (id, path, filename, message, message_embedding, author, size_bytes, project, description, agent, plugin_version, creation_date, last_update_date) VALUES ('${crypto.randomUUID()}', '${sqlStr(sessionPath)}', '${sqlStr(filename)}', '${jsonForSql}'::jsonb, ${embeddingSql}, '${sqlStr(cfg.userName)}', ${Buffer.byteLength(line, "utf-8")}, '${sqlStr(projectName)}', '${sqlStr(msg.role)}', 'openclaw', '${sqlStr(getInstalledVersion() ?? "")}', '${ts}', '${ts}')`;
|
|
2022
2251
|
try {
|
|
2023
2252
|
await dl.query(insertSql);
|
|
2024
2253
|
} catch (e) {
|
package/openclaw/package.json
CHANGED
package/package.json
CHANGED
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
|
|
27
27
|
import {
|
|
28
28
|
readFileSync, existsSync, appendFileSync, mkdirSync, writeFileSync,
|
|
29
|
-
openSync, closeSync, renameSync, readdirSync, statSync, unlinkSync,
|
|
29
|
+
openSync, closeSync, writeSync, renameSync, readdirSync, statSync, unlinkSync,
|
|
30
30
|
constants as fsConstants,
|
|
31
31
|
} from "node:fs";
|
|
32
32
|
import { homedir, tmpdir } from "node:os";
|
|
@@ -164,51 +164,204 @@ async function dlQuery(creds: Creds, sql: string): Promise<unknown[]> {
|
|
|
164
164
|
// Pi avoids importing EmbedClient (which is bundled into other agents but
|
|
165
165
|
// here would break the "raw .ts, zero deps" promise of pi extensions).
|
|
166
166
|
// Instead we open a Unix socket directly to the daemon at the same well-known
|
|
167
|
-
// path EmbedClient uses. If the socket isn't there yet
|
|
168
|
-
//
|
|
169
|
-
// `hivemind embeddings install`)
|
|
170
|
-
//
|
|
171
|
-
//
|
|
172
|
-
//
|
|
167
|
+
// path EmbedClient uses. If the socket isn't there yet AND the canonical
|
|
168
|
+
// daemon binary exists at ~/.hivemind/embed-deps/embed-daemon.js (deposited
|
|
169
|
+
// by `hivemind embeddings install`), we spawn it under an O_EXCL pidfile
|
|
170
|
+
// lock and wait for it to listen. Subsequent agents (codex, CC, cursor,
|
|
171
|
+
// hermes, …) connect to the SAME daemon — pi pays the cold-start cost only
|
|
172
|
+
// when it's the first user on the box. This logic matches the source-tree
|
|
173
|
+
// helper at src/embeddings/standalone-embed-client.ts (kept in lockstep:
|
|
174
|
+
// the unit tests there cover the 11 edge cases mirrored here).
|
|
173
175
|
//
|
|
174
176
|
// Graceful fallback: any failure → return null → caller writes NULL into
|
|
175
|
-
// message_embedding. Embedding is
|
|
177
|
+
// message_embedding. Embedding is NEVER on the critical path; pi must keep
|
|
178
|
+
// working when the daemon is unreachable.
|
|
176
179
|
|
|
177
180
|
const EMBED_DAEMON_ENTRY = join(homedir(), ".hivemind", "embed-deps", "embed-daemon.js");
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
181
|
+
// `process.env.USER` removed as a fallback: even though pi doesn't go
|
|
182
|
+
// through ClawHub static-scan, we keep the source in lockstep with
|
|
183
|
+
// src/embeddings/standalone-embed-client.ts (which DOES) so the two
|
|
184
|
+
// implementations stay byte-identical. On Linux/macOS `process.getuid`
|
|
185
|
+
// is always present; "default" is a fine sentinel elsewhere.
|
|
186
|
+
const EMBED_UID = typeof process.getuid === "function" ? String(process.getuid()) : "default";
|
|
187
|
+
const EMBED_SOCKET_PATH = `/tmp/hivemind-embed-${EMBED_UID}.sock`;
|
|
188
|
+
const EMBED_PID_PATH = `/tmp/hivemind-embed-${EMBED_UID}.pid`;
|
|
189
|
+
|
|
190
|
+
function isPidAlive(pid: number): boolean {
|
|
191
|
+
if (!Number.isFinite(pid) || pid <= 0) return false;
|
|
192
|
+
try { process.kill(pid, 0); return true; } catch { return false; }
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
// Three-state read: "empty" means the file exists but hasn't been
|
|
196
|
+
// written yet — another caller is mid-spawn between openSync(wx) and
|
|
197
|
+
// writeSync(pid). Treating that as stale lets two racing callers each
|
|
198
|
+
// spawn a daemon, the second crashing on bind(). Mirrors
|
|
199
|
+
// src/embeddings/standalone-embed-client.ts:readPidFile.
|
|
200
|
+
function readPidFileInline(path: string): number | "empty" | null {
|
|
201
|
+
let raw: string;
|
|
202
|
+
try { raw = readFileSync(path, "utf-8").trim(); } catch { return null; }
|
|
203
|
+
if (raw === "") return "empty";
|
|
204
|
+
const pid = Number(raw);
|
|
205
|
+
if (!pid || Number.isNaN(pid)) return null;
|
|
206
|
+
return pid;
|
|
207
|
+
}
|
|
182
208
|
|
|
183
|
-
function
|
|
209
|
+
function connectDaemonOnce(timeoutMs: number): Promise<ReturnType<typeof connect> | null> {
|
|
184
210
|
return new Promise((resolve) => {
|
|
185
|
-
let resolved = false;
|
|
186
|
-
const settle = (v: number[] | null) => { if (!resolved) { resolved = true; resolve(v); } };
|
|
187
211
|
const sock = connect(EMBED_SOCKET_PATH);
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
sock.
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
212
|
+
const to = setTimeout(() => { try { sock.destroy(); } catch { /* */ } resolve(null); }, timeoutMs);
|
|
213
|
+
sock.once("connect", () => { clearTimeout(to); resolve(sock); });
|
|
214
|
+
sock.once("error", () => { clearTimeout(to); resolve(null); });
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* Spawn the canonical daemon under an O_EXCL pidfile lock. Returns true
|
|
220
|
+
* if THIS pi turn owns the spawn. Mirrors the helper in
|
|
221
|
+
* src/embeddings/standalone-embed-client.ts:
|
|
222
|
+
* - live pidfile owner (case 6/7) → don't SIGTERM (PID-reuse risk from PR #168), let caller wait
|
|
223
|
+
* - dead/garbage pidfile (case 5) → cleanup + spawn
|
|
224
|
+
* - spawn() throws (case 8) → roll pidfile back so the next turn can retry
|
|
225
|
+
*/
|
|
226
|
+
function trySpawnDaemonInline(): boolean {
|
|
227
|
+
let fd: number;
|
|
228
|
+
try {
|
|
229
|
+
fd = openSync(EMBED_PID_PATH, "wx", 0o600);
|
|
230
|
+
// Write the placeholder PID through the open fd. The previous version
|
|
231
|
+
// used writeFileSync(path, ...) which races with concurrent unlink +
|
|
232
|
+
// re-open elsewhere — it could overwrite another caller's pidfile
|
|
233
|
+
// entirely. writeSync(fd, ...) writes to OUR fd only.
|
|
234
|
+
writeSync(fd, String(process.pid));
|
|
235
|
+
} catch {
|
|
236
|
+
const existing = readPidFileInline(EMBED_PID_PATH);
|
|
237
|
+
// Empty file: another caller won openSync(wx) but hasn't written its
|
|
238
|
+
// PID yet. We MUST NOT unlink + respawn — that lets us race past
|
|
239
|
+
// the legitimate writer and spawn a duplicate daemon. Wait instead.
|
|
240
|
+
if (existing === "empty") return false;
|
|
241
|
+
if (existing !== null && isPidAlive(existing)) {
|
|
242
|
+
// Live owner: another agent / pi turn is bringing the daemon up. Wait.
|
|
243
|
+
return false;
|
|
244
|
+
}
|
|
245
|
+
try { unlinkSync(EMBED_PID_PATH); } catch { /* */ }
|
|
246
|
+
try {
|
|
247
|
+
fd = openSync(EMBED_PID_PATH, "wx", 0o600);
|
|
248
|
+
writeSync(fd, String(process.pid));
|
|
249
|
+
} catch {
|
|
250
|
+
return false; // sub-ms race: another caller claimed it between our unlink and reopen
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
try {
|
|
254
|
+
// No explicit `env: process.env` — it's the spawn default, and a
|
|
255
|
+
// literal `process.env` reference in source kept in lockstep with
|
|
256
|
+
// src/embeddings/standalone-embed-client.ts (which DOES go through
|
|
257
|
+
// ClawHub static-scan from the openclaw bundle).
|
|
258
|
+
const child = spawn(process.execPath, [EMBED_DAEMON_ENTRY], {
|
|
259
|
+
detached: true,
|
|
260
|
+
stdio: "ignore",
|
|
194
261
|
});
|
|
262
|
+
child.unref();
|
|
263
|
+
logHm(`embed: spawned daemon pid=${child.pid}`);
|
|
264
|
+
return true;
|
|
265
|
+
} catch (e: any) {
|
|
266
|
+
logHm(`embed: spawn failed: ${e?.message ?? e}`);
|
|
267
|
+
try { unlinkSync(EMBED_PID_PATH); } catch { /* */ }
|
|
268
|
+
return false;
|
|
269
|
+
} finally {
|
|
270
|
+
try { closeSync(fd); } catch { /* */ }
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
// After a spawnWaitMs timeout with daemon never opening socket, the
|
|
275
|
+
// pidfile still holds OUR placeholder PID. Every subsequent pi turn
|
|
276
|
+
// would see "live owner" (we're still running) and wait forever instead
|
|
277
|
+
// of retrying the spawn. Clean up the placeholder, but only if it's
|
|
278
|
+
// still ours — the daemon may have already overwritten it.
|
|
279
|
+
//
|
|
280
|
+
// Also clears an empty pidfile: if a prior pi turn was SIGKILL'd
|
|
281
|
+
// between openSync(wx) and writeSync(pid), the empty file would persist
|
|
282
|
+
// and every later turn would wait forever. By the time we hit this
|
|
283
|
+
// cleanup we've waited 5s — orders of magnitude longer than the
|
|
284
|
+
// legitimate openSync→writeSync gap.
|
|
285
|
+
function maybeCleanupOwnPlaceholderInline(): void {
|
|
286
|
+
const existing = readPidFileInline(EMBED_PID_PATH);
|
|
287
|
+
if (existing === process.pid || existing === "empty") {
|
|
288
|
+
try { unlinkSync(EMBED_PID_PATH); } catch { /* already gone */ }
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
async function sendEmbedRequest(sock: ReturnType<typeof connect>, text: string, kind: "document" | "query", timeoutMs: number): Promise<number[] | null> {
|
|
293
|
+
return new Promise((resolve) => {
|
|
294
|
+
let resolved = false;
|
|
295
|
+
const settle = (v: number[] | null) => { if (!resolved) { resolved = true; resolve(v); try { sock.destroy(); } catch { /* */ } } };
|
|
296
|
+
let buf = "";
|
|
297
|
+
const timer = setTimeout(() => settle(null), timeoutMs);
|
|
195
298
|
sock.on("data", (chunk: Buffer) => {
|
|
196
299
|
buf += chunk.toString("utf-8");
|
|
197
300
|
const nl = buf.indexOf("\n");
|
|
198
|
-
if (nl
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
301
|
+
if (nl === -1) return;
|
|
302
|
+
clearTimeout(timer);
|
|
303
|
+
try {
|
|
304
|
+
const resp = JSON.parse(buf.slice(0, nl));
|
|
305
|
+
// Daemon may return `{ error: "unknown op" }` from an older protocol — graceful NULL.
|
|
306
|
+
if (!Array.isArray(resp.embedding)) return settle(null);
|
|
307
|
+
// JSON-over-socket is untrusted at runtime. Reject any non-finite
|
|
308
|
+
// element (string, null, NaN, Infinity, object). Without this, a
|
|
309
|
+
// misbehaving daemon could ship bad values that flow into the
|
|
310
|
+
// ARRAY[...]::FLOAT4[] SQL literal.
|
|
311
|
+
for (const v of resp.embedding) {
|
|
312
|
+
if (typeof v !== "number" || !Number.isFinite(v)) return settle(null);
|
|
313
|
+
}
|
|
314
|
+
settle(resp.embedding);
|
|
315
|
+
} catch { settle(null); }
|
|
206
316
|
});
|
|
207
317
|
sock.on("error", () => { clearTimeout(timer); settle(null); });
|
|
208
318
|
sock.on("close", () => { clearTimeout(timer); settle(null); });
|
|
319
|
+
// Protocol shape comes from src/embeddings/protocol.ts: { op, id, kind, text }.
|
|
320
|
+
// id is a string ("1"), not a number, and the verb field is "op" not "type".
|
|
321
|
+
sock.write(JSON.stringify({ op: "embed", id: "1", kind, text }) + "\n");
|
|
209
322
|
});
|
|
210
323
|
}
|
|
211
324
|
|
|
325
|
+
/**
|
|
326
|
+
* Full spawn-on-miss embedding flow. Returns null on any failure; never
|
|
327
|
+
* throws. 11 edge cases mirror the unit tests in
|
|
328
|
+
* tests/shared/standalone-embed-client.test.ts.
|
|
329
|
+
*/
|
|
330
|
+
async function tryEmbedOverSocket(text: string, kind: "document" | "query"): Promise<number[] | null> {
|
|
331
|
+
// Case 3 — happy path: socket alive, daemon ready.
|
|
332
|
+
let sock = await connectDaemonOnce(1000);
|
|
333
|
+
if (!sock) {
|
|
334
|
+
// Case 1 — binary missing: never spawn.
|
|
335
|
+
if (!existsSync(EMBED_DAEMON_ENTRY)) {
|
|
336
|
+
logHm(`embed: no daemon at ${EMBED_DAEMON_ENTRY} — run 'hivemind embeddings install'`);
|
|
337
|
+
return null;
|
|
338
|
+
}
|
|
339
|
+
// Cases 2 / 4 / 5 / 7 / 8 — trySpawn handles them; loser waits.
|
|
340
|
+
trySpawnDaemonInline();
|
|
341
|
+
// Case 9 — poll for socket up to 5s.
|
|
342
|
+
const deadline = Date.now() + 5000;
|
|
343
|
+
let delay = 30;
|
|
344
|
+
while (Date.now() < deadline) {
|
|
345
|
+
await new Promise(r => setTimeout(r, delay));
|
|
346
|
+
delay = Math.min(delay * 1.5, 300);
|
|
347
|
+
if (!existsSync(EMBED_SOCKET_PATH)) continue;
|
|
348
|
+
sock = await connectDaemonOnce(1000);
|
|
349
|
+
if (sock) break;
|
|
350
|
+
}
|
|
351
|
+
if (!sock) {
|
|
352
|
+
// Clean up our placeholder PID so the next pi turn can retry the
|
|
353
|
+
// spawn instead of waiting on us forever.
|
|
354
|
+
maybeCleanupOwnPlaceholderInline();
|
|
355
|
+
logHm(`embed: daemon never opened socket within 5s`);
|
|
356
|
+
return null;
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
// Cases 10 / 11 — request timeout / daemon error → null.
|
|
360
|
+
const v = await sendEmbedRequest(sock, text, kind, 5000);
|
|
361
|
+
if (v === null) logHm(`embed: daemon returned null (timeout or error)`);
|
|
362
|
+
return v;
|
|
363
|
+
}
|
|
364
|
+
|
|
212
365
|
// ---------- summary state + wiki-worker spawn ---------------------------------
|
|
213
366
|
//
|
|
214
367
|
// Mirror of src/hooks/summary-state.ts (same dir, same JSON shape, shared
|
|
@@ -569,39 +722,12 @@ async function embed(text: string): Promise<number[] | null> {
|
|
|
569
722
|
logHm(`embed: skipped (empty text)`);
|
|
570
723
|
return null;
|
|
571
724
|
}
|
|
572
|
-
//
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
// 2) no daemon binary deposited → fallback NULL
|
|
579
|
-
if (!existsSync(EMBED_DAEMON_ENTRY)) {
|
|
580
|
-
logHm(`embed: no daemon at ${EMBED_DAEMON_ENTRY} — run 'hivemind embeddings install'`);
|
|
581
|
-
return null;
|
|
582
|
-
}
|
|
583
|
-
// 3) spawn the canonical daemon detached; daemon's own pidfile lock guards
|
|
584
|
-
// against double-spawn if multiple pi turns race.
|
|
585
|
-
logHm(`embed: spawning daemon at ${EMBED_DAEMON_ENTRY}`);
|
|
586
|
-
try {
|
|
587
|
-
spawn(process.execPath, [EMBED_DAEMON_ENTRY], { detached: true, stdio: "ignore" }).unref();
|
|
588
|
-
} catch (e: any) {
|
|
589
|
-
logHm(`embed: spawn failed: ${e?.message ?? e}`);
|
|
590
|
-
return null;
|
|
591
|
-
}
|
|
592
|
-
// 4) poll for the socket up to ~5s, then retry the embed once
|
|
593
|
-
for (let i = 0; i < 25; i++) {
|
|
594
|
-
await new Promise(r => setTimeout(r, 200));
|
|
595
|
-
if (existsSync(EMBED_SOCKET_PATH)) {
|
|
596
|
-
v = await tryEmbedOverSocket(text, "document");
|
|
597
|
-
if (v !== null) {
|
|
598
|
-
logHm(`embed: ok after spawn (dims=${v.length}, polls=${i + 1})`);
|
|
599
|
-
return v;
|
|
600
|
-
}
|
|
601
|
-
}
|
|
602
|
-
}
|
|
603
|
-
logHm(`embed: timed out after spawn (5s)`);
|
|
604
|
-
return null;
|
|
725
|
+
// Single round-trip: tryEmbedOverSocket spawns the daemon on miss
|
|
726
|
+
// (O_EXCL race-safe, mirrors src/embeddings/standalone-embed-client.ts)
|
|
727
|
+
// and embeds in one call. Returns null on any failure.
|
|
728
|
+
const v = await tryEmbedOverSocket(text, "document");
|
|
729
|
+
if (v !== null) logHm(`embed: ok (dims=${v.length})`);
|
|
730
|
+
return v;
|
|
605
731
|
}
|
|
606
732
|
|
|
607
733
|
function embedSqlLiteral(emb: number[] | null): string {
|