@deeplake/hivemind 0.7.33 → 0.7.35

package/openclaw/dist/index.js

@@ -4,6 +4,10 @@ import {
   saveCredentials
 } from "./chunks/chunk-OSD5GJJ5.js";
 
+// stub:node:child_process
+var spawn = () => {
+};
+
 // src/utils/client-header.ts
 var DEEPLAKE_CLIENT_HEADER = "X-Deeplake-Client";
 function deeplakeClientValue() {
@@ -1153,10 +1157,232 @@ async function readVirtualPathContent(api2, memoryTable2, sessionsTable2, virtua
   return (await readVirtualPathContents(api2, memoryTable2, sessionsTable2, [virtualPath])).get(virtualPath) ?? null;
 }
 
+// src/embeddings/standalone-embed-client.ts
+import { connect } from "node:net";
+import {
+  openSync as openSync2,
+  closeSync as closeSync2,
+  writeSync,
+  unlinkSync as unlinkSync2,
+  existsSync,
+  readFileSync as readFileSync2
+} from "node:fs";
+import { homedir as homedir3 } from "node:os";
+import { join as join3 } from "node:path";
+
+// src/embeddings/protocol.ts
+var DEFAULT_SOCKET_DIR = "/tmp";
+var DEFAULT_IDLE_TIMEOUT_MS = 10 * 60 * 1e3;
+var DEFAULT_CLIENT_TIMEOUT_MS = 2e3;
+function socketPathFor(uid, dir = DEFAULT_SOCKET_DIR) {
+  return `${dir}/hivemind-embed-${uid}.sock`;
+}
+function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) {
+  return `${dir}/hivemind-embed-${uid}.pid`;
+}
+
+// src/embeddings/standalone-embed-client.ts
+var SHARED_DAEMON_PATH = join3(homedir3(), ".hivemind", "embed-deps", "embed-daemon.js");
+var _spawn = spawn;
+function getUid() {
+  const uid = typeof process.getuid === "function" ? process.getuid() : void 0;
+  return uid !== void 0 ? String(uid) : "default";
+}
+function isPidAlive(pid) {
+  if (!Number.isFinite(pid) || pid <= 0) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+function readPidFile(path) {
+  let raw;
+  try {
+    raw = readFileSync2(path, "utf-8").trim();
+  } catch {
+    return null;
+  }
+  if (raw === "") return "empty";
+  const pid = Number(raw);
+  if (!pid || Number.isNaN(pid)) return null;
+  return pid;
+}
+function connectOnce(socketPath, timeoutMs) {
+  return new Promise((resolve2, reject) => {
+    const sock = connect(socketPath);
+    const to = setTimeout(() => {
+      sock.destroy();
+      reject(new Error("connect timeout"));
+    }, timeoutMs);
+    sock.once("connect", () => {
+      clearTimeout(to);
+      resolve2(sock);
+    });
+    sock.once("error", (e) => {
+      clearTimeout(to);
+      reject(e);
+    });
+  });
+}
+function sendAndWait(sock, req, timeoutMs) {
+  return new Promise((resolve2, reject) => {
+    let buf = "";
+    const to = setTimeout(() => {
+      sock.destroy();
+      reject(new Error("request timeout"));
+    }, timeoutMs);
+    sock.setEncoding("utf-8");
+    sock.on("data", (chunk) => {
+      buf += chunk;
+      const nl = buf.indexOf("\n");
+      if (nl === -1) return;
+      clearTimeout(to);
+      try {
+        resolve2(JSON.parse(buf.slice(0, nl)));
+      } catch (e) {
+        reject(e);
+      }
+    });
+    sock.on("error", (e) => {
+      clearTimeout(to);
+      reject(e);
+    });
+    sock.on("end", () => {
+      clearTimeout(to);
+      reject(new Error("connection closed without response"));
+    });
+    sock.write(JSON.stringify(req) + "\n");
+  });
+}
+function trySpawnDaemon(daemonEntry, pidPath) {
+  let fd;
+  try {
+    fd = openSync2(pidPath, "wx", 384);
+    writeSync(fd, String(process.pid));
+  } catch {
+    const existing = readPidFile(pidPath);
+    if (existing === "empty") return false;
+    if (existing !== null && isPidAlive(existing)) {
+      return false;
+    }
+    try {
+      unlinkSync2(pidPath);
+    } catch {
+    }
+    try {
+      fd = openSync2(pidPath, "wx", 384);
+      writeSync(fd, String(process.pid));
+    } catch {
+      return false;
+    }
+  }
+  try {
+    const child = _spawn(process.execPath, [daemonEntry], {
+      detached: true,
+      stdio: "ignore"
+    });
+    child.unref();
+    return true;
+  } catch {
+    try {
+      unlinkSync2(pidPath);
+    } catch {
+    }
+    return false;
+  } finally {
+    try {
+      closeSync2(fd);
+    } catch {
+    }
+  }
+}
+function maybeCleanupOwnPlaceholder(pidPath) {
+  const existing = readPidFile(pidPath);
+  if (existing === process.pid || existing === "empty") {
+    try {
+      unlinkSync2(pidPath);
+    } catch {
+    }
+  }
+}
+async function waitForSocket(socketPath, deadline, connectTimeoutMs) {
+  let delay = 30;
+  while (Date.now() < deadline) {
+    await new Promise((r) => setTimeout(r, delay));
+    delay = Math.min(delay * 1.5, 300);
+    if (!existsSync(socketPath)) continue;
+    try {
+      return await connectOnce(socketPath, connectTimeoutMs);
+    } catch {
+    }
+  }
+  return null;
+}
+async function tryEmbedStandalone(text, kind, opts = {}) {
+  const socketDir = opts.socketDir ?? "/tmp";
+  const daemonEntry = opts.daemonEntry ?? SHARED_DAEMON_PATH;
+  const requestTimeoutMs = opts.requestTimeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS;
+  const spawnWaitMs = opts.spawnWaitMs ?? 5e3;
+  const uid = getUid();
+  const socketPath = socketPathFor(uid, socketDir);
+  const pidPath = pidPathFor(uid, socketDir);
+  let sock = null;
+  try {
+    sock = await connectOnce(socketPath, requestTimeoutMs);
+  } catch {
+  }
+  if (!sock) {
+    if (!existsSync(daemonEntry)) {
+      return null;
+    }
+    trySpawnDaemon(daemonEntry, pidPath);
+    const deadline = Date.now() + spawnWaitMs;
+    sock = await waitForSocket(socketPath, deadline, requestTimeoutMs);
+    if (!sock) {
+      maybeCleanupOwnPlaceholder(pidPath);
+      return null;
+    }
+  }
+  try {
+    const req = { op: "embed", id: "1", kind, text };
+    const resp = await sendAndWait(sock, req, requestTimeoutMs);
+    if (resp.error || !resp.embedding || !Array.isArray(resp.embedding)) {
+      return null;
+    }
+    for (const v of resp.embedding) {
+      if (typeof v !== "number" || !Number.isFinite(v)) return null;
+    }
+    return resp.embedding;
+  } catch {
+    return null;
+  } finally {
+    try {
+      sock.end();
+    } catch {
+    }
+  }
+}
+function _setSpawnImpl(fn) {
+  _spawn = fn ?? spawn;
+}
+
+// src/embeddings/sql.ts
+function embeddingSqlLiteral(vec) {
+  if (!vec || vec.length === 0) return "NULL";
+  const parts = [];
+  for (const v of vec) {
+    if (!Number.isFinite(v)) return "NULL";
+    parts.push(String(v));
+  }
+  return `ARRAY[${parts.join(",")}]::float4[]`;
+}
+
 // openclaw/src/index.ts
 import { fileURLToPath } from "node:url";
 import { join as joinPath, dirname as dirnamePath } from "node:path";
-import { homedir as homedir3, tmpdir } from "node:os";
+import { homedir as homedir4, tmpdir } from "node:os";
 import {
   existsSync as fsExists,
   mkdirSync as fsMkdir,
@@ -1202,6 +1428,7 @@ async function loadConfig() {
 }
 var requireFromOpenclaw = createRequire(import.meta.url);
 var { spawn: realSpawn, execFileSync: realExecFileSync } = requireFromOpenclaw("node:child_process");
+_setSpawnImpl(realSpawn);
 var inheritedEnv = process;
 function applyOpenclawTuning(pluginConfig) {
   const cfg = pluginConfig ?? {};
@@ -1239,7 +1466,7 @@ function extractLatestVersion(body) {
   return typeof v === "string" && v.length > 0 ? v : null;
 }
 function getInstalledVersion() {
-  return "0.7.33".length > 0 ? "0.7.33" : null;
+  return "0.7.35".length > 0 ? "0.7.35" : null;
 }
 function isNewer(latest, current) {
   const parse = (v) => v.replace(/-.*$/, "").split(".").map(Number);
@@ -1346,8 +1573,8 @@ var skillifySpawnedFor = /* @__PURE__ */ new Set();
 var __openclaw_filename = fileURLToPath(import.meta.url);
 var __openclaw_dirname = dirnamePath(__openclaw_filename);
 var OPENCLAW_SKILLIFY_WORKER_PATH = joinPath(__openclaw_dirname, "skillify-worker.js");
-var OPENCLAW_SKILLIFY_STATE_DIR = joinPath(homedir3(), ".deeplake", "state", "skillify");
-var OPENCLAW_SKILLIFY_LEGACY_STATE_DIR = joinPath(homedir3(), ".deeplake", "state", "skilify");
+var OPENCLAW_SKILLIFY_STATE_DIR = joinPath(homedir4(), ".deeplake", "state", "skillify");
+var OPENCLAW_SKILLIFY_LEGACY_STATE_DIR = joinPath(homedir4(), ".deeplake", "state", "skilify");
 var openclawSkillifyMigrationAttempted = false;
 function migrateOpenclawSkillifyLegacyStateDir() {
   if (openclawSkillifyMigrationAttempted) return;
@@ -1463,7 +1690,7 @@ function spawnOpenclawSkillifyWorker(a) {
     sessionsTable,
     skillsTable,
     userName: a.userName,
-    cwd: homedir3(),
+    cwd: homedir4(),
     // sentinel — only used by worker if install=project
     projectKey,
     project,
@@ -1479,7 +1706,7 @@ function spawnOpenclawSkillifyWorker(a) {
     cursorModel: void 0,
     hermesProvider: void 0,
     hermesModel: void 0,
-    skillifyLog: joinPath(homedir3(), ".deeplake", "hivemind-openclaw-skillify.log"),
+    skillifyLog: joinPath(homedir4(), ".deeplake", "hivemind-openclaw-skillify.log"),
     currentSessionId: a.sessionId,
     // Pass the tuning dispatch through so the worker can repopulate its
     // own globalThis (each process has its own globalThis). The worker
@@ -2018,7 +2245,9 @@ One brain for every agent on your team.
                 };
                 const line = JSON.stringify(entry);
                 const jsonForSql = line.replace(/'/g, "''");
-                const insertSql = `INSERT INTO "${sessionsTable}" (id, path, filename, message, author, size_bytes, project, description, agent, plugin_version, creation_date, last_update_date) VALUES ('${crypto.randomUUID()}', '${sqlStr(sessionPath)}', '${sqlStr(filename)}', '${jsonForSql}'::jsonb, '${sqlStr(cfg.userName)}', ${Buffer.byteLength(line, "utf-8")}, '${sqlStr(projectName)}', '${sqlStr(msg.role)}', 'openclaw', '${sqlStr(getInstalledVersion() ?? "")}', '${ts}', '${ts}')`;
+                const embedding = await tryEmbedStandalone(line, "document");
+                const embeddingSql = embeddingSqlLiteral(embedding);
+                const insertSql = `INSERT INTO "${sessionsTable}" (id, path, filename, message, message_embedding, author, size_bytes, project, description, agent, plugin_version, creation_date, last_update_date) VALUES ('${crypto.randomUUID()}', '${sqlStr(sessionPath)}', '${sqlStr(filename)}', '${jsonForSql}'::jsonb, ${embeddingSql}, '${sqlStr(cfg.userName)}', ${Buffer.byteLength(line, "utf-8")}, '${sqlStr(projectName)}', '${sqlStr(msg.role)}', 'openclaw', '${sqlStr(getInstalledVersion() ?? "")}', '${ts}', '${ts}')`;
                 try {
                   await dl.query(insertSql);
                 } catch (e) {

package/openclaw/dist/skillify-worker.js

@@ -3,7 +3,7 @@ globalThis.__hivemind_tuning__ ??= {};
 
 // dist/src/skillify/skillify-worker.js
 import { readFileSync as readFileSync3, writeFileSync as writeFileSync3, existsSync as existsSync5, appendFileSync as appendFileSync2, rmSync } from "node:fs";
-import { join as join6 } from "node:path";
+import { join as join7 } from "node:path";
 
 // dist/src/utils/debug.js
 import { appendFileSync } from "node:fs";
@@ -562,25 +562,34 @@ function resolveRecordScope(args) {
 }
 
 // dist/src/skillify/state.js
-import { readFileSync as readFileSync2, writeFileSync as writeFileSync2, writeSync, mkdirSync as mkdirSync2, renameSync as renameSync2, existsSync as existsSync4, unlinkSync, openSync, closeSync } from "node:fs";
+import { readFileSync as readFileSync2, writeFileSync as writeFileSync2, writeSync, mkdirSync as mkdirSync2, renameSync as renameSync2, rmdirSync, existsSync as existsSync4, lstatSync, unlinkSync, openSync, closeSync } from "node:fs";
 import { execSync } from "node:child_process";
-import { homedir as homedir5 } from "node:os";
 import { createHash } from "node:crypto";
-import { join as join5, basename } from "node:path";
+import { join as join6, basename } from "node:path";
 
 // dist/src/skillify/legacy-migration.js
 import { existsSync as existsSync3, renameSync } from "node:fs";
+import { dirname, join as join5 } from "node:path";
+
+// dist/src/skillify/state-dir.js
 import { homedir as homedir4 } from "node:os";
 import { join as join4 } from "node:path";
+function getStateDir() {
+  const override = globalThis.__hivemind_tuning__.HIVEMIND_STATE_DIR?.trim();
+  return override && override.length > 0 ? override : join4(homedir4(), ".deeplake", "state", "skillify");
+}
+
+// dist/src/skillify/legacy-migration.js
 var dlog = (msg) => log("skillify-migrate", msg);
 var attempted = false;
 function migrateLegacyStateDir() {
+  if (globalThis.__hivemind_tuning__.HIVEMIND_STATE_DIR?.trim())
+    return;
   if (attempted)
     return;
   attempted = true;
-  const root = join4(homedir4(), ".deeplake", "state");
-  const legacy = join4(root, "skilify");
-  const current = join4(root, "skillify");
+  const current = getStateDir();
+  const legacy = join5(dirname(current), "skilify");
   if (!existsSync3(legacy))
     return;
   if (existsSync3(current))
@@ -590,8 +599,8 @@ function migrateLegacyStateDir() {
     dlog(`migrated ${legacy} -> ${current}`);
   } catch (err) {
     const code = err.code;
-    if (code === "EXDEV" || code === "EPERM") {
-      dlog(`migration failed (${code}); leaving legacy dir in place`);
+    if (code === "EXDEV" || code === "EPERM" || code === "ENOENT" || code === "EEXIST" || code === "ENOTEMPTY") {
+      dlog(`migration skipped (${code}); legacy dir left as-is or another process handled it`);
       return;
     }
     throw err;
@@ -600,17 +609,16 @@ function migrateLegacyStateDir() {
 
 // dist/src/skillify/state.js
 var dlog2 = (msg) => log("skillify-state", msg);
-var STATE_DIR = join5(homedir5(), ".deeplake", "state", "skillify");
 var YIELD_BUF = new Int32Array(new SharedArrayBuffer(4));
 var TRIGGER_THRESHOLD = (() => {
   const n = Number(globalThis.__hivemind_tuning__.HIVEMIND_SKILLIFY_EVERY_N_TURNS ?? "");
   return Number.isInteger(n) && n > 0 ? n : 20;
 })();
 function statePath(projectKey) {
-  return join5(STATE_DIR, `${projectKey}.json`);
+  return join6(getStateDir(), `${projectKey}.json`);
 }
 function lockPath(projectKey) {
-  return join5(STATE_DIR, `${projectKey}.lock`);
+  return join6(getStateDir(), `${projectKey}.lock`);
 }
 function readState(projectKey) {
   migrateLegacyStateDir();
@@ -625,7 +633,7 @@ function readState(projectKey) {
 }
 function writeState(projectKey, state) {
   migrateLegacyStateDir();
-  mkdirSync2(STATE_DIR, { recursive: true });
+  mkdirSync2(getStateDir(), { recursive: true });
   const p = statePath(projectKey);
   const tmp = `${p}.${process.pid}.${Date.now()}.tmp`;
   writeFileSync2(tmp, JSON.stringify(state, null, 2));
@@ -633,7 +641,7 @@ function writeState(projectKey, state) {
 }
 function withRmwLock(projectKey, fn) {
   migrateLegacyStateDir();
-  mkdirSync2(STATE_DIR, { recursive: true });
+  mkdirSync2(getStateDir(), { recursive: true });
   const rmw = lockPath(projectKey) + ".rmw";
   const deadline = Date.now() + 2e3;
   let fd = null;
@@ -706,8 +714,8 @@ function releaseWorkerLock(projectKey) {
 var cfg = JSON.parse(readFileSync3(process.argv[2], "utf-8"));
 globalThis.__hivemind_tuning__ = cfg.tuning ?? {};
 var tmpDir = cfg.tmpDir;
-var verdictPath = join6(tmpDir, "verdict.json");
-var promptPath = join6(tmpDir, "prompt.txt");
+var verdictPath = join7(tmpDir, "verdict.json");
+var promptPath = join7(tmpDir, "prompt.txt");
 var SESSIONS_TO_MINE = 10;
 var PAIR_CHAR_CAP = 2e3;
 var TOTAL_PAIRS_CHAR_CAP = 4e4;
@@ -954,9 +962,9 @@ async function main() {
       timeoutMs: 12e4
     });
     try {
-      writeFileSync3(join6(tmpDir, "gate-stdout.txt"), gate.stdout);
+      writeFileSync3(join7(tmpDir, "gate-stdout.txt"), gate.stdout);
       if (gate.stderr)
-        writeFileSync3(join6(tmpDir, "gate-stderr.txt"), gate.stderr);
+        writeFileSync3(join7(tmpDir, "gate-stderr.txt"), gate.stderr);
     } catch {
     }
     if (gate.errored) {

package/openclaw/openclaw.plugin.json

@@ -52,5 +52,5 @@
       }
     }
   },
-  "version": "0.7.33"
+  "version": "0.7.35"
 }

package/openclaw/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hivemind",
-  "version": "0.7.33",
+  "version": "0.7.35",
   "type": "module",
   "description": "Hivemind — cloud-backed persistent shared memory for AI agents, powered by DeepLake",
   "license": "Apache-2.0",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@deeplake/hivemind",
-  "version": "0.7.33",
+  "version": "0.7.35",
   "description": "Cloud-backed persistent shared memory for AI agents powered by Deeplake",
   "type": "module",
   "repository": {