@deeplake/hivemind 0.7.31 → 0.7.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +2 -2
- package/.claude-plugin/plugin.json +1 -1
- package/bundle/cli.js +427 -233
- package/codex/bundle/capture.js +550 -122
- package/codex/bundle/embeddings/embed-daemon.js +55 -4
- package/codex/bundle/pre-tool-use.js +447 -90
- package/codex/bundle/shell/deeplake-shell.js +431 -74
- package/codex/bundle/stop.js +437 -80
- package/codex/bundle/wiki-worker.js +429 -72
- package/cursor/bundle/capture.js +625 -197
- package/cursor/bundle/embeddings/embed-daemon.js +55 -4
- package/cursor/bundle/pre-tool-use.js +432 -75
- package/cursor/bundle/session-start.js +8 -1
- package/cursor/bundle/shell/deeplake-shell.js +431 -74
- package/cursor/bundle/wiki-worker.js +429 -72
- package/hermes/bundle/capture.js +626 -198
- package/hermes/bundle/embeddings/embed-daemon.js +55 -4
- package/hermes/bundle/pre-tool-use.js +431 -74
- package/hermes/bundle/session-start.js +8 -1
- package/hermes/bundle/shell/deeplake-shell.js +431 -74
- package/hermes/bundle/wiki-worker.js +429 -72
- package/openclaw/dist/index.js +1 -1
- package/openclaw/openclaw.plugin.json +1 -1
- package/openclaw/package.json +1 -1
- package/package.json +1 -1
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// dist/src/hooks/codex/wiki-worker.js
|
|
4
|
-
import { readFileSync as
|
|
4
|
+
import { readFileSync as readFileSync5, writeFileSync as writeFileSync4, existsSync as existsSync4, appendFileSync as appendFileSync2, mkdirSync as mkdirSync4, rmSync } from "node:fs";
|
|
5
5
|
import { execFileSync } from "node:child_process";
|
|
6
|
-
import { dirname, join as
|
|
6
|
+
import { dirname as dirname2, join as join7 } from "node:path";
|
|
7
7
|
import { fileURLToPath } from "node:url";
|
|
8
8
|
|
|
9
9
|
// dist/src/hooks/summary-state.js
|
|
@@ -154,9 +154,9 @@ async function uploadSummary(query2, params) {
|
|
|
154
154
|
// dist/src/embeddings/client.js
|
|
155
155
|
import { connect } from "node:net";
|
|
156
156
|
import { spawn } from "node:child_process";
|
|
157
|
-
import { openSync as
|
|
158
|
-
import { homedir as
|
|
159
|
-
import { join as
|
|
157
|
+
import { openSync as openSync3, closeSync as closeSync3, writeSync as writeSync2, unlinkSync as unlinkSync3, existsSync as existsSync3, readFileSync as readFileSync4 } from "node:fs";
|
|
158
|
+
import { homedir as homedir6 } from "node:os";
|
|
159
|
+
import { join as join6 } from "node:path";
|
|
160
160
|
|
|
161
161
|
// dist/src/embeddings/protocol.js
|
|
162
162
|
var DEFAULT_SOCKET_DIR = "/tmp";
|
|
@@ -169,13 +169,234 @@ function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) {
|
|
|
169
169
|
return `${dir}/hivemind-embed-${uid}.pid`;
|
|
170
170
|
}
|
|
171
171
|
|
|
172
|
+
// dist/src/notifications/queue.js
|
|
173
|
+
import { readFileSync as readFileSync2, writeFileSync as writeFileSync2, renameSync as renameSync2, mkdirSync as mkdirSync2, openSync as openSync2, closeSync as closeSync2, unlinkSync as unlinkSync2, statSync } from "node:fs";
|
|
174
|
+
import { join as join3, resolve } from "node:path";
|
|
175
|
+
import { homedir as homedir3 } from "node:os";
|
|
176
|
+
import { setTimeout as sleep } from "node:timers/promises";
|
|
177
|
+
var log2 = (msg) => log("notifications-queue", msg);
|
|
178
|
+
var LOCK_RETRY_MAX = 50;
|
|
179
|
+
var LOCK_RETRY_BASE_MS = 5;
|
|
180
|
+
var LOCK_STALE_MS = 5e3;
|
|
181
|
+
function queuePath() {
|
|
182
|
+
return join3(homedir3(), ".deeplake", "notifications-queue.json");
|
|
183
|
+
}
|
|
184
|
+
function lockPath2() {
|
|
185
|
+
return `${queuePath()}.lock`;
|
|
186
|
+
}
|
|
187
|
+
function readQueue() {
|
|
188
|
+
try {
|
|
189
|
+
const raw = readFileSync2(queuePath(), "utf-8");
|
|
190
|
+
const parsed = JSON.parse(raw);
|
|
191
|
+
if (!parsed || !Array.isArray(parsed.queue)) {
|
|
192
|
+
log2(`queue malformed \u2192 treating as empty`);
|
|
193
|
+
return { queue: [] };
|
|
194
|
+
}
|
|
195
|
+
return { queue: parsed.queue };
|
|
196
|
+
} catch {
|
|
197
|
+
return { queue: [] };
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
function _isQueuePathInsideHome(path, home) {
|
|
201
|
+
const r = resolve(path);
|
|
202
|
+
const h = resolve(home);
|
|
203
|
+
return r.startsWith(h + "/") || r === h;
|
|
204
|
+
}
|
|
205
|
+
function writeQueue(q) {
|
|
206
|
+
const path = queuePath();
|
|
207
|
+
const home = resolve(homedir3());
|
|
208
|
+
if (!_isQueuePathInsideHome(path, home)) {
|
|
209
|
+
throw new Error(`notifications-queue write blocked: ${path} is outside ${home}`);
|
|
210
|
+
}
|
|
211
|
+
mkdirSync2(join3(home, ".deeplake"), { recursive: true, mode: 448 });
|
|
212
|
+
const tmp = `${path}.${process.pid}.tmp`;
|
|
213
|
+
writeFileSync2(tmp, JSON.stringify(q, null, 2), { mode: 384 });
|
|
214
|
+
renameSync2(tmp, path);
|
|
215
|
+
}
|
|
216
|
+
async function withQueueLock(fn) {
|
|
217
|
+
const path = lockPath2();
|
|
218
|
+
mkdirSync2(join3(homedir3(), ".deeplake"), { recursive: true, mode: 448 });
|
|
219
|
+
let fd = null;
|
|
220
|
+
for (let attempt = 0; attempt < LOCK_RETRY_MAX; attempt++) {
|
|
221
|
+
try {
|
|
222
|
+
fd = openSync2(path, "wx", 384);
|
|
223
|
+
break;
|
|
224
|
+
} catch (e) {
|
|
225
|
+
const code = e.code;
|
|
226
|
+
if (code !== "EEXIST")
|
|
227
|
+
throw e;
|
|
228
|
+
try {
|
|
229
|
+
const age = Date.now() - statSync(path).mtimeMs;
|
|
230
|
+
if (age > LOCK_STALE_MS) {
|
|
231
|
+
unlinkSync2(path);
|
|
232
|
+
continue;
|
|
233
|
+
}
|
|
234
|
+
} catch {
|
|
235
|
+
}
|
|
236
|
+
const delay = LOCK_RETRY_BASE_MS * (attempt + 1);
|
|
237
|
+
await sleep(delay);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
if (fd === null) {
|
|
241
|
+
log2(`lock acquisition gave up after ${LOCK_RETRY_MAX} attempts \u2014 proceeding unlocked (last-writer-wins)`);
|
|
242
|
+
return fn();
|
|
243
|
+
}
|
|
244
|
+
try {
|
|
245
|
+
return fn();
|
|
246
|
+
} finally {
|
|
247
|
+
try {
|
|
248
|
+
closeSync2(fd);
|
|
249
|
+
} catch {
|
|
250
|
+
}
|
|
251
|
+
try {
|
|
252
|
+
unlinkSync2(path);
|
|
253
|
+
} catch {
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
function sameDedupKey(a, b) {
|
|
258
|
+
if (a.id !== b.id)
|
|
259
|
+
return false;
|
|
260
|
+
return JSON.stringify(a.dedupKey) === JSON.stringify(b.dedupKey);
|
|
261
|
+
}
|
|
262
|
+
async function enqueueNotification(n) {
|
|
263
|
+
await withQueueLock(() => {
|
|
264
|
+
const q = readQueue();
|
|
265
|
+
if (q.queue.some((existing) => sameDedupKey(existing, n))) {
|
|
266
|
+
return;
|
|
267
|
+
}
|
|
268
|
+
q.queue.push(n);
|
|
269
|
+
writeQueue(q);
|
|
270
|
+
});
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
// dist/src/embeddings/disable.js
|
|
274
|
+
import { createRequire } from "node:module";
|
|
275
|
+
import { homedir as homedir5 } from "node:os";
|
|
276
|
+
import { join as join5 } from "node:path";
|
|
277
|
+
import { pathToFileURL } from "node:url";
|
|
278
|
+
|
|
279
|
+
// dist/src/user-config.js
|
|
280
|
+
import { existsSync as existsSync2, mkdirSync as mkdirSync3, readFileSync as readFileSync3, renameSync as renameSync3, writeFileSync as writeFileSync3 } from "node:fs";
|
|
281
|
+
import { homedir as homedir4 } from "node:os";
|
|
282
|
+
import { dirname, join as join4 } from "node:path";
|
|
283
|
+
var _configPath = () => process.env.HIVEMIND_CONFIG_PATH ?? join4(homedir4(), ".deeplake", "config.json");
|
|
284
|
+
var _cache = null;
|
|
285
|
+
var _migrated = false;
|
|
286
|
+
function readUserConfig() {
|
|
287
|
+
if (_cache !== null)
|
|
288
|
+
return _cache;
|
|
289
|
+
const path = _configPath();
|
|
290
|
+
if (!existsSync2(path)) {
|
|
291
|
+
_cache = {};
|
|
292
|
+
return _cache;
|
|
293
|
+
}
|
|
294
|
+
try {
|
|
295
|
+
const raw = readFileSync3(path, "utf-8");
|
|
296
|
+
const parsed = JSON.parse(raw);
|
|
297
|
+
_cache = isPlainObject(parsed) ? parsed : {};
|
|
298
|
+
} catch {
|
|
299
|
+
_cache = {};
|
|
300
|
+
}
|
|
301
|
+
return _cache;
|
|
302
|
+
}
|
|
303
|
+
function writeUserConfig(patch) {
|
|
304
|
+
const current = readUserConfig();
|
|
305
|
+
const merged = deepMerge(current, patch);
|
|
306
|
+
const path = _configPath();
|
|
307
|
+
const dir = dirname(path);
|
|
308
|
+
if (!existsSync2(dir))
|
|
309
|
+
mkdirSync3(dir, { recursive: true });
|
|
310
|
+
const tmp = `${path}.tmp.${process.pid}`;
|
|
311
|
+
writeFileSync3(tmp, JSON.stringify(merged, null, 2) + "\n", "utf-8");
|
|
312
|
+
renameSync3(tmp, path);
|
|
313
|
+
_cache = merged;
|
|
314
|
+
return merged;
|
|
315
|
+
}
|
|
316
|
+
function getEmbeddingsEnabled() {
|
|
317
|
+
const cfg2 = readUserConfig();
|
|
318
|
+
if (cfg2.embeddings && typeof cfg2.embeddings.enabled === "boolean") {
|
|
319
|
+
return cfg2.embeddings.enabled;
|
|
320
|
+
}
|
|
321
|
+
if (_migrated) {
|
|
322
|
+
return migrationValueFromEnv();
|
|
323
|
+
}
|
|
324
|
+
_migrated = true;
|
|
325
|
+
const enabled = migrationValueFromEnv();
|
|
326
|
+
try {
|
|
327
|
+
writeUserConfig({ embeddings: { enabled } });
|
|
328
|
+
} catch {
|
|
329
|
+
_cache = { ...cfg2 ?? {}, embeddings: { ...cfg2?.embeddings ?? {}, enabled } };
|
|
330
|
+
}
|
|
331
|
+
return enabled;
|
|
332
|
+
}
|
|
333
|
+
function migrationValueFromEnv() {
|
|
334
|
+
const raw = process.env.HIVEMIND_EMBEDDINGS;
|
|
335
|
+
if (raw === void 0)
|
|
336
|
+
return false;
|
|
337
|
+
if (raw === "false")
|
|
338
|
+
return false;
|
|
339
|
+
return true;
|
|
340
|
+
}
|
|
341
|
+
function isPlainObject(value) {
|
|
342
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
343
|
+
}
|
|
344
|
+
function deepMerge(base, patch) {
|
|
345
|
+
const out = { ...base };
|
|
346
|
+
for (const key of Object.keys(patch)) {
|
|
347
|
+
const patchVal = patch[key];
|
|
348
|
+
const baseVal = base[key];
|
|
349
|
+
if (isPlainObject(patchVal) && isPlainObject(baseVal)) {
|
|
350
|
+
out[key] = { ...baseVal, ...patchVal };
|
|
351
|
+
} else if (patchVal !== void 0) {
|
|
352
|
+
out[key] = patchVal;
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
return out;
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
// dist/src/embeddings/disable.js
|
|
359
|
+
var cachedStatus = null;
|
|
360
|
+
function defaultResolveTransformers() {
|
|
361
|
+
const sharedDir = join5(homedir5(), ".hivemind", "embed-deps");
|
|
362
|
+
try {
|
|
363
|
+
createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers");
|
|
364
|
+
return;
|
|
365
|
+
} catch {
|
|
366
|
+
}
|
|
367
|
+
createRequire(import.meta.url).resolve("@huggingface/transformers");
|
|
368
|
+
}
|
|
369
|
+
var _resolve = defaultResolveTransformers;
|
|
370
|
+
var _readEnabled = getEmbeddingsEnabled;
|
|
371
|
+
function detectStatus() {
|
|
372
|
+
if (!_readEnabled())
|
|
373
|
+
return "user-disabled";
|
|
374
|
+
try {
|
|
375
|
+
_resolve();
|
|
376
|
+
return "enabled";
|
|
377
|
+
} catch {
|
|
378
|
+
return "no-transformers";
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
function embeddingsStatus() {
|
|
382
|
+
if (cachedStatus !== null)
|
|
383
|
+
return cachedStatus;
|
|
384
|
+
cachedStatus = detectStatus();
|
|
385
|
+
return cachedStatus;
|
|
386
|
+
}
|
|
387
|
+
function embeddingsDisabled() {
|
|
388
|
+
return embeddingsStatus() !== "enabled";
|
|
389
|
+
}
|
|
390
|
+
|
|
172
391
|
// dist/src/embeddings/client.js
|
|
173
|
-
var SHARED_DAEMON_PATH =
|
|
174
|
-
var
|
|
392
|
+
var SHARED_DAEMON_PATH = join6(homedir6(), ".hivemind", "embed-deps", "embed-daemon.js");
|
|
393
|
+
var log3 = (m) => log("embed-client", m);
|
|
175
394
|
function getUid() {
|
|
176
395
|
const uid = typeof process.getuid === "function" ? process.getuid() : void 0;
|
|
177
396
|
return uid !== void 0 ? String(uid) : process.env.USER ?? "default";
|
|
178
397
|
}
|
|
398
|
+
var _signalledMissingDeps = false;
|
|
399
|
+
var _recycledStuckDaemon = false;
|
|
179
400
|
var EmbedClient = class {
|
|
180
401
|
socketPath;
|
|
181
402
|
pidPath;
|
|
@@ -184,13 +405,14 @@ var EmbedClient = class {
|
|
|
184
405
|
autoSpawn;
|
|
185
406
|
spawnWaitMs;
|
|
186
407
|
nextId = 0;
|
|
408
|
+
helloVerified = false;
|
|
187
409
|
constructor(opts = {}) {
|
|
188
410
|
const uid = getUid();
|
|
189
411
|
const dir = opts.socketDir ?? "/tmp";
|
|
190
412
|
this.socketPath = socketPathFor(uid, dir);
|
|
191
413
|
this.pidPath = pidPathFor(uid, dir);
|
|
192
414
|
this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS;
|
|
193
|
-
this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (
|
|
415
|
+
this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync3(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0);
|
|
194
416
|
this.autoSpawn = opts.autoSpawn ?? true;
|
|
195
417
|
this.spawnWaitMs = opts.spawnWaitMs ?? 5e3;
|
|
196
418
|
}
|
|
@@ -200,8 +422,33 @@ var EmbedClient = class {
|
|
|
200
422
|
*
|
|
201
423
|
* Fire-and-forget spawn on miss: if the daemon isn't up, this call returns
|
|
202
424
|
* null AND kicks off a background spawn. The next call finds a ready daemon.
|
|
425
|
+
*
|
|
426
|
+
* Stuck-daemon recycle: if the daemon returns a transformers-missing
|
|
427
|
+
* error (typical after a marketplace upgrade left an older daemon process
|
|
428
|
+
* alive but with no node_modules accessible from its bundle path), we
|
|
429
|
+
* SIGTERM it and clear its sock/pid so the very next call spawns a fresh
|
|
430
|
+
* daemon from the current bundle. Without this, the stuck daemon would
|
|
431
|
+
* keep poisoning every session until its 10-minute idle-out fires.
|
|
203
432
|
*/
|
|
204
433
|
async embed(text, kind = "document") {
|
|
434
|
+
const v = await this.embedAttempt(text, kind);
|
|
435
|
+
if (v !== "recycled")
|
|
436
|
+
return v;
|
|
437
|
+
if (!this.autoSpawn)
|
|
438
|
+
return null;
|
|
439
|
+
this.trySpawnDaemon();
|
|
440
|
+
await this.waitForDaemonReady();
|
|
441
|
+
const retry = await this.embedAttempt(text, kind);
|
|
442
|
+
return retry === "recycled" ? null : retry;
|
|
443
|
+
}
|
|
444
|
+
/**
|
|
445
|
+
* One round-trip: connect → verify → embed. Returns:
|
|
446
|
+
* - number[] : embedding vector (happy path)
|
|
447
|
+
* - null : timeout / daemon error / transformers-missing
|
|
448
|
+
* - "recycled": verifyDaemonOnce killed the daemon mid-call;
|
|
449
|
+
* caller should respawn and retry once.
|
|
450
|
+
*/
|
|
451
|
+
async embedAttempt(text, kind) {
|
|
205
452
|
let sock;
|
|
206
453
|
try {
|
|
207
454
|
sock = await this.connectOnce();
|
|
@@ -211,17 +458,25 @@ var EmbedClient = class {
|
|
|
211
458
|
return null;
|
|
212
459
|
}
|
|
213
460
|
try {
|
|
461
|
+
const recycled = await this.verifyDaemonOnce(sock);
|
|
462
|
+
if (recycled) {
|
|
463
|
+
return "recycled";
|
|
464
|
+
}
|
|
214
465
|
const id = String(++this.nextId);
|
|
215
466
|
const req = { op: "embed", id, kind, text };
|
|
216
467
|
const resp = await this.sendAndWait(sock, req);
|
|
217
468
|
if (resp.error || !("embedding" in resp) || !resp.embedding) {
|
|
218
|
-
|
|
469
|
+
const err = resp.error ?? "no embedding";
|
|
470
|
+
log3(`embed err: ${err}`);
|
|
471
|
+
if (isTransformersMissingError(err)) {
|
|
472
|
+
this.handleTransformersMissing(err);
|
|
473
|
+
}
|
|
219
474
|
return null;
|
|
220
475
|
}
|
|
221
476
|
return resp.embedding;
|
|
222
477
|
} catch (e) {
|
|
223
478
|
const err = e instanceof Error ? e.message : String(e);
|
|
224
|
-
|
|
479
|
+
log3(`embed failed: ${err}`);
|
|
225
480
|
return null;
|
|
226
481
|
} finally {
|
|
227
482
|
try {
|
|
@@ -230,6 +485,139 @@ var EmbedClient = class {
|
|
|
230
485
|
}
|
|
231
486
|
}
|
|
232
487
|
}
|
|
488
|
+
/**
|
|
489
|
+
* Poll for the sock file to come back after `trySpawnDaemon` — used by
|
|
490
|
+
* the recycle retry path. Best-effort: caps at `spawnWaitMs` and
|
|
491
|
+
* returns regardless so the retry attempt can run.
|
|
492
|
+
*/
|
|
493
|
+
async waitForDaemonReady() {
|
|
494
|
+
const deadline = Date.now() + this.spawnWaitMs;
|
|
495
|
+
while (Date.now() < deadline) {
|
|
496
|
+
if (existsSync3(this.socketPath))
|
|
497
|
+
return;
|
|
498
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
/**
|
|
502
|
+
* Send a `hello` on first successful connect per EmbedClient instance.
|
|
503
|
+
* If the daemon answers with a path that doesn't match our configured
|
|
504
|
+
* daemonEntry — typical after a marketplace upgrade replaced the bundle
|
|
505
|
+
* — SIGTERM the daemon + clear sock/pid so the next call spawns from the
|
|
506
|
+
* current bundle.
|
|
507
|
+
*
|
|
508
|
+
* `helloVerified` is set ONLY after we've seen a compatible response,
|
|
509
|
+
* so a transient probe failure or a recycle-triggering mismatch leaves
|
|
510
|
+
* the flag false; the next reconnect re-runs verification against
|
|
511
|
+
* whatever daemon is then live (typically the fresh spawn).
|
|
512
|
+
*/
|
|
513
|
+
async verifyDaemonOnce(sock) {
|
|
514
|
+
if (this.helloVerified)
|
|
515
|
+
return false;
|
|
516
|
+
if (!this.daemonEntry) {
|
|
517
|
+
this.helloVerified = true;
|
|
518
|
+
return false;
|
|
519
|
+
}
|
|
520
|
+
const id = String(++this.nextId);
|
|
521
|
+
const req = { op: "hello", id };
|
|
522
|
+
let resp;
|
|
523
|
+
try {
|
|
524
|
+
resp = await this.sendAndWait(sock, req);
|
|
525
|
+
} catch (e) {
|
|
526
|
+
log3(`hello probe failed (inconclusive, will retry next connect): ${e instanceof Error ? e.message : String(e)}`);
|
|
527
|
+
return false;
|
|
528
|
+
}
|
|
529
|
+
const hello = resp;
|
|
530
|
+
if (_recycledStuckDaemon) {
|
|
531
|
+
return false;
|
|
532
|
+
}
|
|
533
|
+
if (!hello.daemonPath) {
|
|
534
|
+
_recycledStuckDaemon = true;
|
|
535
|
+
log3(`daemon does not implement hello (older protocol); recycling`);
|
|
536
|
+
this.recycleDaemon(hello.pid);
|
|
537
|
+
return true;
|
|
538
|
+
}
|
|
539
|
+
if (hello.daemonPath !== this.daemonEntry && !existsSync3(hello.daemonPath)) {
|
|
540
|
+
_recycledStuckDaemon = true;
|
|
541
|
+
log3(`daemon path no longer on disk \u2014 running=${hello.daemonPath} (gone) expected=${this.daemonEntry}; recycling`);
|
|
542
|
+
this.recycleDaemon(hello.pid);
|
|
543
|
+
return true;
|
|
544
|
+
}
|
|
545
|
+
this.helloVerified = true;
|
|
546
|
+
return false;
|
|
547
|
+
}
|
|
548
|
+
/**
|
|
549
|
+
* On a transformers-missing error from the daemon, SIGTERM the stuck
|
|
550
|
+
* daemon (the bundle daemon that can't find its deps) and clear
|
|
551
|
+
* sock/pid so the next call spawns fresh. Also enqueue a one-time
|
|
552
|
+
* notification telling the user to run `hivemind embeddings install`
|
|
553
|
+
* — but only when the user has opted in. Suppressed when
|
|
554
|
+
* embeddingsStatus() === "user-disabled" so we don't nag users who
|
|
555
|
+
* explicitly chose to turn embeddings off.
|
|
556
|
+
*/
|
|
557
|
+
handleTransformersMissing(detail) {
|
|
558
|
+
if (!_recycledStuckDaemon) {
|
|
559
|
+
_recycledStuckDaemon = true;
|
|
560
|
+
this.recycleDaemon(null);
|
|
561
|
+
}
|
|
562
|
+
if (_signalledMissingDeps)
|
|
563
|
+
return;
|
|
564
|
+
_signalledMissingDeps = true;
|
|
565
|
+
let status;
|
|
566
|
+
try {
|
|
567
|
+
status = embeddingsStatus();
|
|
568
|
+
} catch {
|
|
569
|
+
status = "enabled";
|
|
570
|
+
}
|
|
571
|
+
if (status === "user-disabled")
|
|
572
|
+
return;
|
|
573
|
+
enqueueNotification({
|
|
574
|
+
id: "embed-deps-missing",
|
|
575
|
+
severity: "warn",
|
|
576
|
+
title: "Hivemind embeddings disabled \u2014 deps missing",
|
|
577
|
+
body: `Semantic memory search is off because @huggingface/transformers is not installed where the daemon can find it. Run \`hivemind embeddings install\` to enable.`,
|
|
578
|
+
dedupKey: { reason: "transformers-missing", detail: detail.slice(0, 200) }
|
|
579
|
+
}).catch((e) => {
|
|
580
|
+
log3(`enqueue embed-deps-missing failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
581
|
+
});
|
|
582
|
+
}
|
|
583
|
+
/**
|
|
584
|
+
* Best-effort SIGTERM + sock/pid cleanup. Tolerant of every missing-file
|
|
585
|
+
* combination and dead-PID cases.
|
|
586
|
+
*
|
|
587
|
+
* Identity check: gate the SIGTERM on the daemon's socket file still
|
|
588
|
+
* existing. We know the daemon was alive moments ago (we either just
|
|
589
|
+
* got a hello response or the caller saw a transformers-missing error
|
|
590
|
+
* the daemon emitted), but if the socket file is gone by the time we
|
|
591
|
+
* try to kill, the daemon process is also gone and the PID we
|
|
592
|
+
* captured may already have been recycled by the OS to an unrelated
|
|
593
|
+
* user process. Mirrors the gate added to `killEmbedDaemon` in the
|
|
594
|
+
* CLI — same failure mode, rarer trigger.
|
|
595
|
+
*/
|
|
596
|
+
recycleDaemon(reportedPid) {
|
|
597
|
+
let pid = reportedPid;
|
|
598
|
+
if (pid === null) {
|
|
599
|
+
try {
|
|
600
|
+
pid = Number.parseInt(readFileSync4(this.pidPath, "utf-8").trim(), 10);
|
|
601
|
+
} catch {
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
if (Number.isFinite(pid) && pid !== null && pid > 0 && existsSync3(this.socketPath)) {
|
|
605
|
+
try {
|
|
606
|
+
process.kill(pid, "SIGTERM");
|
|
607
|
+
} catch {
|
|
608
|
+
}
|
|
609
|
+
} else if (pid !== null) {
|
|
610
|
+
log3(`recycle: socket gone, skipping SIGTERM on possibly-stale pid ${pid}`);
|
|
611
|
+
}
|
|
612
|
+
try {
|
|
613
|
+
unlinkSync3(this.socketPath);
|
|
614
|
+
} catch {
|
|
615
|
+
}
|
|
616
|
+
try {
|
|
617
|
+
unlinkSync3(this.pidPath);
|
|
618
|
+
} catch {
|
|
619
|
+
}
|
|
620
|
+
}
|
|
233
621
|
/**
|
|
234
622
|
* Wait up to spawnWaitMs for the daemon to accept connections, spawning if
|
|
235
623
|
* necessary. Meant for SessionStart / long-running batches — not the hot path.
|
|
@@ -253,7 +641,7 @@ var EmbedClient = class {
|
|
|
253
641
|
}
|
|
254
642
|
}
|
|
255
643
|
connectOnce() {
|
|
256
|
-
return new Promise((
|
|
644
|
+
return new Promise((resolve2, reject) => {
|
|
257
645
|
const sock = connect(this.socketPath);
|
|
258
646
|
const to = setTimeout(() => {
|
|
259
647
|
sock.destroy();
|
|
@@ -261,7 +649,7 @@ var EmbedClient = class {
|
|
|
261
649
|
}, this.timeoutMs);
|
|
262
650
|
sock.once("connect", () => {
|
|
263
651
|
clearTimeout(to);
|
|
264
|
-
|
|
652
|
+
resolve2(sock);
|
|
265
653
|
});
|
|
266
654
|
sock.once("error", (e) => {
|
|
267
655
|
clearTimeout(to);
|
|
@@ -272,16 +660,16 @@ var EmbedClient = class {
|
|
|
272
660
|
trySpawnDaemon() {
|
|
273
661
|
let fd;
|
|
274
662
|
try {
|
|
275
|
-
fd =
|
|
663
|
+
fd = openSync3(this.pidPath, "wx", 384);
|
|
276
664
|
writeSync2(fd, String(process.pid));
|
|
277
665
|
} catch (e) {
|
|
278
666
|
if (this.isPidFileStale()) {
|
|
279
667
|
try {
|
|
280
|
-
|
|
668
|
+
unlinkSync3(this.pidPath);
|
|
281
669
|
} catch {
|
|
282
670
|
}
|
|
283
671
|
try {
|
|
284
|
-
fd =
|
|
672
|
+
fd = openSync3(this.pidPath, "wx", 384);
|
|
285
673
|
writeSync2(fd, String(process.pid));
|
|
286
674
|
} catch {
|
|
287
675
|
return;
|
|
@@ -290,11 +678,11 @@ var EmbedClient = class {
|
|
|
290
678
|
return;
|
|
291
679
|
}
|
|
292
680
|
}
|
|
293
|
-
if (!this.daemonEntry || !
|
|
294
|
-
|
|
681
|
+
if (!this.daemonEntry || !existsSync3(this.daemonEntry)) {
|
|
682
|
+
log3(`daemonEntry not configured or missing: ${this.daemonEntry}`);
|
|
295
683
|
try {
|
|
296
|
-
|
|
297
|
-
|
|
684
|
+
closeSync3(fd);
|
|
685
|
+
unlinkSync3(this.pidPath);
|
|
298
686
|
} catch {
|
|
299
687
|
}
|
|
300
688
|
return;
|
|
@@ -306,14 +694,14 @@ var EmbedClient = class {
|
|
|
306
694
|
env: process.env
|
|
307
695
|
});
|
|
308
696
|
child.unref();
|
|
309
|
-
|
|
697
|
+
log3(`spawned daemon pid=${child.pid}`);
|
|
310
698
|
} finally {
|
|
311
|
-
|
|
699
|
+
closeSync3(fd);
|
|
312
700
|
}
|
|
313
701
|
}
|
|
314
702
|
isPidFileStale() {
|
|
315
703
|
try {
|
|
316
|
-
const raw =
|
|
704
|
+
const raw = readFileSync4(this.pidPath, "utf-8").trim();
|
|
317
705
|
const pid = Number(raw);
|
|
318
706
|
if (!pid || Number.isNaN(pid))
|
|
319
707
|
return true;
|
|
@@ -331,9 +719,9 @@ var EmbedClient = class {
|
|
|
331
719
|
const deadline = Date.now() + this.spawnWaitMs;
|
|
332
720
|
let delay = 30;
|
|
333
721
|
while (Date.now() < deadline) {
|
|
334
|
-
await
|
|
722
|
+
await sleep2(delay);
|
|
335
723
|
delay = Math.min(delay * 1.5, 300);
|
|
336
|
-
if (!
|
|
724
|
+
if (!existsSync3(this.socketPath))
|
|
337
725
|
continue;
|
|
338
726
|
try {
|
|
339
727
|
return await this.connectOnce();
|
|
@@ -343,7 +731,7 @@ var EmbedClient = class {
|
|
|
343
731
|
throw new Error("daemon did not become ready within spawnWaitMs");
|
|
344
732
|
}
|
|
345
733
|
sendAndWait(sock, req) {
|
|
346
|
-
return new Promise((
|
|
734
|
+
return new Promise((resolve2, reject) => {
|
|
347
735
|
let buf = "";
|
|
348
736
|
const to = setTimeout(() => {
|
|
349
737
|
sock.destroy();
|
|
@@ -358,7 +746,7 @@ var EmbedClient = class {
|
|
|
358
746
|
const line = buf.slice(0, nl);
|
|
359
747
|
clearTimeout(to);
|
|
360
748
|
try {
|
|
361
|
-
|
|
749
|
+
resolve2(JSON.parse(line));
|
|
362
750
|
} catch (e) {
|
|
363
751
|
reject(e);
|
|
364
752
|
}
|
|
@@ -375,44 +763,13 @@ var EmbedClient = class {
|
|
|
375
763
|
});
|
|
376
764
|
}
|
|
377
765
|
};
|
|
378
|
-
function
|
|
766
|
+
function sleep2(ms) {
|
|
379
767
|
return new Promise((r) => setTimeout(r, ms));
|
|
380
768
|
}
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
import { join as join4 } from "node:path";
|
|
386
|
-
import { pathToFileURL } from "node:url";
|
|
387
|
-
var cachedStatus = null;
|
|
388
|
-
function defaultResolveTransformers() {
|
|
389
|
-
try {
|
|
390
|
-
createRequire(import.meta.url).resolve("@huggingface/transformers");
|
|
391
|
-
return;
|
|
392
|
-
} catch {
|
|
393
|
-
}
|
|
394
|
-
const sharedDir = join4(homedir4(), ".hivemind", "embed-deps");
|
|
395
|
-
createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers");
|
|
396
|
-
}
|
|
397
|
-
var _resolve = defaultResolveTransformers;
|
|
398
|
-
function detectStatus() {
|
|
399
|
-
if (process.env.HIVEMIND_EMBEDDINGS === "false")
|
|
400
|
-
return "env-disabled";
|
|
401
|
-
try {
|
|
402
|
-
_resolve();
|
|
403
|
-
return "enabled";
|
|
404
|
-
} catch {
|
|
405
|
-
return "no-transformers";
|
|
406
|
-
}
|
|
407
|
-
}
|
|
408
|
-
function embeddingsStatus() {
|
|
409
|
-
if (cachedStatus !== null)
|
|
410
|
-
return cachedStatus;
|
|
411
|
-
cachedStatus = detectStatus();
|
|
412
|
-
return cachedStatus;
|
|
413
|
-
}
|
|
414
|
-
function embeddingsDisabled() {
|
|
415
|
-
return embeddingsStatus() !== "enabled";
|
|
769
|
+
function isTransformersMissingError(err) {
|
|
770
|
+
if (/hivemind embeddings install/i.test(err))
|
|
771
|
+
return true;
|
|
772
|
+
return /@huggingface\/transformers/i.test(err);
|
|
416
773
|
}
|
|
417
774
|
|
|
418
775
|
// dist/src/utils/client-header.js
|
|
@@ -426,13 +783,13 @@ function deeplakeClientHeader() {
|
|
|
426
783
|
|
|
427
784
|
// dist/src/hooks/codex/wiki-worker.js
|
|
428
785
|
var dlog2 = (msg) => log("codex-wiki-worker", msg);
|
|
429
|
-
var cfg = JSON.parse(
|
|
786
|
+
var cfg = JSON.parse(readFileSync5(process.argv[2], "utf-8"));
|
|
430
787
|
var tmpDir = cfg.tmpDir;
|
|
431
|
-
var tmpJsonl =
|
|
432
|
-
var tmpSummary =
|
|
788
|
+
var tmpJsonl = join7(tmpDir, "session.jsonl");
|
|
789
|
+
var tmpSummary = join7(tmpDir, "summary.md");
|
|
433
790
|
function wlog(msg) {
|
|
434
791
|
try {
|
|
435
|
-
|
|
792
|
+
mkdirSync4(cfg.hooksDir, { recursive: true });
|
|
436
793
|
appendFileSync2(cfg.wikiLog, `[${(/* @__PURE__ */ new Date()).toISOString().replace("T", " ").slice(0, 19)}] wiki-worker(${cfg.sessionId}): ${msg}
|
|
437
794
|
`);
|
|
438
795
|
} catch {
|
|
@@ -464,7 +821,7 @@ async function query(sql, retries = 4) {
|
|
|
464
821
|
const base = Math.min(3e4, 2e3 * Math.pow(2, attempt));
|
|
465
822
|
const delay = base + Math.floor(Math.random() * 1e3);
|
|
466
823
|
wlog(`API ${r.status}, retrying in ${delay}ms (attempt ${attempt + 1}/${retries})`);
|
|
467
|
-
await new Promise((
|
|
824
|
+
await new Promise((resolve2) => setTimeout(resolve2, delay));
|
|
468
825
|
continue;
|
|
469
826
|
}
|
|
470
827
|
throw new Error(`API ${r.status}: ${(await r.text()).slice(0, 200)}`);
|
|
@@ -490,7 +847,7 @@ async function main() {
|
|
|
490
847
|
const jsonlLines = rows.length;
|
|
491
848
|
const pathRows = await query(`SELECT DISTINCT path FROM "${cfg.sessionsTable}" WHERE path LIKE '${esc2(`/sessions/%${cfg.sessionId}%`)}' LIMIT 1`);
|
|
492
849
|
const jsonlServerPath = pathRows.length > 0 ? pathRows[0].path : `/sessions/unknown/${cfg.sessionId}.jsonl`;
|
|
493
|
-
|
|
850
|
+
writeFileSync4(tmpJsonl, jsonlContent);
|
|
494
851
|
wlog(`found ${jsonlLines} events at ${jsonlServerPath}`);
|
|
495
852
|
let prevOffset = 0;
|
|
496
853
|
try {
|
|
@@ -500,7 +857,7 @@ async function main() {
|
|
|
500
857
|
const match = existing.match(/\*\*JSONL offset\*\*:\s*(\d+)/);
|
|
501
858
|
if (match)
|
|
502
859
|
prevOffset = parseInt(match[1], 10);
|
|
503
|
-
|
|
860
|
+
writeFileSync4(tmpSummary, existing);
|
|
504
861
|
wlog(`existing summary found, offset=${prevOffset}`);
|
|
505
862
|
}
|
|
506
863
|
} catch {
|
|
@@ -521,15 +878,15 @@ async function main() {
|
|
|
521
878
|
} catch (e) {
|
|
522
879
|
wlog(`codex exec failed: ${e.status ?? e.message}`);
|
|
523
880
|
}
|
|
524
|
-
if (
|
|
525
|
-
const text =
|
|
881
|
+
if (existsSync4(tmpSummary)) {
|
|
882
|
+
const text = readFileSync5(tmpSummary, "utf-8");
|
|
526
883
|
if (text.trim()) {
|
|
527
884
|
const fname = `${cfg.sessionId}.md`;
|
|
528
885
|
const vpath = `/summaries/${cfg.userName}/${fname}`;
|
|
529
886
|
let embedding = null;
|
|
530
887
|
if (!embeddingsDisabled()) {
|
|
531
888
|
try {
|
|
532
|
-
const daemonEntry =
|
|
889
|
+
const daemonEntry = join7(dirname2(fileURLToPath(import.meta.url)), "embeddings", "embed-daemon.js");
|
|
533
890
|
embedding = await new EmbedClient({ daemonEntry }).embed(text, "document");
|
|
534
891
|
} catch (e) {
|
|
535
892
|
wlog(`summary embedding failed, writing NULL: ${e.message}`);
|