@deeplake/hivemind 0.6.47

package/cursor/bundle/capture.js

@@ -0,0 +1,485 @@
+#!/usr/bin/env node
+
+// dist/src/utils/stdin.js
+function readStdin() {
+  return new Promise((resolve, reject) => {
+    let data = "";
+    process.stdin.setEncoding("utf-8");
+    process.stdin.on("data", (chunk) => data += chunk);
+    process.stdin.on("end", () => {
+      try {
+        resolve(JSON.parse(data));
+      } catch (err) {
+        reject(new Error(`Failed to parse hook input: ${err}`));
+      }
+    });
+    process.stdin.on("error", reject);
+  });
+}
+
+// dist/src/config.js
+import { readFileSync, existsSync } from "node:fs";
+import { join } from "node:path";
+import { homedir, userInfo } from "node:os";
+function loadConfig() {
+  const home = homedir();
+  const credPath = join(home, ".deeplake", "credentials.json");
+  let creds = null;
+  if (existsSync(credPath)) {
+    try {
+      creds = JSON.parse(readFileSync(credPath, "utf-8"));
+    } catch {
+      return null;
+    }
+  }
+  const token = process.env.HIVEMIND_TOKEN ?? creds?.token;
+  const orgId = process.env.HIVEMIND_ORG_ID ?? creds?.orgId;
+  if (!token || !orgId)
+    return null;
+  return {
+    token,
+    orgId,
+    orgName: creds?.orgName ?? orgId,
+    userName: creds?.userName || userInfo().username || "unknown",
+    workspaceId: process.env.HIVEMIND_WORKSPACE_ID ?? creds?.workspaceId ?? "default",
+    apiUrl: process.env.HIVEMIND_API_URL ?? creds?.apiUrl ?? "https://api.deeplake.ai",
+    tableName: process.env.HIVEMIND_TABLE ?? "memory",
+    sessionsTableName: process.env.HIVEMIND_SESSIONS_TABLE ?? "sessions",
+    memoryPath: process.env.HIVEMIND_MEMORY_PATH ?? join(home, ".deeplake", "memory")
+  };
+}
+
+// dist/src/deeplake-api.js
+import { randomUUID } from "node:crypto";
+import { existsSync as existsSync2, mkdirSync, readFileSync as readFileSync2, writeFileSync } from "node:fs";
+import { join as join3 } from "node:path";
+import { tmpdir } from "node:os";
+
+// dist/src/utils/debug.js
+import { appendFileSync } from "node:fs";
+import { join as join2 } from "node:path";
+import { homedir as homedir2 } from "node:os";
+var DEBUG = process.env.HIVEMIND_DEBUG === "1";
+var LOG = join2(homedir2(), ".deeplake", "hook-debug.log");
+function log(tag, msg) {
+  if (!DEBUG)
+    return;
+  appendFileSync(LOG, `${(/* @__PURE__ */ new Date()).toISOString()} [${tag}] ${msg}
+`);
+}
+
+// dist/src/utils/sql.js
+function sqlStr(value) {
+  return value.replace(/\\/g, "\\\\").replace(/'/g, "''").replace(/\0/g, "").replace(/[\x01-\x08\x0b\x0c\x0e-\x1f\x7f]/g, "");
+}
+
+// dist/src/deeplake-api.js
+var log2 = (msg) => log("sdk", msg);
+function summarizeSql(sql, maxLen = 220) {
+  const compact = sql.replace(/\s+/g, " ").trim();
+  return compact.length > maxLen ? `${compact.slice(0, maxLen)}...` : compact;
+}
+function traceSql(msg) {
+  const traceEnabled = process.env.HIVEMIND_TRACE_SQL === "1" || process.env.HIVEMIND_DEBUG === "1";
+  if (!traceEnabled)
+    return;
+  process.stderr.write(`[deeplake-sql] ${msg}
+`);
+  if (process.env.HIVEMIND_DEBUG === "1")
+    log2(msg);
+}
+var RETRYABLE_CODES = /* @__PURE__ */ new Set([429, 500, 502, 503, 504]);
+var MAX_RETRIES = 3;
+var BASE_DELAY_MS = 500;
+var MAX_CONCURRENCY = 5;
+var QUERY_TIMEOUT_MS = Number(process.env.HIVEMIND_QUERY_TIMEOUT_MS ?? 1e4);
+var INDEX_MARKER_TTL_MS = Number(process.env.HIVEMIND_INDEX_MARKER_TTL_MS ?? 6 * 60 * 6e4);
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function isTimeoutError(error) {
+  const name = error instanceof Error ? error.name.toLowerCase() : "";
+  const message = error instanceof Error ? error.message.toLowerCase() : String(error).toLowerCase();
+  return name.includes("timeout") || name === "aborterror" || message.includes("timeout") || message.includes("timed out");
+}
+function isDuplicateIndexError(error) {
+  const message = error instanceof Error ? error.message.toLowerCase() : String(error).toLowerCase();
+  return message.includes("duplicate key value violates unique constraint") || message.includes("pg_class_relname_nsp_index") || message.includes("already exists");
+}
+function isSessionInsertQuery(sql) {
+  return /^\s*insert\s+into\s+"[^"]+"\s*\(\s*id\s*,\s*path\s*,\s*filename\s*,\s*message\s*,/i.test(sql);
+}
+function isTransientHtml403(text) {
+  const body = text.toLowerCase();
+  return body.includes("<html") || body.includes("403 forbidden") || body.includes("cloudflare") || body.includes("nginx");
+}
+function getIndexMarkerDir() {
+  return process.env.HIVEMIND_INDEX_MARKER_DIR ?? join3(tmpdir(), "hivemind-deeplake-indexes");
+}
+var Semaphore = class {
+  max;
+  waiting = [];
+  active = 0;
+  constructor(max) {
+    this.max = max;
+  }
+  async acquire() {
+    if (this.active < this.max) {
+      this.active++;
+      return;
+    }
+    await new Promise((resolve) => this.waiting.push(resolve));
+  }
+  release() {
+    this.active--;
+    const next = this.waiting.shift();
+    if (next) {
+      this.active++;
+      next();
+    }
+  }
+};
+var DeeplakeApi = class {
+  token;
+  apiUrl;
+  orgId;
+  workspaceId;
+  tableName;
+  _pendingRows = [];
+  _sem = new Semaphore(MAX_CONCURRENCY);
+  _tablesCache = null;
+  constructor(token, apiUrl, orgId, workspaceId, tableName) {
+    this.token = token;
+    this.apiUrl = apiUrl;
+    this.orgId = orgId;
+    this.workspaceId = workspaceId;
+    this.tableName = tableName;
+  }
+  /** Execute SQL with retry on transient errors and bounded concurrency. */
+  async query(sql) {
+    const startedAt = Date.now();
+    const summary = summarizeSql(sql);
+    traceSql(`query start: ${summary}`);
+    await this._sem.acquire();
+    try {
+      const rows = await this._queryWithRetry(sql);
+      traceSql(`query ok (${Date.now() - startedAt}ms, rows=${rows.length}): ${summary}`);
+      return rows;
+    } catch (e) {
+      const message = e instanceof Error ? e.message : String(e);
+      traceSql(`query fail (${Date.now() - startedAt}ms): ${summary} :: ${message}`);
+      throw e;
+    } finally {
+      this._sem.release();
+    }
+  }
+  async _queryWithRetry(sql) {
+    let lastError;
+    for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+      let resp;
+      try {
+        const signal = AbortSignal.timeout(QUERY_TIMEOUT_MS);
+        resp = await fetch(`${this.apiUrl}/workspaces/${this.workspaceId}/tables/query`, {
+          method: "POST",
+          headers: {
+            Authorization: `Bearer ${this.token}`,
+            "Content-Type": "application/json",
+            "X-Activeloop-Org-Id": this.orgId
+          },
+          signal,
+          body: JSON.stringify({ query: sql })
+        });
+      } catch (e) {
+        if (isTimeoutError(e)) {
+          lastError = new Error(`Query timeout after ${QUERY_TIMEOUT_MS}ms`);
+          throw lastError;
+        }
+        lastError = e instanceof Error ? e : new Error(String(e));
+        if (attempt < MAX_RETRIES) {
+          const delay = BASE_DELAY_MS * Math.pow(2, attempt) + Math.random() * 200;
+          log2(`query retry ${attempt + 1}/${MAX_RETRIES} (fetch error: ${lastError.message}) in ${delay.toFixed(0)}ms`);
+          await sleep(delay);
+          continue;
+        }
+        throw lastError;
+      }
+      if (resp.ok) {
+        const raw = await resp.json();
+        if (!raw?.rows || !raw?.columns)
+          return [];
+        return raw.rows.map((row) => Object.fromEntries(raw.columns.map((col, i) => [col, row[i]])));
+      }
+      const text = await resp.text().catch(() => "");
+      const retryable403 = isSessionInsertQuery(sql) && (resp.status === 401 || resp.status === 403 && (text.length === 0 || isTransientHtml403(text)));
+      if (attempt < MAX_RETRIES && (RETRYABLE_CODES.has(resp.status) || retryable403)) {
+        const delay = BASE_DELAY_MS * Math.pow(2, attempt) + Math.random() * 200;
+        log2(`query retry ${attempt + 1}/${MAX_RETRIES} (${resp.status}) in ${delay.toFixed(0)}ms`);
+        await sleep(delay);
+        continue;
+      }
+      throw new Error(`Query failed: ${resp.status}: ${text.slice(0, 200)}`);
+    }
+    throw lastError ?? new Error("Query failed: max retries exceeded");
+  }
+  // ── Writes ──────────────────────────────────────────────────────────────────
+  /** Queue rows for writing. Call commit() to flush. */
+  appendRows(rows) {
+    this._pendingRows.push(...rows);
+  }
+  /** Flush pending rows via SQL. */
+  async commit() {
+    if (this._pendingRows.length === 0)
+      return;
+    const rows = this._pendingRows;
+    this._pendingRows = [];
+    const CONCURRENCY = 10;
+    for (let i = 0; i < rows.length; i += CONCURRENCY) {
+      const chunk = rows.slice(i, i + CONCURRENCY);
+      await Promise.allSettled(chunk.map((r) => this.upsertRowSql(r)));
+    }
+    log2(`commit: ${rows.length} rows`);
+  }
+  async upsertRowSql(row) {
+    const ts = (/* @__PURE__ */ new Date()).toISOString();
+    const cd = row.creationDate ?? ts;
+    const lud = row.lastUpdateDate ?? ts;
+    const exists = await this.query(`SELECT path FROM "${this.tableName}" WHERE path = '${sqlStr(row.path)}' LIMIT 1`);
+    if (exists.length > 0) {
+      let setClauses = `summary = E'${sqlStr(row.contentText)}', mime_type = '${sqlStr(row.mimeType)}', size_bytes = ${row.sizeBytes}, last_update_date = '${lud}'`;
+      if (row.project !== void 0)
+        setClauses += `, project = '${sqlStr(row.project)}'`;
+      if (row.description !== void 0)
+        setClauses += `, description = '${sqlStr(row.description)}'`;
+      await this.query(`UPDATE "${this.tableName}" SET ${setClauses} WHERE path = '${sqlStr(row.path)}'`);
+    } else {
+      const id = randomUUID();
+      let cols = "id, path, filename, summary, mime_type, size_bytes, creation_date, last_update_date";
+      let vals = `'${id}', '${sqlStr(row.path)}', '${sqlStr(row.filename)}', E'${sqlStr(row.contentText)}', '${sqlStr(row.mimeType)}', ${row.sizeBytes}, '${cd}', '${lud}'`;
+      if (row.project !== void 0) {
+        cols += ", project";
+        vals += `, '${sqlStr(row.project)}'`;
+      }
+      if (row.description !== void 0) {
+        cols += ", description";
+        vals += `, '${sqlStr(row.description)}'`;
+      }
+      await this.query(`INSERT INTO "${this.tableName}" (${cols}) VALUES (${vals})`);
+    }
+  }
+  /** Update specific columns on a row by path. */
+  async updateColumns(path, columns) {
+    const setClauses = Object.entries(columns).map(([col, val]) => typeof val === "number" ? `${col} = ${val}` : `${col} = '${sqlStr(String(val))}'`).join(", ");
+    await this.query(`UPDATE "${this.tableName}" SET ${setClauses} WHERE path = '${sqlStr(path)}'`);
+  }
+  // ── Convenience ─────────────────────────────────────────────────────────────
+  /** Create a BM25 search index on a column. */
+  async createIndex(column) {
+    await this.query(`CREATE INDEX IF NOT EXISTS idx_${sqlStr(column)}_bm25 ON "${this.tableName}" USING deeplake_index ("${column}")`);
+  }
+  buildLookupIndexName(table, suffix) {
+    return `idx_${table}_${suffix}`.replace(/[^a-zA-Z0-9_]/g, "_");
+  }
+  getLookupIndexMarkerPath(table, suffix) {
+    const markerKey = [
+      this.workspaceId,
+      this.orgId,
+      table,
+      suffix
+    ].join("__").replace(/[^a-zA-Z0-9_.-]/g, "_");
+    return join3(getIndexMarkerDir(), `${markerKey}.json`);
+  }
+  hasFreshLookupIndexMarker(table, suffix) {
+    const markerPath = this.getLookupIndexMarkerPath(table, suffix);
+    if (!existsSync2(markerPath))
+      return false;
+    try {
+      const raw = JSON.parse(readFileSync2(markerPath, "utf-8"));
+      const updatedAt = raw.updatedAt ? new Date(raw.updatedAt).getTime() : NaN;
+      if (!Number.isFinite(updatedAt) || Date.now() - updatedAt > INDEX_MARKER_TTL_MS)
+        return false;
+      return true;
+    } catch {
+      return false;
+    }
+  }
+  markLookupIndexReady(table, suffix) {
+    mkdirSync(getIndexMarkerDir(), { recursive: true });
+    writeFileSync(this.getLookupIndexMarkerPath(table, suffix), JSON.stringify({ updatedAt: (/* @__PURE__ */ new Date()).toISOString() }), "utf-8");
+  }
+  async ensureLookupIndex(table, suffix, columnsSql) {
+    if (this.hasFreshLookupIndexMarker(table, suffix))
+      return;
+    const indexName = this.buildLookupIndexName(table, suffix);
+    try {
+      await this.query(`CREATE INDEX IF NOT EXISTS "${indexName}" ON "${table}" ${columnsSql}`);
+      this.markLookupIndexReady(table, suffix);
+    } catch (e) {
+      if (isDuplicateIndexError(e)) {
+        this.markLookupIndexReady(table, suffix);
+        return;
+      }
+      log2(`index "${indexName}" skipped: ${e.message}`);
+    }
+  }
+  /** List all tables in the workspace (with retry). */
+  async listTables(forceRefresh = false) {
+    if (!forceRefresh && this._tablesCache)
+      return [...this._tablesCache];
+    const { tables, cacheable } = await this._fetchTables();
+    if (cacheable)
+      this._tablesCache = [...tables];
+    return tables;
+  }
+  async _fetchTables() {
+    for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+      try {
+        const resp = await fetch(`${this.apiUrl}/workspaces/${this.workspaceId}/tables`, {
+          headers: {
+            Authorization: `Bearer ${this.token}`,
+            "X-Activeloop-Org-Id": this.orgId
+          }
+        });
+        if (resp.ok) {
+          const data = await resp.json();
+          return {
+            tables: (data.tables ?? []).map((t) => t.table_name),
+            cacheable: true
+          };
+        }
+        if (attempt < MAX_RETRIES && RETRYABLE_CODES.has(resp.status)) {
+          await sleep(BASE_DELAY_MS * Math.pow(2, attempt) + Math.random() * 200);
+          continue;
+        }
+        return { tables: [], cacheable: false };
+      } catch {
+        if (attempt < MAX_RETRIES) {
+          await sleep(BASE_DELAY_MS * Math.pow(2, attempt));
+          continue;
+        }
+        return { tables: [], cacheable: false };
+      }
+    }
+    return { tables: [], cacheable: false };
+  }
+  /** Create the memory table if it doesn't already exist. Migrate columns on existing tables. */
+  async ensureTable(name) {
+    const tbl = name ?? this.tableName;
+    const tables = await this.listTables();
+    if (!tables.includes(tbl)) {
+      log2(`table "${tbl}" not found, creating`);
+      await this.query(`CREATE TABLE IF NOT EXISTS "${tbl}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'text/plain', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`);
+      log2(`table "${tbl}" created`);
+      if (!tables.includes(tbl))
+        this._tablesCache = [...tables, tbl];
+    }
+  }
+  /** Create the sessions table (uses JSONB for message since every row is a JSON event). */
+  async ensureSessionsTable(name) {
+    const tables = await this.listTables();
+    if (!tables.includes(name)) {
+      log2(`table "${name}" not found, creating`);
+      await this.query(`CREATE TABLE IF NOT EXISTS "${name}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', message JSONB, author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'application/json', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`);
+      log2(`table "${name}" created`);
+      if (!tables.includes(name))
+        this._tablesCache = [...tables, name];
+    }
+    await this.ensureLookupIndex(name, "path_creation_date", `("path", "creation_date")`);
+  }
+};
+
+// dist/src/utils/session-path.js
+function buildSessionPath(config, sessionId) {
+  const workspace = config.workspaceId ?? "default";
+  return `/sessions/${config.userName}/${config.userName}_${config.orgName}_${workspace}_${sessionId}.jsonl`;
+}
+
+// dist/src/hooks/cursor/capture.js
+var log3 = (msg) => log("cursor-capture", msg);
+var CAPTURE = process.env.HIVEMIND_CAPTURE !== "false";
+function resolveCwd(input) {
+  if (typeof input.cwd === "string" && input.cwd)
+    return input.cwd;
+  const roots = input.workspace_roots;
+  if (Array.isArray(roots) && roots.length > 0 && typeof roots[0] === "string")
+    return roots[0];
+  return "";
+}
+async function main() {
+  if (!CAPTURE)
+    return;
+  const input = await readStdin();
+  const config = loadConfig();
+  if (!config) {
+    log3("no config");
+    return;
+  }
+  const sessionId = input.conversation_id ?? `cursor-${Date.now()}`;
+  const event = input.hook_event_name ?? "";
+  const cwd = resolveCwd(input);
+  const sessionsTable = config.sessionsTableName;
+  const api = new DeeplakeApi(config.token, config.apiUrl, config.orgId, config.workspaceId, sessionsTable);
+  const ts = (/* @__PURE__ */ new Date()).toISOString();
+  const meta = {
+    session_id: sessionId,
+    transcript_path: input.transcript_path,
+    cwd,
+    hook_event_name: event,
+    model: input.model,
+    timestamp: ts
+  };
+  let entry = null;
+  if (event === "beforeSubmitPrompt" && typeof input.prompt === "string") {
+    log3(`user session=${sessionId}`);
+    entry = { id: crypto.randomUUID(), ...meta, type: "user_message", content: input.prompt };
+  } else if (event === "postToolUse" && typeof input.tool_name === "string") {
+    log3(`tool=${input.tool_name} session=${sessionId}`);
+    entry = {
+      id: crypto.randomUUID(),
+      ...meta,
+      type: "tool_call",
+      tool_name: input.tool_name,
+      tool_use_id: input.tool_use_id,
+      tool_input: JSON.stringify(input.tool_input),
+      // Cursor delivers tool_output as a JSON-encoded string already.
+      tool_response: typeof input.tool_output === "string" ? input.tool_output : JSON.stringify(input.tool_output)
+    };
+  } else if (event === "afterAgentResponse" && typeof input.text === "string") {
+    log3(`assistant session=${sessionId}`);
+    entry = { id: crypto.randomUUID(), ...meta, type: "assistant_message", content: input.text };
+  } else if (event === "stop") {
+    log3(`stop session=${sessionId} status=${input.status ?? "unknown"}`);
+    entry = {
+      id: crypto.randomUUID(),
+      ...meta,
+      type: "stop",
+      status: input.status,
+      loop_count: input.loop_count
+    };
+  } else {
+    log3(`unknown event: ${event}, skipping`);
+    return;
+  }
+  const sessionPath = buildSessionPath(config, sessionId);
+  const line = JSON.stringify(entry);
+  log3(`writing to ${sessionPath}`);
+  const projectName = cwd.split("/").pop() || "unknown";
+  const filename = sessionPath.split("/").pop() ?? "";
+  const jsonForSql = line.replace(/'/g, "''");
+  const insertSql = `INSERT INTO "${sessionsTable}" (id, path, filename, message, author, size_bytes, project, description, agent, creation_date, last_update_date) VALUES ('${crypto.randomUUID()}', '${sqlStr(sessionPath)}', '${sqlStr(filename)}', '${jsonForSql}'::jsonb, '${sqlStr(config.userName)}', ${Buffer.byteLength(line, "utf-8")}, '${sqlStr(projectName)}', '${sqlStr(event)}', 'cursor', '${ts}', '${ts}')`;
+  try {
+    await api.query(insertSql);
+  } catch (e) {
+    if (e.message?.includes("permission denied") || e.message?.includes("does not exist")) {
+      log3("table missing, creating and retrying");
+      await api.ensureSessionsTable(sessionsTable);
+      await api.query(insertSql);
+    } else {
+      throw e;
+    }
+  }
+  log3("capture ok \u2192 cloud");
+}
+main().catch((e) => {
+  log3(`fatal: ${e.message}`);
+  process.exit(0);
+});