@askexenow/exe-os 0.8.36 → 0.8.37

package/dist/lib/cloud-sync.js

@@ -400,6 +400,18 @@ function logError(msg) {
   }
 }
 var LOCALHOST_PATTERNS = /^(localhost|127\.0\.0\.1|\[::1\])$/i;
+var FETCH_TIMEOUT_MS = 3e4;
+async function fetchWithRetry(url, init) {
+  const attempt = async () => {
+    const signal = AbortSignal.timeout(FETCH_TIMEOUT_MS);
+    return fetch(url, { ...init, signal });
+  };
+  const resp = await attempt();
+  if (resp.status >= 500) {
+    return attempt();
+  }
+  return resp;
+}
 function assertSecureEndpoint(endpoint) {
   if (endpoint.startsWith("https://")) return;
   if (endpoint.startsWith("http://")) {
@@ -421,7 +433,7 @@ async function cloudPush(records, maxVersion, config) {
     const json = JSON.stringify(records);
     const compressed = compress(Buffer.from(json, "utf8"));
     const blob = encryptSyncBlob(compressed);
-    const resp = await fetch(`${config.endpoint}/sync/push`, {
+    const resp = await fetchWithRetry(`${config.endpoint}/sync/push`, {
       method: "POST",
       headers: {
         Authorization: `Bearer ${config.apiKey}`,
@@ -439,7 +451,7 @@ async function cloudPush(records, maxVersion, config) {
 async function cloudPull(sinceVersion, config) {
   assertSecureEndpoint(config.endpoint);
   try {
-    const response = await fetch(`${config.endpoint}/sync/pull`, {
+    const response = await fetchWithRetry(`${config.endpoint}/sync/pull`, {
       method: "POST",
       headers: {
         Authorization: `Bearer ${config.apiKey}`,
@@ -555,20 +567,87 @@ async function cloudSync(config) {
   try {
     await cloudPushRoster(config);
   } catch (err) {
-    process.stderr.write(`[cloud-sync] Roster push warning: ${err instanceof Error ? err.message : String(err)}
-`);
+    logError(`[cloud-sync] Roster push: ${err instanceof Error ? err.message : String(err)}`);
   }
   try {
     await cloudPullRoster(config);
   } catch (err) {
-    process.stderr.write(`[cloud-sync] Roster pull warning: ${err instanceof Error ? err.message : String(err)}
-`);
+    logError(`[cloud-sync] Roster pull: ${err instanceof Error ? err.message : String(err)}`);
+  }
+  let behaviorsResult = { pushed: false, pulled: 0 };
+  try {
+    behaviorsResult.pushed = await cloudPushBehaviors(config);
+  } catch (err) {
+    logError(`[cloud-sync] Behaviors push: ${err instanceof Error ? err.message : String(err)}`);
+  }
+  try {
+    const pullResult2 = await cloudPullBehaviors(config);
+    behaviorsResult.pulled = pullResult2.pulled;
+  } catch (err) {
+    logError(`[cloud-sync] Behaviors pull: ${err instanceof Error ? err.message : String(err)}`);
+  }
+  let graphragResult = { pushed: false, pulled: 0 };
+  try {
+    graphragResult.pushed = await cloudPushGraphRAG(config);
+  } catch (err) {
+    logError(`[cloud-sync] GraphRAG push: ${err instanceof Error ? err.message : String(err)}`);
+  }
+  try {
+    const pullResult2 = await cloudPullGraphRAG(config);
+    graphragResult.pulled = pullResult2.pulled;
+  } catch (err) {
+    logError(`[cloud-sync] GraphRAG pull: ${err instanceof Error ? err.message : String(err)}`);
+  }
+  let tasksResult = { pushed: false, pulled: 0 };
+  try {
+    tasksResult.pushed = await cloudPushTasks(config);
+  } catch (err) {
+    logError(`[cloud-sync] Tasks push: ${err instanceof Error ? err.message : String(err)}`);
   }
-  return { pushed, pulled };
+  try {
+    const pullResult2 = await cloudPullTasks(config);
+    tasksResult.pulled = pullResult2.pulled;
+  } catch (err) {
+    logError(`[cloud-sync] Tasks pull: ${err instanceof Error ? err.message : String(err)}`);
+  }
+  let conversationsResult = { pushed: false, pulled: 0 };
+  try {
+    conversationsResult.pushed = await cloudPushConversations(config);
+  } catch (err) {
+    logError(`[cloud-sync] Conversations push: ${err instanceof Error ? err.message : String(err)}`);
+  }
+  try {
+    const pullResult2 = await cloudPullConversations(config);
+    conversationsResult.pulled = pullResult2.pulled;
+  } catch (err) {
+    logError(`[cloud-sync] Conversations pull: ${err instanceof Error ? err.message : String(err)}`);
+  }
+  let documentsResult = { pushed: false, pulled: 0 };
+  try {
+    documentsResult.pushed = await cloudPushDocuments(config);
+  } catch (err) {
+    logError(`[cloud-sync] Documents push: ${err instanceof Error ? err.message : String(err)}`);
+  }
+  try {
+    const pullResult2 = await cloudPullDocuments(config);
+    documentsResult.pulled = pullResult2.pulled;
+  } catch (err) {
+    logError(`[cloud-sync] Documents pull: ${err instanceof Error ? err.message : String(err)}`);
+  }
+  return {
+    pushed,
+    pulled,
+    behaviors: behaviorsResult,
+    graphrag: graphragResult,
+    tasks: tasksResult,
+    conversations: conversationsResult,
+    documents: documentsResult
+  };
 }
 function buildRosterBlob(paths) {
   const rosterPath = paths?.rosterPath ?? path5.join(EXE_AI_DIR, "exe-employees.json");
   const identityDir = paths?.identityDir ?? path5.join(EXE_AI_DIR, "identity");
+  const configPath = paths?.configPath ?? path5.join(EXE_AI_DIR, "config.json");
   let roster = [];
   if (existsSync5(rosterPath)) {
     try {
@@ -585,9 +664,16 @@ function buildRosterBlob(paths) {
       }
     }
   }
-  const content = JSON.stringify({ roster, identities });
+  let config;
+  if (existsSync5(configPath)) {
+    try {
+      config = JSON.parse(readFileSync5(configPath, "utf-8"));
+    } catch {
+    }
+  }
+  const content = JSON.stringify({ roster, identities, config });
   const hash = Buffer.from(content).length;
-  return { roster, identities, version: hash };
+  return { roster, identities, config, version: hash };
 }
 async function cloudPushRoster(config) {
   assertSecureEndpoint(config.endpoint);
@@ -606,7 +692,7 @@ async function cloudPushRoster(config) {
     const json = JSON.stringify(blob);
     const compressed = compress(Buffer.from(json, "utf8"));
     const encrypted = encryptSyncBlob(compressed);
-    const resp = await fetch(`${config.endpoint}/sync/push-roster`, {
+    const resp = await fetchWithRetry(`${config.endpoint}/sync/push-roster`, {
       method: "POST",
       headers: {
         Authorization: `Bearer ${config.apiKey}`,
@@ -635,7 +721,7 @@ async function cloudPushRoster(config) {
 async function cloudPullRoster(config) {
   assertSecureEndpoint(config.endpoint);
   try {
-    const resp = await fetch(`${config.endpoint}/sync/pull-roster`, {
+    const resp = await fetchWithRetry(`${config.endpoint}/sync/pull-roster`, {
       method: "GET",
       headers: {
         Authorization: `Bearer ${config.apiKey}`,
@@ -655,6 +741,20 @@ async function cloudPullRoster(config) {
     return { added: 0 };
   }
 }
+function mergeConfig(remoteConfig, configPath) {
+  const cfgPath = configPath ?? path5.join(EXE_AI_DIR, "config.json");
+  let local = {};
+  if (existsSync5(cfgPath)) {
+    try {
+      local = JSON.parse(readFileSync5(cfgPath, "utf-8"));
+    } catch {
+    }
+  }
+  const merged = { ...remoteConfig, ...local };
+  const dir = path5.dirname(cfgPath);
+  if (!existsSync5(dir)) mkdirSync2(dir, { recursive: true });
+  writeFileSync2(cfgPath, JSON.stringify(merged, null, 2), "utf-8");
+}
 async function mergeRosterFromRemote(remote, paths) {
   const rosterPath = paths?.rosterPath ?? void 0;
   const identityDir = paths?.identityDir ?? path5.join(EXE_AI_DIR, "identity");
@@ -681,14 +781,405 @@ async function mergeRosterFromRemote(remote, paths) {
   if (added > 0) {
     await saveEmployees(localEmployees, rosterPath);
   }
+  if (remote.config && Object.keys(remote.config).length > 0) {
+    try {
+      mergeConfig(remote.config, paths?.configPath);
+    } catch {
+    }
+  }
   return { added };
 }
+async function cloudPushBlob(route, data, metaKey, config) {
+  if (data.length === 0) return { ok: true };
+  assertSecureEndpoint(config.endpoint);
+  const json = JSON.stringify(data);
+  const version = Buffer.from(json).length;
+  try {
+    const client = getClient();
+    const meta = await client.execute({
+      sql: "SELECT value FROM sync_meta WHERE key = ?",
+      args: [metaKey]
+    });
+    const lastVersion = meta.rows.length > 0 ? Number(meta.rows[0].value) : 0;
+    if (version === lastVersion) return { ok: true };
+  } catch {
+  }
+  try {
+    const compressed = compress(Buffer.from(json, "utf8"));
+    const encrypted = encryptSyncBlob(compressed);
+    const resp = await fetchWithRetry(`${config.endpoint}${route}`, {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${config.apiKey}`,
+        "Content-Type": "application/json",
+        "X-Device-Id": loadDeviceId()
+      },
+      body: JSON.stringify({ blob: encrypted })
+    });
+    if (resp.ok) {
+      try {
+        const client = getClient();
+        await client.execute({
+          sql: "INSERT OR REPLACE INTO sync_meta (key, value) VALUES (?, ?)",
+          args: [metaKey, String(version)]
+        });
+      } catch {
+      }
+    }
+    return { ok: resp.ok };
+  } catch (err) {
+    logError(`[cloud-sync] PUSH ${route}: ${err instanceof Error ? err.message : String(err)}`);
+    return { ok: false };
+  }
+}
+async function cloudPullBlob(route, config) {
+  assertSecureEndpoint(config.endpoint);
+  try {
+    const resp = await fetchWithRetry(`${config.endpoint}${route}`, {
+      method: "GET",
+      headers: {
+        Authorization: `Bearer ${config.apiKey}`,
+        "X-Device-Id": loadDeviceId()
+      }
+    });
+    if (!resp.ok) return null;
+    const data = await resp.json();
+    if (!data.blob) return null;
+    const compressed = decryptSyncBlob(data.blob);
+    const json = decompress(compressed).toString("utf8");
+    return JSON.parse(json);
+  } catch (err) {
+    logError(`[cloud-sync] PULL ${route}: ${err instanceof Error ? err.message : String(err)}`);
+    return null;
+  }
+}
+async function cloudPushBehaviors(config) {
+  const client = getClient();
+  const result = await client.execute("SELECT * FROM behaviors");
+  const rows = result.rows;
+  const { ok } = await cloudPushBlob(
+    "/sync/push-behaviors",
+    rows,
+    "last_behaviors_push_version",
+    config
+  );
+  return ok;
+}
+async function cloudPullBehaviors(config) {
+  const remoteBehaviors = await cloudPullBlob(
+    "/sync/pull-behaviors",
+    config
+  );
+  if (!remoteBehaviors || remoteBehaviors.length === 0) return { pulled: 0 };
+  const client = getClient();
+  let pulled = 0;
+  for (const behavior of remoteBehaviors) {
+    const existing = await client.execute({
+      sql: `SELECT COUNT(*) as cnt FROM behaviors
+            WHERE agent_id = ? AND content = ?`,
+      args: [behavior.agent_id, behavior.content]
+    });
+    if (Number(existing.rows[0]?.cnt) > 0) continue;
+    await client.execute({
+      sql: `INSERT OR IGNORE INTO behaviors
+            (id, agent_id, project_name, domain, content, active, priority, created_at, updated_at)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      args: [
+        behavior.id,
+        behavior.agent_id,
+        behavior.project_name ?? null,
+        behavior.domain ?? null,
+        behavior.content,
+        behavior.active,
+        behavior.priority ?? "p1",
+        behavior.created_at,
+        behavior.updated_at
+      ]
+    });
+    pulled++;
+  }
+  return { pulled };
+}
+async function cloudPushGraphRAG(config) {
+  const client = getClient();
+  const [entities, relationships, aliases, entityMems, relMems, hyperedges, hyperedgeNodes] = await Promise.all([
+    client.execute("SELECT * FROM entities"),
+    client.execute("SELECT * FROM relationships"),
+    client.execute("SELECT * FROM entity_aliases"),
+    client.execute("SELECT * FROM entity_memories"),
+    client.execute("SELECT * FROM relationship_memories"),
+    client.execute("SELECT * FROM hyperedges"),
+    client.execute("SELECT * FROM hyperedge_nodes")
+  ]);
+  const blob = {
+    entities: entities.rows,
+    relationships: relationships.rows,
+    entity_aliases: aliases.rows,
+    entity_memories: entityMems.rows,
+    relationship_memories: relMems.rows,
+    hyperedges: hyperedges.rows,
+    hyperedge_nodes: hyperedgeNodes.rows
+  };
+  const { ok } = await cloudPushBlob(
+    "/sync/push-graphrag",
+    [blob],
+    "last_graphrag_push_version",
+    config
+  );
+  return ok;
+}
+async function cloudPullGraphRAG(config) {
+  const data = await cloudPullBlob(
+    "/sync/pull-graphrag",
+    config
+  );
+  if (!data || data.length === 0) return { pulled: 0 };
+  const blob = data[0];
+  const client = getClient();
+  let pulled = 0;
+  if (blob.entities.length > 0) {
+    const stmts = blob.entities.map((e) => ({
+      sql: `INSERT OR IGNORE INTO entities (id, name, type, first_seen, last_seen, properties)
+            VALUES (?, ?, ?, ?, ?, ?)`,
+      args: [e.id, e.name, e.type, e.first_seen, e.last_seen, e.properties ?? "{}"]
+    }));
+    await client.batch(stmts, "write");
+    pulled += stmts.length;
+  }
+  if (blob.relationships.length > 0) {
+    const stmts = blob.relationships.map((r) => ({
+      sql: `INSERT OR IGNORE INTO relationships
+            (id, source_entity_id, target_entity_id, type, weight, timestamp, properties, confidence, confidence_label)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      args: [
+        r.id,
+        r.source_entity_id,
+        r.target_entity_id,
+        r.type,
+        r.weight ?? 1,
+        r.timestamp,
+        r.properties ?? "{}",
+        r.confidence ?? 1,
+        r.confidence_label ?? "extracted"
+      ]
+    }));
+    await client.batch(stmts, "write");
+    pulled += stmts.length;
+  }
+  if (blob.entity_aliases.length > 0) {
+    const stmts = blob.entity_aliases.map((a) => ({
+      sql: `INSERT OR IGNORE INTO entity_aliases (alias, canonical_entity_id) VALUES (?, ?)`,
+      args: [a.alias, a.canonical_entity_id]
+    }));
+    await client.batch(stmts, "write");
+    pulled += stmts.length;
+  }
+  if (blob.entity_memories.length > 0) {
+    const stmts = blob.entity_memories.map((em) => ({
+      sql: `INSERT OR IGNORE INTO entity_memories (entity_id, memory_id) VALUES (?, ?)`,
+      args: [em.entity_id, em.memory_id]
+    }));
+    await client.batch(stmts, "write");
+    pulled += stmts.length;
+  }
+  if (blob.relationship_memories.length > 0) {
+    const stmts = blob.relationship_memories.map((rm) => ({
+      sql: `INSERT OR IGNORE INTO relationship_memories (relationship_id, memory_id) VALUES (?, ?)`,
+      args: [rm.relationship_id, rm.memory_id]
+    }));
+    await client.batch(stmts, "write");
+    pulled += stmts.length;
+  }
+  if (blob.hyperedges.length > 0) {
+    const stmts = blob.hyperedges.map((h) => ({
+      sql: `INSERT OR IGNORE INTO hyperedges (id, label, relation, confidence, timestamp)
+            VALUES (?, ?, ?, ?, ?)`,
+      args: [h.id, h.label, h.relation, h.confidence ?? 1, h.timestamp]
+    }));
+    await client.batch(stmts, "write");
+    pulled += stmts.length;
+  }
+  if (blob.hyperedge_nodes.length > 0) {
+    const stmts = blob.hyperedge_nodes.map((hn) => ({
+      sql: `INSERT OR IGNORE INTO hyperedge_nodes (hyperedge_id, entity_id) VALUES (?, ?)`,
+      args: [hn.hyperedge_id, hn.entity_id]
+    }));
+    await client.batch(stmts, "write");
+    pulled += stmts.length;
+  }
+  return { pulled };
+}
+async function cloudPushTasks(config) {
+  const client = getClient();
+  const result = await client.execute("SELECT * FROM tasks");
+  const rows = result.rows;
+  const { ok } = await cloudPushBlob(
+    "/sync/push-tasks",
+    rows,
+    "last_tasks_push_version",
+    config
+  );
+  return ok;
+}
+async function cloudPullTasks(config) {
+  const remoteTasks = await cloudPullBlob(
+    "/sync/pull-tasks",
+    config
+  );
+  if (!remoteTasks || remoteTasks.length === 0) return { pulled: 0 };
+  const client = getClient();
+  const stmts = remoteTasks.map((t) => ({
+    sql: `INSERT OR IGNORE INTO tasks
+          (id, title, assigned_to, assigned_by, project_name, priority, status, task_file, created_at, updated_at,
+           blocked_by, parent_task_id, budget_tokens, budget_fallback_model, tokens_used, tokens_warned_at)
+          VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+    args: [
+      t.id,
+      t.title,
+      t.assigned_to,
+      t.assigned_by,
+      t.project_name,
+      t.priority ?? "p1",
+      t.status ?? "open",
+      t.task_file ?? null,
+      t.created_at,
+      t.updated_at,
+      t.blocked_by ?? null,
+      t.parent_task_id ?? null,
+      t.budget_tokens ?? null,
+      t.budget_fallback_model ?? null,
+      t.tokens_used ?? 0,
+      t.tokens_warned_at ?? null
+    ]
+  }));
+  await client.batch(stmts, "write");
+  return { pulled: remoteTasks.length };
+}
+async function cloudPushConversations(config) {
+  const client = getClient();
+  const result = await client.execute("SELECT * FROM conversations");
+  const rows = result.rows;
+  const { ok } = await cloudPushBlob(
+    "/sync/push-conversations",
+    rows,
+    "last_conversations_push_version",
+    config
+  );
+  return ok;
+}
+async function cloudPullConversations(config) {
+  const remoteConvos = await cloudPullBlob(
+    "/sync/pull-conversations",
+    config
+  );
+  if (!remoteConvos || remoteConvos.length === 0) return { pulled: 0 };
+  const client = getClient();
+  const stmts = remoteConvos.map((c) => ({
+    sql: `INSERT OR IGNORE INTO conversations
+          (id, platform, external_id, sender_id, sender_name, sender_phone, sender_email,
+           recipient_id, channel_id, thread_id, reply_to_id, content_text, content_media,
+           content_metadata, agent_response, agent_name, timestamp, ingested_at)
+          VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+    args: [
+      c.id,
+      c.platform,
+      c.external_id ?? null,
+      c.sender_id,
+      c.sender_name ?? null,
+      c.sender_phone ?? null,
+      c.sender_email ?? null,
+      c.recipient_id ?? null,
+      c.channel_id,
+      c.thread_id ?? null,
+      c.reply_to_id ?? null,
+      c.content_text ?? null,
+      c.content_media ?? null,
+      c.content_metadata ?? null,
+      c.agent_response ?? null,
+      c.agent_name ?? null,
+      c.timestamp,
+      c.ingested_at
+    ]
+  }));
+  await client.batch(stmts, "write");
+  return { pulled: remoteConvos.length };
+}
+async function cloudPushDocuments(config) {
+  const client = getClient();
+  const [workspaces, documents] = await Promise.all([
+    client.execute("SELECT * FROM workspaces"),
+    client.execute("SELECT * FROM documents")
+  ]);
+  const blob = {
+    workspaces: workspaces.rows,
+    documents: documents.rows
+  };
+  const { ok } = await cloudPushBlob(
+    "/sync/push-documents",
+    [blob],
+    "last_documents_push_version",
+    config
+  );
+  return ok;
+}
+async function cloudPullDocuments(config) {
+  const data = await cloudPullBlob(
+    "/sync/pull-documents",
+    config
+  );
+  if (!data || data.length === 0) return { pulled: 0 };
+  const blob = data[0];
+  const client = getClient();
+  let pulled = 0;
+  if (blob.workspaces.length > 0) {
+    const stmts = blob.workspaces.map((w) => ({
+      sql: `INSERT OR IGNORE INTO workspaces (id, slug, name, owner_agent_id, created_at, metadata)
+            VALUES (?, ?, ?, ?, ?, ?)`,
+      args: [w.id, w.slug, w.name, w.owner_agent_id ?? null, w.created_at, w.metadata ?? null]
+    }));
+    await client.batch(stmts, "write");
+    pulled += stmts.length;
+  }
+  if (blob.documents.length > 0) {
+    const stmts = blob.documents.map((d) => ({
+      sql: `INSERT OR IGNORE INTO documents
+            (id, workspace_id, filename, mime, source_type, user_id, uploaded_at, metadata)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+      args: [
+        d.id,
+        d.workspace_id,
+        d.filename,
+        d.mime ?? null,
+        d.source_type ?? null,
+        d.user_id ?? null,
+        d.uploaded_at,
+        d.metadata ?? null
+      ]
+    }));
+    await client.batch(stmts, "write");
+    pulled += stmts.length;
+  }
+  return { pulled };
+}
 export {
   buildRosterBlob,
   cloudPull,
+  cloudPullBehaviors,
+  cloudPullBlob,
+  cloudPullConversations,
+  cloudPullDocuments,
+  cloudPullGraphRAG,
   cloudPullRoster,
+  cloudPullTasks,
   cloudPush,
+  cloudPushBehaviors,
+  cloudPushBlob,
+  cloudPushConversations,
+  cloudPushDocuments,
+  cloudPushGraphRAG,
   cloudPushRoster,
+  cloudPushTasks,
   cloudSync,
+  mergeConfig,
   mergeRosterFromRemote
 };