aios-management-web 0.1.0

package/server/src/services/catalog-sync-service.js

@@ -0,0 +1,712 @@
+import { jsonParse, withSerializedTransaction } from "../db/index.js";
+import {
+  normalizeAgentStatus,
+  normalizeUsage,
+  statusAfterUsageRefresh
+} from "./agent-quota.js";
+
+const USAGE_SYNC_CONCURRENCY = 2;
+const USAGE_FIELDS = [
+  "daily",
+  "day",
+  "dailyTokens",
+  "today",
+  "totalTokens"
+];
+
+function hasUsageFields(payload) {
+  if (!payload || typeof payload !== "object") {
+    return false;
+  }
+
+  return USAGE_FIELDS.some((field) => Object.prototype.hasOwnProperty.call(payload, field));
+}
+
+function parseUsagePayload(payload) {
+  if (!payload || typeof payload !== "object") {
+    throw new Error("agent.usage returned an empty payload");
+  }
+
+  const source = payload.usage || payload.tokens || payload;
+  if (!hasUsageFields(source)) {
+    throw new Error("agent.usage did not return usage fields");
+  }
+
+  return normalizeUsage(payload);
+}
+
+async function mapWithConcurrency(items, limit, mapper) {
+  const results = new Array(items.length);
+  let nextIndex = 0;
+
+  async function worker() {
+    while (nextIndex < items.length) {
+      const currentIndex = nextIndex;
+      nextIndex += 1;
+      results[currentIndex] = await mapper(items[currentIndex], currentIndex);
+    }
+  }
+
+  const workerCount = Math.min(Math.max(1, limit), Math.max(1, items.length));
+  await Promise.all(Array.from({ length: workerCount }, () => worker()));
+  return results;
+}
+
+function firstText(...values) {
+  for (const value of values) {
+    if (typeof value === "string" && value.trim()) {
+      return value.trim();
+    }
+  }
+
+  return "";
+}
+
+function isBuiltInItem(item) {
+  return Boolean(item?.["is-built-in"] ?? item?.is_builtin ?? item?.isBuiltIn);
+}
+
+export class CatalogSyncService {
+  constructor({ db, rpcClient, auditLogService = null, env = null }) {
+    this.db = db;
+    this.rpcClient = rpcClient;
+    this.auditLogService = auditLogService;
+    this.managementTimeoutMs = Number(env?.managementTimeoutMs || 120000);
+    this.pendingUsageRefresh = null;
+    this.pendingAgentSync = null;
+    this.pendingSkillSync = null;
+    this.pendingTemplateSync = null;
+    this.pendingSystemSync = null;
+  }
+
+  getUsageRefreshStatus() {
+    return this.readState("usage_refresh_state");
+  }
+
+  getAgentSyncStatus() {
+    return this.readState("agent_sync_state");
+  }
+
+  getSkillSyncStatus() {
+    return this.readState("skill_sync_state");
+  }
+
+  getTemplateSyncStatus() {
+    return this.readState("template_sync_state");
+  }
+
+  getSystemSyncStatus() {
+    return this.readState("system_sync_state");
+  }
+
+  callManagement(action, params = {}, timeoutMs = this.managementTimeoutMs) {
+    return this.rpcClient.call(action, params, timeoutMs);
+  }
+
+  async syncAgentsFromKernel(now = new Date().toISOString()) {
+    const remote = await this.callManagement("agent.list");
+    const items = Array.isArray(remote?.items) ? remote.items : [];
+    await withSerializedTransaction(this.db, () => {
+      this.syncAgents(items, new Map(), now);
+    });
+    return items.length;
+  }
+
+  async syncTemplatesFromKernel(now = new Date().toISOString()) {
+    const remote = await this.callManagement("agent.template.list");
+    const items = Array.isArray(remote?.items) ? remote.items : [];
+    await withSerializedTransaction(this.db, () => {
+      this.syncTemplates(items, now);
+    });
+    return items.length;
+  }
+
+  async syncSkillsFromKernel(now = new Date().toISOString()) {
+    const remote = await this.callManagement("skills.global.list");
+    const items = Array.isArray(remote?.items) ? remote.items : [];
+    await withSerializedTransaction(this.db, () => {
+      this.syncSkills(items, now);
+    });
+    return items.length;
+  }
+
+  async syncSystemsFromKernel(now = new Date().toISOString()) {
+    const [remote, ontologies] = await Promise.all([
+      this.callManagement("apps.list"),
+      this.callManagement("ontology.list")
+    ]);
+    const items = Array.isArray(remote?.items) ? remote.items : [];
+    const ontologyItems = Array.isArray(ontologies?.items) ? ontologies.items : [];
+    const builtInOntologies = new Set(ontologyItems
+      .filter(isBuiltInItem)
+      .map((item) => firstText(item?.name, item?.ontologyName, item?.id).toLowerCase())
+      .filter(Boolean));
+    await withSerializedTransaction(this.db, () => {
+      this.syncSystems(items, now, builtInOntologies);
+    });
+    return items.length;
+  }
+
+  async syncAgentsTask({ trigger = "manual" } = {}) {
+    if (this.pendingAgentSync) {
+      return await this.pendingAgentSync;
+    }
+
+    this.pendingAgentSync = this.runSingleSync({
+      tableName: "agent_sync_state",
+      trigger,
+      run: () => this.syncAgentsFromKernel(),
+      summaryKey: "agents"
+    }).finally(() => {
+      this.pendingAgentSync = null;
+    });
+
+    return await this.pendingAgentSync;
+  }
+
+  async syncSkillsTask({ trigger = "manual" } = {}) {
+    if (this.pendingSkillSync) {
+      return await this.pendingSkillSync;
+    }
+
+    this.pendingSkillSync = this.runSingleSync({
+      tableName: "skill_sync_state",
+      trigger,
+      run: () => this.syncSkillsFromKernel(),
+      summaryKey: "skills"
+    }).finally(() => {
+      this.pendingSkillSync = null;
+    });
+
+    return await this.pendingSkillSync;
+  }
+
+  async syncTemplatesTask({ trigger = "manual" } = {}) {
+    if (this.pendingTemplateSync) {
+      return await this.pendingTemplateSync;
+    }
+
+    this.pendingTemplateSync = this.runSingleSync({
+      tableName: "template_sync_state",
+      trigger,
+      run: () => this.syncTemplatesFromKernel(),
+      summaryKey: "templates"
+    }).finally(() => {
+      this.pendingTemplateSync = null;
+    });
+
+    return await this.pendingTemplateSync;
+  }
+
+  async syncSystemsTask({ trigger = "manual" } = {}) {
+    if (this.pendingSystemSync) {
+      return await this.pendingSystemSync;
+    }
+
+    this.pendingSystemSync = this.runSingleSync({
+      tableName: "system_sync_state",
+      trigger,
+      run: () => this.syncSystemsFromKernel(),
+      summaryKey: "systems"
+    }).finally(() => {
+      this.pendingSystemSync = null;
+    });
+
+    return await this.pendingSystemSync;
+  }
+
+  async fetchAgentUsageSnapshots(agentItems, { strict = false } = {}) {
+    const pairs = await mapWithConcurrency(agentItems, USAGE_SYNC_CONCURRENCY, async (item) => {
+      const agentId = String(item?.agentId || "").trim();
+      if (!agentId) {
+        return [agentId, null];
+      }
+
+      try {
+        const raw = await this.callManagement("agent.usage", { agentId });
+        return [agentId, {
+          raw,
+          usage: parseUsagePayload(raw),
+          captured_at: new Date().toISOString()
+        }];
+      } catch (error) {
+        if (strict) {
+          throw error;
+        }
+        return [agentId, null];
+      }
+    });
+
+    return new Map(pairs.filter(([agentId]) => agentId));
+  }
+
+  async runSingleSync({ tableName, trigger, run, summaryKey }) {
+    const startedAt = new Date().toISOString();
+
+    if (!this.rpcClient.isConfigured()) {
+      this.updateState(tableName, {
+        status: "skipped",
+        triggerSource: trigger,
+        startedAt,
+        finishedAt: startedAt,
+        lastSuccessAt: null,
+        errorMessage: "Management RPC is not configured",
+        summary: {}
+      });
+      return this.readState(tableName);
+    }
+
+    this.updateState(tableName, {
+      status: "running",
+      triggerSource: trigger,
+      startedAt,
+      finishedAt: null,
+      errorMessage: null,
+      summary: {}
+    });
+
+    try {
+      const count = await run();
+      const finishedAt = new Date().toISOString();
+      this.updateState(tableName, {
+        status: "success",
+        triggerSource: trigger,
+        startedAt,
+        finishedAt,
+        lastSuccessAt: finishedAt,
+        errorMessage: null,
+        summary: {
+          [summaryKey]: count
+        }
+      });
+      return this.readState(tableName);
+    } catch (error) {
+      const finishedAt = new Date().toISOString();
+      this.updateState(tableName, {
+        status: "failed",
+        triggerSource: trigger,
+        startedAt,
+        finishedAt,
+        errorMessage: error instanceof Error ? error.message : String(error),
+        summary: {}
+      });
+      return this.readState(tableName);
+    }
+  }
+
+  async refreshAgentUsage({ trigger = "scheduled" } = {}) {
+    if (this.pendingUsageRefresh) {
+      return await this.pendingUsageRefresh;
+    }
+
+    this.pendingUsageRefresh = this.runUsageRefresh(trigger).finally(() => {
+      this.pendingUsageRefresh = null;
+    });
+
+    return await this.pendingUsageRefresh;
+  }
+
+  async runUsageRefresh(trigger) {
+    const startedAt = new Date().toISOString();
+
+    if (!this.rpcClient.isConfigured()) {
+      this.updateState("usage_refresh_state", {
+        status: "skipped",
+        triggerSource: trigger,
+        startedAt,
+        finishedAt: startedAt,
+        lastSuccessAt: null,
+        errorMessage: "Management RPC is not configured",
+        summary: {
+          refreshed_agents: 0
+        }
+      });
+      return this.getUsageRefreshStatus();
+    }
+
+    this.updateState("usage_refresh_state", {
+      status: "running",
+      triggerSource: trigger,
+      startedAt,
+      finishedAt: null,
+      errorMessage: null,
+      summary: {}
+    });
+
+    const agentRows = this.db.prepare(`
+      SELECT *
+      FROM agents
+      ORDER BY slug
+    `).all();
+
+    if (agentRows.length === 0) {
+      this.updateState("usage_refresh_state", {
+        status: "skipped",
+        triggerSource: trigger,
+        startedAt,
+        finishedAt: startedAt,
+        lastSuccessAt: null,
+        errorMessage: null,
+        summary: {}
+      });
+      return this.getUsageRefreshStatus();
+    }
+
+    const remoteItems = agentRows.map((row) => ({
+      agentId: row.slug
+    }));
+    const now = new Date();
+    const capturedAt = now.toISOString();
+    const updateAgent = this.db.prepare(`
+      UPDATE agents
+      SET usage_snapshot_json = ?, status = ?, updated_at = ?
+      WHERE id = ?
+    `);
+
+    try {
+      const usageMap = await this.fetchAgentUsageSnapshots(remoteItems, { strict: true });
+      let refreshedAgents = 0;
+
+      await withSerializedTransaction(this.db, () => {
+        for (const row of agentRows) {
+          const snapshot = usageMap.get(row.slug);
+          if (!snapshot) {
+            continue;
+          }
+
+          const nextStatus = statusAfterUsageRefresh(row, snapshot.usage);
+          updateAgent.run(JSON.stringify(snapshot), nextStatus, capturedAt, row.id);
+          refreshedAgents += 1;
+        }
+
+        const finishedAt = new Date().toISOString();
+        this.updateState("usage_refresh_state", {
+          status: "success",
+          triggerSource: trigger,
+          startedAt,
+          finishedAt,
+          lastSuccessAt: finishedAt,
+          errorMessage: null,
+          summary: {
+            refreshed_agents: refreshedAgents,
+            captured_at: capturedAt
+          }
+        });
+      });
+
+      return this.getUsageRefreshStatus();
+    } catch (error) {
+      const finishedAt = new Date().toISOString();
+      this.updateState("usage_refresh_state", {
+        status: "failed",
+        triggerSource: trigger,
+        startedAt,
+        finishedAt,
+        errorMessage: error instanceof Error ? error.message : String(error),
+        summary: {}
+      });
+      return this.getUsageRefreshStatus();
+    }
+  }
+
+  syncAgents(items, usageMap, now) {
+    const existingRows = this.db.prepare("SELECT * FROM agents").all();
+    const existingMap = new Map(existingRows.map((row) => [row.slug, row]));
+    const seen = new Set();
+    const insert = this.db.prepare(`
+      INSERT INTO agents (
+        slug, agent_name, description, docs_content, template_name, status, tags_json,
+        daily_limit, usage_snapshot_json, remote_state_json, created_at, updated_at
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+    const update = this.db.prepare(`
+      UPDATE agents
+      SET agent_name = ?, template_name = ?, status = ?, usage_snapshot_json = ?, remote_state_json = ?, updated_at = ?
+      WHERE id = ?
+    `);
+    const remove = this.db.prepare("DELETE FROM agents WHERE id = ?");
+
+    for (const item of items) {
+      const slug = String(item?.agentId || "").trim();
+      if (!slug) {
+        continue;
+      }
+
+      seen.add(slug);
+      const existing = existingMap.get(slug);
+      const existingUsageSnapshot = jsonParse(existing?.usage_snapshot_json, {
+        usage: { daily: 0 }
+      });
+      const usageSnapshot = usageMap.get(slug) || {
+        ...existingUsageSnapshot,
+        usage: normalizeUsage(existingUsageSnapshot.usage)
+      };
+      const remoteStateJson = JSON.stringify(item ?? {});
+      const agentName = firstText(item?.name, slug);
+      const status = normalizeAgentStatus(item?.status);
+      const templateName = firstText(item?.templateName, item?.template, existing?.template_name, "default");
+
+      if (existing) {
+        const nextStatus = existing.status === "disabled"
+          ? "disabled"
+          : (existing.status === "overlimit" ? "overlimit" : normalizeAgentStatus(item?.status));
+        update.run(
+          agentName,
+          templateName,
+          nextStatus,
+          JSON.stringify(usageSnapshot),
+          remoteStateJson,
+          now,
+          existing.id
+        );
+        continue;
+      }
+
+      insert.run(
+        slug,
+        agentName,
+        "",
+        "",
+        templateName,
+        status,
+        "[]",
+        -1,
+        JSON.stringify(usageSnapshot),
+        remoteStateJson,
+        now,
+        now
+      );
+    }
+
+    for (const row of existingRows) {
+      if (!seen.has(row.slug)) {
+        remove.run(row.id);
+      }
+    }
+  }
+
+  syncTemplates(items, now) {
+    const existingRows = this.db.prepare("SELECT * FROM agent_templates").all();
+    const existingMap = new Map(existingRows.map((row) => [row.template_name, row]));
+    const seen = new Set();
+    const insert = this.db.prepare(`
+      INSERT INTO agent_templates (
+        template_name, description, artifact_id, remote_status, remote_result_json, is_builtin, created_at, updated_at
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+    const update = this.db.prepare(`
+      UPDATE agent_templates
+      SET description = ?, artifact_id = ?, remote_status = ?, remote_result_json = ?, is_builtin = ?, updated_at = ?
+      WHERE id = ?
+    `);
+    const remove = this.db.prepare("DELETE FROM agent_templates WHERE id = ?");
+
+    for (const item of items) {
+      const templateName = String(item?.templateName || "").trim();
+      if (!templateName) {
+        continue;
+      }
+
+      seen.add(templateName);
+      const existing = existingMap.get(templateName);
+      const artifactId = existing?.artifact_id ?? this.createPlaceholderArtifact({
+        kind: "template",
+        originalName: firstText(item?.path, `${templateName}.remote`)
+      });
+      const remoteResultJson = JSON.stringify(item ?? {});
+      const isBuiltin = isBuiltInItem(item) ? 1 : 0;
+
+      if (existing) {
+        update.run(
+          existing.description || "",
+          artifactId,
+          "ready",
+          remoteResultJson,
+          isBuiltin,
+          now,
+          existing.id
+        );
+        continue;
+      }
+
+      insert.run(
+        templateName,
+        "",
+        artifactId,
+        "ready",
+        remoteResultJson,
+        isBuiltin,
+        now,
+        now
+      );
+    }
+
+    for (const row of existingRows) {
+      if (!seen.has(row.template_name)) {
+        remove.run(row.id);
+      }
+    }
+  }
+
+  syncSkills(items, now) {
+    const existingRows = this.db.prepare("SELECT * FROM skills").all();
+    const existingMap = new Map(existingRows.map((row) => [row.slug, row]));
+    const seen = new Set();
+    const insert = this.db.prepare(`
+      INSERT INTO skills (
+        slug, description, artifact_id, remote_status, is_builtin, created_at, updated_at
+      ) VALUES (?, ?, ?, ?, ?, ?, ?)
+    `);
+    const update = this.db.prepare(`
+      UPDATE skills
+      SET description = ?, artifact_id = ?, remote_status = ?, is_builtin = ?, updated_at = ?
+      WHERE id = ?
+    `);
+    const remove = this.db.prepare("DELETE FROM skills WHERE id = ?");
+
+    for (const item of items) {
+      const slug = String(item?.slug || "").trim();
+      if (!slug) {
+        continue;
+      }
+
+      seen.add(slug);
+      const existing = existingMap.get(slug);
+      const origin = item?.origin && typeof item.origin === "object" ? item.origin : {};
+      const description = firstText(origin.description, existing?.description);
+      const isBuiltin = isBuiltInItem(item) ? 1 : 0;
+
+      if (existing) {
+        update.run(
+          description,
+          existing.artifact_id ?? null,
+          "installed",
+          isBuiltin,
+          now,
+          existing.id
+        );
+        continue;
+      }
+
+      insert.run(
+        slug,
+        description,
+        null,
+        "installed",
+        isBuiltin,
+        now,
+        now
+      );
+    }
+
+    for (const row of existingRows) {
+      if (!seen.has(row.slug)) {
+        remove.run(row.id);
+      }
+    }
+  }
+
+  syncSystems(items, now = new Date().toISOString(), builtInOntologies = new Set()) {
+    const existingRows = this.db.prepare("SELECT * FROM business_systems").all();
+    const seen = new Set();
+    const updateBuiltin = this.db.prepare(`
+      UPDATE business_systems
+      SET is_builtin = ?, updated_at = ?
+      WHERE id = ?
+    `);
+    const remove = this.db.prepare("DELETE FROM business_systems WHERE id = ?");
+
+    for (const item of items) {
+      const applicationName = firstText(item?.applicationName, item?.id, item?.name).toLowerCase();
+      if (!applicationName) {
+        continue;
+      }
+
+      seen.add(applicationName);
+    }
+
+    for (const row of existingRows) {
+      if (!seen.has(row.application_name)) {
+        remove.run(row.id);
+        continue;
+      }
+
+      const isBuiltin = builtInOntologies.has(String(row.application_name || "").toLowerCase()) ? 1 : 0;
+      if (Number(row.is_builtin || 0) !== isBuiltin) {
+        updateBuiltin.run(isBuiltin, now, row.id);
+      }
+    }
+  }
+
+  createPlaceholderArtifact({ kind, originalName }) {
+    const result = this.db.prepare(`
+      INSERT INTO artifacts (
+        kind, bucket, object_key, original_name, mime_type, byte_size, created_by, created_at
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(
+      kind,
+      "",
+      "",
+      originalName,
+      "application/octet-stream",
+      0,
+      null,
+      new Date().toISOString()
+    );
+
+    return result.lastInsertRowid;
+  }
+
+  readState(tableName) {
+    const row = this.db.prepare(`SELECT * FROM ${tableName} WHERE id = 1`).get();
+    if (!row) {
+      return {
+        id: 1,
+        status: "idle",
+        trigger_source: null,
+        started_at: null,
+        finished_at: null,
+        last_success_at: null,
+        error_message: null,
+        summary: {}
+      };
+    }
+
+    return {
+      ...row,
+      summary: jsonParse(row.summary_json, {})
+    };
+  }
+
+  updateState(tableName, { status, triggerSource, startedAt, finishedAt, lastSuccessAt, errorMessage, summary }) {
+    const current = this.db.prepare(`SELECT * FROM ${tableName} WHERE id = 1`).get();
+    const createdAt = current?.created_at || new Date().toISOString();
+    const nextLastSuccessAt = lastSuccessAt ?? current?.last_success_at ?? null;
+
+    this.db.prepare(`
+      INSERT INTO ${tableName} (
+        id, status, trigger_source, started_at, finished_at, last_success_at,
+        error_message, summary_json, created_at, updated_at
+      ) VALUES (1, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ON CONFLICT(id) DO UPDATE SET
+        status = excluded.status,
+        trigger_source = excluded.trigger_source,
+        started_at = excluded.started_at,
+        finished_at = excluded.finished_at,
+        last_success_at = excluded.last_success_at,
+        error_message = excluded.error_message,
+        summary_json = excluded.summary_json,
+        updated_at = excluded.updated_at
+    `).run(
+      status,
+      triggerSource || null,
+      startedAt || null,
+      finishedAt || null,
+      nextLastSuccessAt,
+      errorMessage || null,
+      JSON.stringify(summary || {}),
+      createdAt,
+      new Date().toISOString()
+    );
+  }
+}