abs-zalo-bot 0.2.0

package/src/backfill.js

@@ -0,0 +1,364 @@
+// Historical + identity corpus backfill. READ_ONLY — never sends.
+import { normalizeInboundMessage, utcNow } from "./schema.js";
+import { listGroupsDetailed } from "./discovery.js";
+
+function sleep(ms) {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+function parseMemberToken(token) {
+  // memVerList entries look like "696878...038_0"
+  const s = String(token || "");
+  const idx = s.lastIndexOf("_");
+  if (idx > 0) return s.slice(0, idx);
+  return s;
+}
+
+function asMessageLike(groupMsg, groupId) {
+  if (groupMsg?.data && groupMsg?.threadId != null) {
+    return {
+      type: groupMsg.type ?? 1,
+      threadId: String(groupMsg.threadId || groupId),
+      isSelf: Boolean(groupMsg.isSelf),
+      data: groupMsg.data,
+    };
+  }
+  const data = groupMsg?.data || groupMsg || {};
+  return {
+    type: 1,
+    threadId: String(groupId),
+    isSelf: false,
+    data,
+  };
+}
+
+async function fetchOldGroupMessages(api, { timeoutMs = 15000, maxPages = 5 } = {}) {
+  if (!api?.listener?.requestOldMessages) return [];
+  const ThreadTypeGroup = 1;
+  try {
+    api.listener.start?.({ retryOnClose: true });
+  } catch {
+    /* already started */
+  }
+  await sleep(1000);
+
+  const collected = [];
+  const seen = new Set();
+  let lastId = null;
+
+  for (let page = 0; page < maxPages; page++) {
+    const batch = await new Promise((resolve) => {
+      let done = false;
+      const rows = [];
+      const finish = () => {
+        if (done) return;
+        done = true;
+        try {
+          api.listener.off("old_messages", onOld);
+        } catch {
+          /* ignore */
+        }
+        clearTimeout(timer);
+        resolve(rows);
+      };
+      const onOld = (messages, type) => {
+        if (!(type === ThreadTypeGroup || type === 1 || type === "group" || type == null)) return;
+        for (const m of Array.isArray(messages) ? messages : []) rows.push(m);
+        // settle after first batch for this page
+        setTimeout(finish, 1200);
+      };
+      const timer = setTimeout(finish, Math.max(3000, Math.floor(timeoutMs / maxPages)));
+      api.listener.on("old_messages", onOld);
+      try {
+        api.listener.requestOldMessages(ThreadTypeGroup, lastId);
+      } catch {
+        finish();
+      }
+    });
+
+    if (!batch.length) break;
+    let newCount = 0;
+    for (const m of batch) {
+      const mid = String(m?.data?.msgId || m?.data?.cliMsgId || m?.messageId || "");
+      const key = `${m?.threadId || ""}:${mid}`;
+      if (seen.has(key)) continue;
+      seen.add(key);
+      collected.push(m);
+      newCount += 1;
+      if (mid) lastId = mid;
+    }
+    if (!newCount) break;
+    await sleep(400);
+  }
+  return collected;
+}
+
+/**
+ * Full corpus crawl for one account:
+ * - groups (id/name/members)
+ * - users (id/display)
+ * - recent history messages per group (via websocket old_messages when REST history 404)
+ */
+export async function backfillAccountCorpus({
+  api,
+  store,
+  accountId,
+  historyCount = 50,
+  maxGroups = 200,
+  delayMs = 250,
+  groupIds = null,
+}) {
+  const result = {
+    account_id: accountId,
+    started_at: utcNow(),
+    groups_seen: 0,
+    members_seen: 0,
+    users_upserted: 0,
+    messages_ingested: 0,
+    messages_skipped: 0,
+    history_method: "none",
+    errors: [],
+    finished_at: null,
+  };
+
+  if (!api) {
+    result.errors.push("not_connected");
+    result.finished_at = utcNow();
+    return result;
+  }
+
+  let groups = [];
+  try {
+    groups = await listGroupsDetailed(api);
+  } catch (err) {
+    result.errors.push(`list_groups:${String(err?.message || err)}`);
+    groups = store.getSetting(`discovered_groups_${accountId}`, []) || [];
+  }
+  if (Array.isArray(groupIds) && groupIds.length) {
+    const allow = new Set(groupIds.map(String));
+    groups = groups.filter((g) => allow.has(String(g.source_id)));
+  }
+  groups = groups.slice(0, maxGroups);
+  result.groups_seen = groups.length;
+  store.setSetting(`discovered_groups_${accountId}`, groups);
+
+  // One websocket old_messages dump can contain many groups — pull once then partition.
+  let sharedOld = [];
+  try {
+    sharedOld = await fetchOldGroupMessages(api, { timeoutMs: 20000, maxPages: 8 });
+    if (sharedOld.length) result.history_method = "ws_old_messages";
+    result.ws_old_count = sharedOld.length;
+  } catch (err) {
+    result.errors.push(`ws_old:${String(err?.message || err)}`);
+  }
+
+  const oldByGroup = new Map();
+  for (const m of sharedOld) {
+    const tid = String(m.threadId || m.data?.idTo || "");
+    if (!tid) continue;
+    if (!oldByGroup.has(tid)) oldByGroup.set(tid, []);
+    oldByGroup.get(tid).push(m);
+  }
+
+  for (const g of groups) {
+    const gid = String(g.source_id || "");
+    if (!gid) continue;
+    let gname = String(g.source_name || "");
+
+    store.upsertSource({
+      accountId,
+      sourceId: gid,
+      sourceType: "group",
+      sourceName: gname,
+      mode: "listen_only",
+      isAllowed: true,
+    });
+
+    let memberIds = [];
+    let adminIds = [];
+    let creatorId = "";
+    try {
+      if (typeof api.getGroupInfo === "function") {
+        const info = await api.getGroupInfo(gid);
+        const gi = info?.gridInfoMap?.[gid] || {};
+        if (gi.name) gname = String(gi.name);
+        creatorId = String(gi.creatorId || "");
+        adminIds = Array.isArray(gi.adminIds) ? gi.adminIds.map(String) : [];
+        if (Array.isArray(gi.memberIds) && gi.memberIds.length) {
+          memberIds = gi.memberIds.map(String);
+        } else if (Array.isArray(gi.memVerList) && gi.memVerList.length) {
+          memberIds = gi.memVerList.map(parseMemberToken).filter(Boolean);
+        }
+        if (Array.isArray(gi.currentMems)) {
+          for (const m of gi.currentMems) {
+            const uid = String(m.id || m.uid || m.userId || "");
+            if (!uid) continue;
+            store.upsertUser({
+              accountId,
+              userId: uid,
+              displayName: m.dName || m.displayName || m.zaloName || "",
+              avatar: m.avatar || "",
+              globalId: m.globalId || "",
+              raw: m,
+            });
+            result.users_upserted += 1;
+          }
+        }
+        store.upsertSource({
+          accountId,
+          sourceId: gid,
+          sourceType: "group",
+          sourceName: gname,
+          mode: "listen_only",
+          isAllowed: true,
+        });
+      }
+    } catch (err) {
+      result.errors.push(`group_info:${gid}:${String(err?.message || err)}`);
+    }
+
+    if (creatorId) memberIds.push(creatorId);
+    memberIds = [...new Set(memberIds.filter(Boolean))];
+
+    const roleOf = (uid) => {
+      if (creatorId && uid === creatorId) return "creator";
+      if (adminIds.includes(uid)) return "admin";
+      return "member";
+    };
+    for (const uid of memberIds) {
+      store.upsertSourceMember({
+        accountId,
+        sourceId: gid,
+        userId: uid,
+        role: roleOf(uid),
+      });
+      result.members_seen += 1;
+    }
+
+    if (memberIds.length && typeof api.getGroupMembersInfo === "function") {
+      for (let i = 0; i < memberIds.length; i += 20) {
+        const chunk = memberIds.slice(i, i + 20);
+        try {
+          const res = await api.getGroupMembersInfo(chunk);
+          const profiles = res?.profiles || {};
+          for (const [uid, p] of Object.entries(profiles)) {
+            store.upsertUser({
+              accountId,
+              userId: uid,
+              displayName: p.displayName || p.zaloName || "",
+              avatar: p.avatar || "",
+              globalId: p.globalId || "",
+              accountStatus: p.accountStatus,
+              raw: p,
+            });
+            result.users_upserted += 1;
+          }
+        } catch (err) {
+          result.errors.push(`members_info:${gid}:${String(err?.message || err)}`);
+        }
+        await sleep(delayMs);
+      }
+    }
+
+    // History: prefer the current cloud-history adapter, then legacy REST,
+    // then the shared websocket dump. Zalo removed the legacy endpoint in
+    // August 2026, so the v2 adapter must win when installed.
+    let msgs = [];
+    if (typeof api.getGroupChatHistoryV2 === "function") {
+      try {
+        const hist = await api.getGroupChatHistoryV2({ groupId: gid, count: historyCount });
+        msgs = Array.isArray(hist?.groupMsgs) ? hist.groupMsgs : [];
+        if (msgs.length) result.history_method = "cloud_history_v2";
+      } catch (err) {
+        result.errors.push(`history_v2:${gid}:${String(err?.message || err).slice(0, 180)}`);
+      }
+    }
+    if (!msgs.length && typeof api.getGroupChatHistory === "function") {
+      try {
+        const hist = await api.getGroupChatHistory(gid, historyCount);
+        msgs = Array.isArray(hist?.groupMsgs) ? hist.groupMsgs : [];
+        if (msgs.length) result.history_method = "rest_history";
+      } catch (err) {
+        result.errors.push(`history_rest:${gid}:404/err`);
+      }
+    }
+    if (!msgs.length) {
+      msgs = (oldByGroup.get(gid) || []).slice(0, historyCount);
+    }
+
+    for (const m of msgs) {
+      try {
+        const message = asMessageLike(m, gid);
+        if (!message.threadId) message.threadId = gid;
+        const event = normalizeInboundMessage({
+          accountId,
+          message,
+          sourceName: gname,
+        });
+        event.raw_metadata = {
+          ...(event.raw_metadata || {}),
+          sender_user_id: event.sender_id,
+          backfill: true,
+        };
+        if (event.sender_id) {
+          store.upsertUser({
+            accountId,
+            userId: event.sender_id,
+            displayName: event.sender_name || "",
+          });
+        }
+        const inserted = store.putEvent(event);
+        if (inserted) result.messages_ingested += 1;
+        else result.messages_skipped += 1;
+      } catch (err) {
+        result.messages_skipped += 1;
+        result.errors.push(`msg:${gid}:${String(err?.message || err)}`);
+      }
+    }
+
+    await sleep(delayMs);
+    store.setHealth(`backfill_progress_${accountId}`, {
+      at: utcNow(),
+      last_group_id: gid,
+      groups_done: result.groups_seen,
+      messages_ingested: result.messages_ingested,
+      users_upserted: result.users_upserted,
+      members_seen: result.members_seen,
+    });
+  }
+
+  try {
+    if (typeof api.getAllFriends === "function") {
+      const friends = await api.getAllFriends(200, 1);
+      const list = Array.isArray(friends) ? friends : friends?.data || friends?.friends || [];
+      for (const f of list) {
+        const uid = String(f.userId || f.uid || f.id || "");
+        if (!uid) continue;
+        store.upsertUser({
+          accountId,
+          userId: uid,
+          displayName: f.displayName || f.zaloName || f.dName || "",
+          avatar: f.avatar || "",
+          globalId: f.globalId || "",
+          raw: { friend: true },
+        });
+        result.users_upserted += 1;
+      }
+    }
+  } catch (err) {
+    result.errors.push(`friends:${String(err?.message || err)}`);
+  }
+
+  result.finished_at = utcNow();
+  store.setHealth(`backfill_${accountId}`, {
+    ...result,
+    errors: result.errors.slice(-50),
+  });
+  store.audit({
+    accountId,
+    actorId: "system",
+    action: "corpus_backfill",
+    detail: `groups=${result.groups_seen};msgs=${result.messages_ingested};users=${result.users_upserted};members=${result.members_seen}`,
+  });
+  return result;
+}

package/src/bot_registry.js

@@ -0,0 +1,180 @@
+import fs from "node:fs";
+import path from "node:path";
+
+import { OaAdapter } from "./oa_adapter.js";
+
+const ID_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
+const ENV_RE = /^[A-Z][A-Z0-9_]{0,127}$/;
+const ADAPTERS = new Set(["zalo_personal_qr", "zalo_oa"]);
+// `inbound_auto_reply` may be reached only by a signed, normalized OA webhook.
+// It is not accepted by the generic outbound-send policy.
+const POLICY_MODES = new Set(["disabled", "draft_first", "approved_send", "inbound_auto_reply", "paused"]);
+
+export class BotRegistryError extends Error {
+  constructor(code, message = code) {
+    super(message);
+    Object.defineProperties(this, {
+      name: { value: "BotRegistryError", enumerable: false, configurable: true },
+      code: { value: code, enumerable: false },
+    });
+  }
+}
+
+function text(value) {
+  return String(value ?? "").trim();
+}
+
+function requireId(value, code) {
+  const id = text(value);
+  if (!ID_RE.test(id)) throw new BotRegistryError(code, `${code}: invalid identifier`);
+  return id;
+}
+
+function requireEnvRef(value) {
+  const name = text(value);
+  if (!ENV_RE.test(name)) throw new BotRegistryError("invalid_credential_ref", "credential must reference an env variable");
+  return name;
+}
+
+function normalizePolicy(policy = {}) {
+  if (!policy || typeof policy !== "object" || Array.isArray(policy)) {
+    throw new BotRegistryError("invalid_policy", "policy must be an object");
+  }
+  const mode = text(policy.mode || "draft_first").toLowerCase();
+  if (!POLICY_MODES.has(mode)) throw new BotRegistryError("invalid_policy_mode", `unsupported policy mode: ${mode}`);
+  const allowUserIds = Array.isArray(policy.allow_user_ids)
+    ? policy.allow_user_ids.map((id) => text(id)).filter(Boolean)
+    : [];
+  if (allowUserIds.some((id) => id.length > 256)) {
+    throw new BotRegistryError("invalid_policy", "allow_user_ids contains an oversized id");
+  }
+  return {
+    mode,
+    allow_user_ids: [...new Set(allowUserIds)],
+    max_messages_per_hour: Number.isFinite(Number(policy.max_messages_per_hour))
+      ? Math.max(0, Math.floor(Number(policy.max_messages_per_hour)))
+      : 20,
+  };
+}
+
+function normalizeBot(bot) {
+  if (!bot || typeof bot !== "object" || Array.isArray(bot)) {
+    throw new BotRegistryError("invalid_bot", "each bot must be an object");
+  }
+  const botId = requireId(bot.bot_id, "invalid_bot_id");
+  const tenantId = requireId(bot.tenant_id || "default", "invalid_tenant_id");
+  const adapter = text(bot.adapter).toLowerCase();
+  if (!ADAPTERS.has(adapter)) throw new BotRegistryError("invalid_adapter", `unsupported adapter: ${adapter}`);
+
+  const normalized = {
+    bot_id: botId,
+    tenant_id: tenantId,
+    name: text(bot.name || botId).slice(0, 120),
+    adapter,
+    enabled: bot.enabled !== false,
+    policy: normalizePolicy(bot.policy),
+  };
+
+  if (adapter === "zalo_oa") {
+    const credential = bot.credential;
+    if (!credential || typeof credential !== "object" || Array.isArray(credential)) {
+      throw new BotRegistryError("credential_missing", `${botId}: OA credential references are required`);
+    }
+    normalized.credential = {
+      app_id_env: requireEnvRef(credential.app_id_env),
+      app_secret_env: requireEnvRef(credential.app_secret_env),
+      refresh_token_env: requireEnvRef(credential.refresh_token_env),
+      token_url: text(credential.token_url || ""),
+      message_url: text(credential.message_url || ""),
+    };
+  } else {
+    const accountId = text(bot.account_id || botId);
+    if (!ID_RE.test(accountId)) throw new BotRegistryError("invalid_account_id", `${botId}: invalid account_id`);
+    normalized.account_id = accountId;
+  }
+  return normalized;
+}
+
+export function validateBotRegistry(input) {
+  if (!input || typeof input !== "object" || Array.isArray(input)) {
+    throw new BotRegistryError("config_invalid", "bot registry must be an object");
+  }
+  const version = Number(input.version || 1);
+  if (version !== 1) throw new BotRegistryError("unsupported_registry_version", `unsupported registry version: ${version}`);
+  if (!Array.isArray(input.bots)) throw new BotRegistryError("invalid_bots", "bots must be an array");
+  if (input.bots.length > 1000) throw new BotRegistryError("invalid_bots", "too many bots");
+
+  const bots = input.bots.map(normalizeBot);
+  const ids = new Set();
+  for (const bot of bots) {
+    if (ids.has(bot.bot_id)) throw new BotRegistryError("duplicate_bot_id", `duplicate bot_id: ${bot.bot_id}`);
+    ids.add(bot.bot_id);
+  }
+  return { version: 1, bots };
+}
+
+export function loadBotRegistry(filePath = process.env.BOTS_FILE || "./config/bots.json") {
+  const abs = path.resolve(filePath);
+  if (!fs.existsSync(abs)) return { version: 1, bots: [] };
+  let parsed;
+  try {
+    parsed = JSON.parse(fs.readFileSync(abs, "utf8"));
+  } catch {
+    throw new BotRegistryError("config_invalid", "bot registry is not valid JSON");
+  }
+  return validateBotRegistry(parsed);
+}
+
+function envValue(env, key) {
+  return text(env?.[key]);
+}
+
+export function createOaAdapterMap({ registry, env = process.env, fetchImpl, now } = {}) {
+  const normalized = validateBotRegistry(registry || { version: 1, bots: [] });
+  const adapters = new Map();
+  for (const bot of normalized.bots) {
+    if (!bot.enabled || bot.adapter !== "zalo_oa") continue;
+    adapters.set(
+      bot.bot_id,
+      new OaAdapter({
+        botId: bot.bot_id,
+        tenantId: bot.tenant_id,
+        appId: envValue(env, bot.credential.app_id_env),
+        appSecret: envValue(env, bot.credential.app_secret_env),
+        refreshToken: envValue(env, bot.credential.refresh_token_env),
+        tokenUrl: bot.credential.token_url || undefined,
+        messageUrl: bot.credential.message_url || undefined,
+        fetchImpl,
+        now,
+      }),
+    );
+  }
+  return adapters;
+}
+
+export function redactBotRegistry(registry) {
+  const normalized = validateBotRegistry(registry || { version: 1, bots: [] });
+  return {
+    version: normalized.version,
+    bots: normalized.bots.map((bot) => ({
+      ...bot,
+      credential: bot.credential
+        ? {
+            app_id_env: bot.credential.app_id_env,
+            app_secret_env: bot.credential.app_secret_env,
+            refresh_token_env: bot.credential.refresh_token_env,
+            ...(bot.credential.token_url ? { token_url: bot.credential.token_url } : {}),
+            ...(bot.credential.message_url ? { message_url: bot.credential.message_url } : {}),
+          }
+        : undefined,
+    })),
+  };
+}
+
+export function getBot(registry, botId) {
+  const normalized = validateBotRegistry(registry || { version: 1, bots: [] });
+  return normalized.bots.find((bot) => bot.bot_id === text(botId)) || null;
+}
+
+export const BOT_ADAPTERS = Object.freeze([...ADAPTERS]);
+export const BOT_POLICY_MODES = Object.freeze([...POLICY_MODES]);

package/src/brand.js

@@ -0,0 +1,31 @@
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+
+/**
+ * anyone running this fork should be able to present it as their own tool.
+ */
+function readPackage() {
+  try {
+    return JSON.parse(fs.readFileSync(path.join(ROOT, "package.json"), "utf8"));
+  } catch {
+    return {};
+  }
+}
+
+const pkg = readPackage();
+
+export const serviceMetadata = Object.freeze({
+  name: pkg.name || "abs-zalo-bot",
+  version: pkg.version || "0.0.0",
+  license: pkg.license || "MIT",
+});
+
+/** Kept for backward compatibility with existing imports. */
+export function publicBrandMetadata() {
+  return { ...serviceMetadata };
+}
+
+export default serviceMetadata;