@uniquedivine/tg-sai 0.0.5 → 0.1.0

package/custodyWorker.ts

@@ -2,7 +2,17 @@ import { randomUUID } from "node:crypto"
 import { mkdir, open, readFile } from "node:fs/promises"
 import { dirname, join } from "node:path"
 import { configDir } from "./profiles"
-import { provisionSetup, safeProvisionError } from "./provisioning"
+import {
+  closeProcessHoldStores,
+  processHoldStore,
+} from "./mtcuteAdapter"
+import {
+  cloudRetryAt,
+  HOLD_STATE_UNAVAILABLE,
+  HOLD_STATE_UNAVAILABLE_RETRY_MS,
+  provisionSetup,
+  safeProvisionError,
+} from "./provisioning"
 import type {
   ProvisionCloud,
   ProvisionJournal,
@@ -15,6 +25,12 @@ import {
   withAccountLock,
   withProvisionTelegram,
 } from "./provisioningRuntime"
+import type { HoldStoreSource } from "./state/guard"
+import { HoldStore, remainingSeconds } from "./state/holds"
+import type { HoldGroup } from "./state/holds"
+import { WORKER_TICK_EVENT } from "./telemetry/db"
+import { withTelemetryJob } from "./telemetry/middleware"
+import type { TelemetryRun } from "./telemetry/run"
 
 export const CUSTODY_USER_ID = 8274304175
 export const REQUIRED_UNIQUE_ADMIN_ID = 1912700300
@@ -44,6 +60,8 @@ export interface WorkerOptions {
   run?: boolean
   now?: number
   after?: string
+  /** Once aborted, no new job starts; the running one is cut short. */
+  signal?: AbortSignal
 }
 /** Persistent random identity retains ownership of sticky cloud claims after restart. */
 export async function stableWorkerId(root = configDir()): Promise<string> {
@@ -68,12 +86,64 @@ export async function stableWorkerId(root = configDir()): Promise<string> {
   return id
 }
 
+/** Hold groups a setup job uses: resolve handles, create, add members. */
+export const SETUP_HOLD_GROUPS: readonly HoldGroup[] = [
+  "create",
+  "invite",
+  "resolve",
+]
+
+/** Groups a setup that already has its chat still uses: no create. */
+export const CONFIGURE_HOLD_GROUPS: readonly HoldGroup[] = ["invite", "resolve"]
+
+/** Hold groups that block `setup`: create only matters before its chat. */
+export function setupHoldGroups(
+  setup: Pick<Setup, "chatId">,
+): readonly HoldGroup[] {
+  return setup.chatId ? CONFIGURE_HOLD_GROUPS : SETUP_HOLD_GROUPS
+}
+
+/**
+ * When the custody account's saved flood holds allow setup work again:
+ * the latest `untilMs` of its holds on `groups` (default create, invite,
+ * and resolve), or `HOLD_STATE_UNAVAILABLE_RETRY_MS` from now when the
+ * store cannot be read. `null` when no hold applies. Never throws.
+ */
+export function setupHoldUntil(
+  holds: HoldStoreSource,
+  accountId: number,
+  nowMs: number,
+  groups: readonly HoldGroup[] = SETUP_HOLD_GROUPS,
+): { untilMs: number; code: string } | null {
+  try {
+    const store = typeof holds === "function" ? holds() : holds
+    let untilMs = 0
+    for (const grp of groups) {
+      const hold = store.activeHold(accountId, grp, nowMs)
+      if (hold && hold.untilMs > untilMs) untilMs = hold.untilMs
+    }
+    return untilMs > nowMs
+      ? { untilMs, code: `TG_SAI_HOLD_${remainingSeconds(untilMs, nowMs)}` }
+      : null
+  } catch {
+    return {
+      untilMs: nowMs + HOLD_STATE_UNAVAILABLE_RETRY_MS,
+      code: HOLD_STATE_UNAVAILABLE,
+    }
+  }
+}
+
 /** One bounded page per tick; authoritative queue and local checks both scope jobs. */
 export async function runWorkerOnce(
   options: WorkerOptions,
   deps: {
     cloud: ProvisionCloud & PendingQueue
     journal: ProvisionJournal
+    /**
+     * Saved flood holds; a held account is neither claimed nor contacted.
+     * A provider is called each time, so an unavailable store can reopen.
+     */
+    holds: HoldStoreSource
     telegram: <T>(
       setup: Setup,
       operation: (adapter: TelegramProvisioner) => Promise<T>,
@@ -108,6 +178,7 @@ export async function runWorkerOnce(
     error: row.reason,
   }))
   for (const selected of page.requests) {
+    if (options.signal?.aborted) break
     const setup = await deps.cloud.view(selected.requestId)
     if (
       setup.expectedUserId !== CUSTODY_USER_ID ||
@@ -163,6 +234,17 @@ export async function runWorkerOnce(
       })
       continue
     }
+    const nowMs = options.now ?? Date.now()
+    const hold = setupHoldUntil(
+      deps.holds,
+      setup.expectedUserId,
+      nowMs,
+      setupHoldGroups(setup),
+    )
+    if (hold) {
+      results.push(await reportHold(deps.cloud, setup, hold, nowMs))
+      continue
+    }
     try {
       const result = await deps.telegram(setup, (telegram) =>
         provisionSetup({
@@ -195,6 +277,46 @@ export async function runWorkerOnce(
   return { dryRun: !options.run, results, nextAfter: page.nextAfter }
 }
 
+/**
+ * Waiting result for a held account. An incomplete setup this worker already
+ * holds the claim for gets `cloudRetryAt` of the hold's end as its cloud
+ * `retryAt`, through the
+ * same `progress` call a flood retry uses, so Deal Desk shows the time. Other
+ * setups are not claimed; Deal Desk cannot store a `retryAt` for them.
+ */
+async function reportHold(
+  cloud: ProvisionCloud,
+  setup: Setup,
+  hold: { untilMs: number; code: string },
+  nowMs: number,
+): Promise<WorkerResult> {
+  // Same value provisioning saves for a hold, so the cloud time never ends
+  // before the local hold and the next tick does not report it again.
+  const retryAt =
+    cloudRetryAt(hold.untilMs, nowMs) ?? new Date(hold.untilMs).toISOString()
+  const result: WorkerResult = {
+    requestId: setup.requestId,
+    state: "waiting",
+    ...(setup.chatId ? { chatId: setup.chatId } : {}),
+    retryAt,
+    error: hold.code,
+  }
+  if (setup.state !== "incomplete" || !setup.claimToken) return result
+  try {
+    await cloud.progress(setup.requestId, setup.claimToken, {
+      state: "incomplete",
+      ...(setup.chatId ? { chatId: setup.chatId } : {}),
+      ...(setup.binding ? { binding: setup.binding } : {}),
+      error: hold.code,
+      retryAt,
+    })
+  } catch (error) {
+    // The local hold still keeps the job waiting; the cloud may lag.
+    return { ...result, error: `${hold.code}:${safeProvisionError(error)}` }
+  }
+  return result
+}
+
 export async function runCustodyWorker(options: {
   custodyProfile: string
   once?: boolean
@@ -202,6 +324,8 @@ export async function runCustodyWorker(options: {
   limit?: number
   intervalMs?: number
   signal?: AbortSignal
+  /** Process run: worker.tick rows and per-job telemetry. */
+  telemetry?: Pick<TelemetryRun, "sink" | "context" | "client">
 }): Promise<void> {
   const interval = options.intervalMs ?? 5000
   if (!Number.isSafeInteger(interval) || interval < 1000 || interval > 60000)
@@ -212,44 +336,91 @@ export async function runCustodyWorker(options: {
     )
   const root = configDir()
   const workerId = await stableWorkerId(root)
+  const telemetry = options.telemetry
+  const clientTelemetry = telemetry?.client({
+    accountId: CUSTODY_USER_ID,
+    profile: options.custodyProfile,
+  })
+  // Each job's Telegram calls carry its setup ID as job_id.
+  const telegram = <T>(
+    setup: Setup,
+    operation: (adapter: TelegramProvisioner) => Promise<T>,
+  ): Promise<T> =>
+    withTelemetryJob(setup.requestId, () =>
+      withProvisionTelegram(setup, operation, clientTelemetry, options.signal),
+    )
+  let lastTickRecordedMs: number | null = null
+  const recordTick = (tick: {
+    startedAt: number
+    result?: Awaited<ReturnType<typeof runWorkerOnce>>
+    error?: unknown
+  }) => {
+    if (!telemetry) return
+    const nowMs = Date.now()
+    if (
+      !shouldRecordTick({
+        ...tick,
+        nowMs,
+        lastRecordedMs: lastTickRecordedMs,
+      })
+    ) {
+      return
+    }
+    lastTickRecordedMs = nowMs
+    recordWorkerTick(telemetry, {
+      ...tick,
+      custodyProfile: options.custodyProfile,
+    })
+  }
   const execute = async () => {
     const cloud = new CliProvisionCloud()
+    // Preview never reaches the hold check, so it opens no state file. A
+    // run shares the guard's store, which reopens after a failed open.
+    const holds: HoldStoreSource = options.run
+      ? () => processHoldStore()
+      : HoldStore.unavailable("preview")
     let after: string | undefined
-    do {
-      try {
-        const result = await runWorkerOnce(
-          { ...options, workerId, after },
-          {
-            cloud,
-            journal: new FileProvisionJournal(root),
-            telegram: withProvisionTelegram,
-          },
-        )
-        after = result.nextAfter ?? undefined
-        console.log(
-          JSON.stringify({ ...result, observedAt: new Date().toISOString() }),
-        )
-      } catch (error) {
-        if (options.once) throw error
-        console.error(
-          JSON.stringify({
+    try {
+      do {
+        const tickStartedAt = Date.now()
+        try {
+          const result = await runWorkerOnce(
+            { ...options, workerId, after },
+            {
+              cloud,
+              journal: new FileProvisionJournal(root),
+              holds,
+              telegram,
+            },
+          )
+          after = result.nextAfter ?? undefined
+          const line = { ...result, observedAt: new Date().toISOString() }
+          console.log(JSON.stringify(line))
+          recordTick({ startedAt: tickStartedAt, result })
+        } catch (error) {
+          recordTick({ startedAt: tickStartedAt, error })
+          if (options.once) throw error
+          const line = {
             state: "worker_tick_failed",
             observedAt: new Date().toISOString(),
             retryInMs: interval,
-          }),
-        )
-      }
-      if (options.once || options.signal?.aborted) break
-      await new Promise<void>((resolve) => {
-        const done = () => {
-          clearTimeout(timer)
-          options.signal?.removeEventListener("abort", done)
-          resolve()
+          }
+          console.error(JSON.stringify(line))
         }
-        const timer = setTimeout(done, interval)
-        options.signal?.addEventListener("abort", done, { once: true })
-      })
-    } while (!options.signal?.aborted)
+        if (options.once || options.signal?.aborted) break
+        await new Promise<void>((resolve) => {
+          const done = () => {
+            clearTimeout(timer)
+            options.signal?.removeEventListener("abort", done)
+            resolve()
+          }
+          const timer = setTimeout(done, interval)
+          options.signal?.addEventListener("abort", done, { once: true })
+        })
+      } while (!options.signal?.aborted)
+    } finally {
+      if (options.run) closeProcessHoldStores()
+    }
   }
   if (!options.run) return execute()
   await withAccountLock(
@@ -257,3 +428,74 @@ export async function runCustodyWorker(options: {
     execute,
   )
 }
+
+/** Counts results per state, e.g. `{ ready: 2, waiting: 1 }`. */
+export function countStates(results: WorkerResult[]): Record<string, number> {
+  const counts: Record<string, number> = {}
+  for (const result of results) {
+    counts[result.state] = (counts[result.state] ?? 0) + 1
+  }
+  return counts
+}
+
+/** Idle ticks still write a `worker.tick` row this often, as a heartbeat. */
+export const TICK_HEARTBEAT_MS = 5 * 60 * 1000
+
+/** Job states that mean the tick did no work. */
+const IDLE_STATES = new Set(["waiting", "blocked"])
+
+/**
+ * Whether a tick earns a `worker.tick` row: it threw, a job did something
+ * (any result not `waiting`/`blocked`), it is the first tick, or the last
+ * recorded tick is `TICK_HEARTBEAT_MS` old. Idle polls are otherwise skipped
+ * so a 5s interval does not write ~17k rows a day.
+ */
+export function shouldRecordTick(tick: {
+  result?: Awaited<ReturnType<typeof runWorkerOnce>>
+  error?: unknown
+  nowMs: number
+  lastRecordedMs: number | null
+}): boolean {
+  if (tick.error !== undefined) return true
+  if (tick.result?.results.some((r) => !IDLE_STATES.has(r.state))) return true
+  if (tick.lastRecordedMs === null) return true
+  return tick.nowMs - tick.lastRecordedMs >= TICK_HEARTBEAT_MS
+}
+
+/** Writes one `worker.tick` row; never throws. */
+export function recordWorkerTick(
+  telemetry: Pick<TelemetryRun, "sink" | "context">,
+  tick: {
+    startedAt: number
+    custodyProfile: string
+    result?: Awaited<ReturnType<typeof runWorkerOnce>>
+    error?: unknown
+  },
+): void {
+  try {
+    const { context } = telemetry
+    telemetry.sink.record({
+      runId: context.runId,
+      subsystem: context.subsystem,
+      command: context.command,
+      version: context.version,
+      accountId: CUSTODY_USER_ID,
+      profile: tick.custodyProfile,
+      event: WORKER_TICK_EVENT,
+      outcome: tick.error === undefined ? "ok" : "error",
+      durationMs: Date.now() - tick.startedAt,
+      err:
+        tick.error === undefined ? null : safeProvisionError(tick.error),
+      res: tick.result
+        ? {
+            dryRun: tick.result.dryRun,
+            jobs: tick.result.results.length,
+            states: countStates(tick.result.results),
+            more: tick.result.nextAfter !== null,
+          }
+        : null,
+    })
+  } catch {
+    // Telemetry never stops the worker.
+  }
+}

package/custodyWorkerCli.ts

@@ -1,5 +1,6 @@
 import type { Command } from "commander"
 import { runCustodyWorker } from "./custodyWorker"
+import { activeTelemetryRun } from "./telemetry/run"
 
 /** Root main.ts calls this once beside addProvisionCommands. No MCP is involved. */
 export function addCustodyWorkerCommands(
@@ -32,7 +33,11 @@ export function addCustodyWorkerCommands(
         intervalMs: string
       }) => {
         const controller = new AbortController()
-        const stop = () => controller.abort()
+        const telemetry = activeTelemetryRun() ?? undefined
+        const stop = () => {
+          telemetry?.markAborted()
+          controller.abort()
+        }
         process.once("SIGTERM", stop)
         process.once("SIGINT", stop)
         try {
@@ -42,6 +47,7 @@ export function addCustodyWorkerCommands(
             limit: Number(options.limit),
             intervalMs: Number(options.intervalMs),
             signal: controller.signal,
+            ...(telemetry ? { telemetry } : {}),
           })
         } finally {
           process.removeListener("SIGTERM", stop)