@uniquedivine/tg-sai 0.0.4 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -0
- package/RUNBOOK.md +204 -24
- package/actionExecutor.ts +183 -131
- package/adminAudit.ts +60 -60
- package/captureProvisioningGolden.ts +98 -0
- package/cfg.ts +12 -1
- package/custodyWorker.ts +501 -0
- package/custodyWorkerCli.ts +58 -0
- package/health.ts +31 -50
- package/main.ts +860 -214
- package/mtcuteAdapter.ts +266 -10
- package/ops.ts +12 -0
- package/package.json +14 -5
- package/profiles.ts +157 -32
- package/provisioning.ts +805 -0
- package/provisioningCli.ts +147 -0
- package/provisioningCloud.ts +214 -0
- package/provisioningGoldens.ts +70 -0
- package/provisioningRuntime.ts +417 -0
- package/runtimeAssets.d.ts +4 -0
- package/runtimeHealth.ts +132 -0
- package/runtimeHealthCli.ts +48 -0
- package/session_string_generator.py +36 -24
- package/state/cli.ts +641 -0
- package/state/db.ts +72 -0
- package/state/guard.ts +270 -0
- package/state/holds.ts +381 -0
- package/state/migrations/0001-flood-hold.sql +16 -0
- package/telegramRuntime.ts +25 -0
- package/telemetry/cli.ts +698 -0
- package/telemetry/db.ts +471 -0
- package/telemetry/middleware.ts +475 -0
- package/telemetry/migrations/0001-event.sql +33 -0
- package/telemetry/migrations/0002-event-job-index.sql +2 -0
- package/telemetry/migrations/0003-event-log.sql +10 -0
- package/telemetry/queries.ts +475 -0
- package/telemetry/run.ts +491 -0
- package/telemetry/sqlAssets.d.ts +4 -0
- package/telemetry/summarize.ts +219 -0
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"
|
|
2
|
+
import { tmpdir } from "node:os"
|
|
3
|
+
import { join } from "node:path"
|
|
4
|
+
import { Command } from "commander"
|
|
5
|
+
import { getChatMember, iterDialogs } from "@mtcute/bun/methods.js"
|
|
6
|
+
import { createMtcuteClient } from "./mtcuteAdapter"
|
|
7
|
+
import { strictProfileCredentials } from "./profiles"
|
|
8
|
+
import { creationMarker, PROVISIONING_BOT_ID } from "./provisioning"
|
|
9
|
+
import { setupFromCloud } from "./provisioningCloud"
|
|
10
|
+
import { requireExactCaptureDialog, sanitizedProvisioningGolden } from "./provisioningGoldens"
|
|
11
|
+
import { resolveProvisioningPeer } from "./provisioningRuntime"
|
|
12
|
+
|
|
13
|
+
// Only read methods are used. An explicit persisted setup binds the target chat
|
|
14
|
+
// and identities; a fresh temporary store avoids touching worker session files.
|
|
15
|
+
export async function captureProvisioningGolden(options: {
|
|
16
|
+
profile: string; chat: string; setup: string; out: string
|
|
17
|
+
}) {
|
|
18
|
+
const chatId = Number(options.chat)
|
|
19
|
+
const setup = setupFromCloud(JSON.parse(await readFile(options.setup, "utf8")))
|
|
20
|
+
if (!Number.isSafeInteger(chatId) || chatId >= 0 || setup.chatId !== chatId ||
|
|
21
|
+
setup.expectedUserId !== 8274304175 || !setup.binding ||
|
|
22
|
+
setup.custodyProfile !== options.profile)
|
|
23
|
+
throw new Error("Capture requires the exact custody setup and group")
|
|
24
|
+
const credentials = await strictProfileCredentials(options.profile, setup.expectedUserId)
|
|
25
|
+
const directory = await mkdtemp(join(tmpdir(), "tg-sai-golden-"))
|
|
26
|
+
let client: Awaited<ReturnType<typeof createMtcuteClient>> | undefined
|
|
27
|
+
let stage = "connect"
|
|
28
|
+
try {
|
|
29
|
+
client = await createMtcuteClient({ apiId: credentials.apiId,
|
|
30
|
+
apiHash: credentials.apiHash, session: credentials.sessionString,
|
|
31
|
+
storage: join(directory, "session") })
|
|
32
|
+
stage = "self"
|
|
33
|
+
const me = await client.getMe()
|
|
34
|
+
if (me.id !== setup.expectedUserId || me.isBot)
|
|
35
|
+
throw new Error("Custody identity mismatch")
|
|
36
|
+
stage = "dialogs"
|
|
37
|
+
await requireExactCaptureDialog(iterDialogs(client, {
|
|
38
|
+
limit: 1000, archived: "keep",
|
|
39
|
+
}), chatId)
|
|
40
|
+
stage = "fullchat"
|
|
41
|
+
const chat = await client.getFullChat(chatId)
|
|
42
|
+
const marker = creationMarker(setup)
|
|
43
|
+
console.log(JSON.stringify({ stage: "group_verified_readback", exactId: chat.id === chatId,
|
|
44
|
+
chatType: chat.chatType, isForum: chat.isForum, isCreator: chat.isCreator,
|
|
45
|
+
markerMatches: chat.bio === marker }))
|
|
46
|
+
if (chat.id !== chatId || chat.chatType !== "supergroup" || chat.isForum ||
|
|
47
|
+
!chat.isCreator || chat.bio !== marker)
|
|
48
|
+
throw new Error("Capture group ownership or marker mismatch")
|
|
49
|
+
const subjects = new Map<number, string>([
|
|
50
|
+
[setup.expectedUserId, "custody"], [PROVISIONING_BOT_ID, "bot"],
|
|
51
|
+
])
|
|
52
|
+
setup.binding.admins.forEach((id, i) => {
|
|
53
|
+
if (!subjects.has(id)) subjects.set(id, `required_admin_${i}`)
|
|
54
|
+
})
|
|
55
|
+
setup.binding.participants.forEach((id, i) => {
|
|
56
|
+
if (!subjects.has(id)) subjects.set(id, `ordinary_${i}`)
|
|
57
|
+
})
|
|
58
|
+
const members = []
|
|
59
|
+
for (const [userId, subject] of subjects) {
|
|
60
|
+
stage = `member_${subject}`
|
|
61
|
+
if (subject.startsWith("ordinary_"))
|
|
62
|
+
await resolveProvisioningPeer(peer => client!.getUser(peer), userId)
|
|
63
|
+
const value = await getChatMember(client, { chatId, userId })
|
|
64
|
+
members.push({ subject, value })
|
|
65
|
+
console.log(JSON.stringify({ subject, status: value?.status ?? "absent" }))
|
|
66
|
+
}
|
|
67
|
+
stage = "invitations"
|
|
68
|
+
const invites = await client.getInviteLinks(chatId, { limit: 100 })
|
|
69
|
+
const golden = sanitizedProvisioningGolden({ capturedAt: new Date().toISOString(),
|
|
70
|
+
chat, expectedMarker: marker, members, invites })
|
|
71
|
+
await writeFile(options.out, `${JSON.stringify(golden, null, 2)}\n`, { flag: "wx", mode: 0o600 })
|
|
72
|
+
return { members: members.length, invitations: invites.length }
|
|
73
|
+
} catch (error) {
|
|
74
|
+
const record = error && typeof error === "object" ? error as Record<string, unknown> : {}
|
|
75
|
+
const code = typeof record.text === "string" && /^[A-Z0-9_]{1,100}$/.test(record.text)
|
|
76
|
+
? record.text : "unclassified"
|
|
77
|
+
console.error(JSON.stringify({ stage, code }))
|
|
78
|
+
throw new Error("Sanitized capture failure")
|
|
79
|
+
} finally {
|
|
80
|
+
try { await client?.disconnect() } finally { await rm(directory, { recursive: true, force: true }) }
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
if (import.meta.main) {
|
|
85
|
+
const command = new Command().description("Capture sanitized read-only evidence from one authorized custody group")
|
|
86
|
+
.requiredOption("--profile <name>", "Exact configured custody profile")
|
|
87
|
+
.requiredOption("--chat <id>", "Exact negative Telegram group ID")
|
|
88
|
+
.requiredOption("--setup <path>", "Deal Desk setup readback JSON with resolved bindings")
|
|
89
|
+
.requiredOption("--out <path>", "New sanitized fixture file; existing files are refused")
|
|
90
|
+
command.parse()
|
|
91
|
+
try {
|
|
92
|
+
const counts = await captureProvisioningGolden(command.opts())
|
|
93
|
+
console.log(`Saved sanitized readback: ${counts.members} members, ${counts.invitations} invitations`)
|
|
94
|
+
} catch {
|
|
95
|
+
console.error("Capture failed. Check the exact setup, custody identity, permissions, and destination. Raw Telegram errors are suppressed.")
|
|
96
|
+
process.exitCode = 1
|
|
97
|
+
}
|
|
98
|
+
}
|
package/cfg.ts
CHANGED
|
@@ -59,6 +59,18 @@ export const USERS = {
|
|
|
59
59
|
displayName: "Corinne Antonia",
|
|
60
60
|
kind: "user",
|
|
61
61
|
},
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Sai custody account: the intended owner of every Sai deal chat. Same
|
|
65
|
+
* numeric ID as `corinneAlt`, renamed and verified through `getMe` on
|
|
66
|
+
* 2026-11-05. Default ownership target for `ops sai-bootstrap`.
|
|
67
|
+
*/
|
|
68
|
+
saiCustody: {
|
|
69
|
+
id: 8274304175,
|
|
70
|
+
handle: "SaiTeam_AI",
|
|
71
|
+
displayName: "Sai AI Automation",
|
|
72
|
+
kind: "user",
|
|
73
|
+
},
|
|
62
74
|
} satisfies Record<string, TelegramIdentity>
|
|
63
75
|
|
|
64
76
|
/**
|
|
@@ -70,4 +82,3 @@ export const nibiruInternal = {
|
|
|
70
82
|
title: "Nibiru | Internal",
|
|
71
83
|
kind: "channel",
|
|
72
84
|
} satisfies TelegramChatIdentity
|
|
73
|
-
|
package/custodyWorker.ts
ADDED
|
@@ -0,0 +1,501 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto"
|
|
2
|
+
import { mkdir, open, readFile } from "node:fs/promises"
|
|
3
|
+
import { dirname, join } from "node:path"
|
|
4
|
+
import { configDir } from "./profiles"
|
|
5
|
+
import {
|
|
6
|
+
closeProcessHoldStores,
|
|
7
|
+
processHoldStore,
|
|
8
|
+
} from "./mtcuteAdapter"
|
|
9
|
+
import {
|
|
10
|
+
cloudRetryAt,
|
|
11
|
+
HOLD_STATE_UNAVAILABLE,
|
|
12
|
+
HOLD_STATE_UNAVAILABLE_RETRY_MS,
|
|
13
|
+
provisionSetup,
|
|
14
|
+
safeProvisionError,
|
|
15
|
+
} from "./provisioning"
|
|
16
|
+
import type {
|
|
17
|
+
ProvisionCloud,
|
|
18
|
+
ProvisionJournal,
|
|
19
|
+
Setup,
|
|
20
|
+
TelegramProvisioner,
|
|
21
|
+
} from "./provisioning"
|
|
22
|
+
import { CliProvisionCloud } from "./provisioningCloud"
|
|
23
|
+
import {
|
|
24
|
+
FileProvisionJournal,
|
|
25
|
+
withAccountLock,
|
|
26
|
+
withProvisionTelegram,
|
|
27
|
+
} from "./provisioningRuntime"
|
|
28
|
+
import type { HoldStoreSource } from "./state/guard"
|
|
29
|
+
import { HoldStore, remainingSeconds } from "./state/holds"
|
|
30
|
+
import type { HoldGroup } from "./state/holds"
|
|
31
|
+
import { WORKER_TICK_EVENT } from "./telemetry/db"
|
|
32
|
+
import { withTelemetryJob } from "./telemetry/middleware"
|
|
33
|
+
import type { TelemetryRun } from "./telemetry/run"
|
|
34
|
+
|
|
35
|
+
export const CUSTODY_USER_ID = 8274304175
|
|
36
|
+
export const REQUIRED_UNIQUE_ADMIN_ID = 1912700300
|
|
37
|
+
export interface PendingQueue {
|
|
38
|
+
pending(input: {
|
|
39
|
+
expectedUserId: number
|
|
40
|
+
workerId: string
|
|
41
|
+
limit: number
|
|
42
|
+
after?: string
|
|
43
|
+
}): Promise<{
|
|
44
|
+
requests: Setup[]
|
|
45
|
+
nextAfter: string | null
|
|
46
|
+
blocked?: { requestId: string; state: string; reason: string }[]
|
|
47
|
+
}>
|
|
48
|
+
}
|
|
49
|
+
export interface WorkerResult {
|
|
50
|
+
requestId: string
|
|
51
|
+
state: string
|
|
52
|
+
chatId?: number
|
|
53
|
+
retryAt?: string | null
|
|
54
|
+
error?: string
|
|
55
|
+
}
|
|
56
|
+
export interface WorkerOptions {
|
|
57
|
+
custodyProfile: string
|
|
58
|
+
workerId: string
|
|
59
|
+
limit?: number
|
|
60
|
+
run?: boolean
|
|
61
|
+
now?: number
|
|
62
|
+
after?: string
|
|
63
|
+
/** Once aborted, no new job starts; the running one is cut short. */
|
|
64
|
+
signal?: AbortSignal
|
|
65
|
+
}
|
|
66
|
+
/** Persistent random identity retains ownership of sticky cloud claims after restart. */
|
|
67
|
+
export async function stableWorkerId(root = configDir()): Promise<string> {
|
|
68
|
+
const path = join(root, "provisioning", "worker-id")
|
|
69
|
+
await mkdir(dirname(path), { recursive: true, mode: 0o700 })
|
|
70
|
+
try {
|
|
71
|
+
const file = await open(path, "wx", 0o600)
|
|
72
|
+
try {
|
|
73
|
+
await file.writeFile(`custody:${CUSTODY_USER_ID}:${randomUUID()}`)
|
|
74
|
+
await file.sync()
|
|
75
|
+
} finally {
|
|
76
|
+
await file.close()
|
|
77
|
+
}
|
|
78
|
+
} catch (error) {
|
|
79
|
+
if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error
|
|
80
|
+
}
|
|
81
|
+
const id = (await readFile(path, "utf8")).trim()
|
|
82
|
+
if (!new RegExp(`^custody:${CUSTODY_USER_ID}:[a-f0-9-]{36}$`).test(id))
|
|
83
|
+
throw new Error(
|
|
84
|
+
"Invalid saved worker identity; inspect private worker state",
|
|
85
|
+
)
|
|
86
|
+
return id
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** Hold groups a setup job uses: resolve handles, create, add members. */
|
|
90
|
+
export const SETUP_HOLD_GROUPS: readonly HoldGroup[] = [
|
|
91
|
+
"create",
|
|
92
|
+
"invite",
|
|
93
|
+
"resolve",
|
|
94
|
+
]
|
|
95
|
+
|
|
96
|
+
/** Groups a setup that already has its chat still uses: no create. */
|
|
97
|
+
export const CONFIGURE_HOLD_GROUPS: readonly HoldGroup[] = ["invite", "resolve"]
|
|
98
|
+
|
|
99
|
+
/** Hold groups that block `setup`: create only matters before its chat. */
|
|
100
|
+
export function setupHoldGroups(
|
|
101
|
+
setup: Pick<Setup, "chatId">,
|
|
102
|
+
): readonly HoldGroup[] {
|
|
103
|
+
return setup.chatId ? CONFIGURE_HOLD_GROUPS : SETUP_HOLD_GROUPS
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* When the custody account's saved flood holds allow setup work again:
|
|
108
|
+
* the latest `untilMs` of its holds on `groups` (default create, invite,
|
|
109
|
+
* and resolve), or `HOLD_STATE_UNAVAILABLE_RETRY_MS` from now when the
|
|
110
|
+
* store cannot be read. `null` when no hold applies. Never throws.
|
|
111
|
+
*/
|
|
112
|
+
export function setupHoldUntil(
|
|
113
|
+
holds: HoldStoreSource,
|
|
114
|
+
accountId: number,
|
|
115
|
+
nowMs: number,
|
|
116
|
+
groups: readonly HoldGroup[] = SETUP_HOLD_GROUPS,
|
|
117
|
+
): { untilMs: number; code: string } | null {
|
|
118
|
+
try {
|
|
119
|
+
const store = typeof holds === "function" ? holds() : holds
|
|
120
|
+
let untilMs = 0
|
|
121
|
+
for (const grp of groups) {
|
|
122
|
+
const hold = store.activeHold(accountId, grp, nowMs)
|
|
123
|
+
if (hold && hold.untilMs > untilMs) untilMs = hold.untilMs
|
|
124
|
+
}
|
|
125
|
+
return untilMs > nowMs
|
|
126
|
+
? { untilMs, code: `TG_SAI_HOLD_${remainingSeconds(untilMs, nowMs)}` }
|
|
127
|
+
: null
|
|
128
|
+
} catch {
|
|
129
|
+
return {
|
|
130
|
+
untilMs: nowMs + HOLD_STATE_UNAVAILABLE_RETRY_MS,
|
|
131
|
+
code: HOLD_STATE_UNAVAILABLE,
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** One bounded page per tick; authoritative queue and local checks both scope jobs. */
|
|
137
|
+
export async function runWorkerOnce(
|
|
138
|
+
options: WorkerOptions,
|
|
139
|
+
deps: {
|
|
140
|
+
cloud: ProvisionCloud & PendingQueue
|
|
141
|
+
journal: ProvisionJournal
|
|
142
|
+
/**
|
|
143
|
+
* Saved flood holds; a held account is neither claimed nor contacted.
|
|
144
|
+
* A provider is called each time, so an unavailable store can reopen.
|
|
145
|
+
*/
|
|
146
|
+
holds: HoldStoreSource
|
|
147
|
+
telegram: <T>(
|
|
148
|
+
setup: Setup,
|
|
149
|
+
operation: (adapter: TelegramProvisioner) => Promise<T>,
|
|
150
|
+
) => Promise<T>
|
|
151
|
+
},
|
|
152
|
+
): Promise<{
|
|
153
|
+
dryRun: boolean
|
|
154
|
+
results: WorkerResult[]
|
|
155
|
+
nextAfter: string | null
|
|
156
|
+
}> {
|
|
157
|
+
if (!options.custodyProfile.trim() || !options.workerId.trim())
|
|
158
|
+
throw new Error("Explicit custody profile and stable worker ID required")
|
|
159
|
+
const limit = options.limit ?? 10
|
|
160
|
+
if (!Number.isSafeInteger(limit) || limit < 1 || limit > 50)
|
|
161
|
+
throw new Error("Worker limit must be 1 through 50")
|
|
162
|
+
const page = await deps.cloud.pending({
|
|
163
|
+
expectedUserId: CUSTODY_USER_ID,
|
|
164
|
+
workerId: options.workerId,
|
|
165
|
+
limit,
|
|
166
|
+
...(options.after ? { after: options.after } : {}),
|
|
167
|
+
})
|
|
168
|
+
if (page.requests.length > limit)
|
|
169
|
+
throw new Error("Pending queue exceeded requested bound")
|
|
170
|
+
if (
|
|
171
|
+
!Array.isArray(page.blocked ?? []) ||
|
|
172
|
+
(page.blocked?.length ?? 0) + page.requests.length > limit
|
|
173
|
+
)
|
|
174
|
+
throw new Error("Pending queue blocked results exceeded requested bound")
|
|
175
|
+
const results: WorkerResult[] = (page.blocked ?? []).map((row) => ({
|
|
176
|
+
requestId: row.requestId,
|
|
177
|
+
state: "blocked",
|
|
178
|
+
error: row.reason,
|
|
179
|
+
}))
|
|
180
|
+
for (const selected of page.requests) {
|
|
181
|
+
if (options.signal?.aborted) break
|
|
182
|
+
const setup = await deps.cloud.view(selected.requestId)
|
|
183
|
+
if (
|
|
184
|
+
setup.expectedUserId !== CUSTODY_USER_ID ||
|
|
185
|
+
setup.custodyProfile !== options.custodyProfile
|
|
186
|
+
)
|
|
187
|
+
throw new Error("Pending job does not match configured custody identity")
|
|
188
|
+
if (
|
|
189
|
+
!setup.admins.some(
|
|
190
|
+
(admin) => String(admin.userId) === String(REQUIRED_UNIQUE_ADMIN_ID),
|
|
191
|
+
)
|
|
192
|
+
) {
|
|
193
|
+
results.push({
|
|
194
|
+
requestId: setup.requestId,
|
|
195
|
+
state: "blocked",
|
|
196
|
+
error: "required_unique_admin_missing",
|
|
197
|
+
})
|
|
198
|
+
continue
|
|
199
|
+
}
|
|
200
|
+
if (
|
|
201
|
+
!["prepared", "creating", "configuring", "incomplete"].includes(
|
|
202
|
+
setup.state,
|
|
203
|
+
)
|
|
204
|
+
) {
|
|
205
|
+
results.push({ requestId: setup.requestId, state: setup.state })
|
|
206
|
+
continue
|
|
207
|
+
}
|
|
208
|
+
// A permanent/unknown configuration failure needs explicit operator recovery.
|
|
209
|
+
// Only a persisted transient retry time authorizes unattended retry.
|
|
210
|
+
if (setup.state === "incomplete" && !setup.retryAt) {
|
|
211
|
+
results.push({
|
|
212
|
+
requestId: setup.requestId,
|
|
213
|
+
state: "blocked",
|
|
214
|
+
error: "operator_recovery_required",
|
|
215
|
+
})
|
|
216
|
+
continue
|
|
217
|
+
}
|
|
218
|
+
if (
|
|
219
|
+
setup.retryAt &&
|
|
220
|
+
Date.parse(setup.retryAt) > (options.now ?? Date.now())
|
|
221
|
+
) {
|
|
222
|
+
results.push({
|
|
223
|
+
requestId: setup.requestId,
|
|
224
|
+
state: "waiting",
|
|
225
|
+
retryAt: setup.retryAt,
|
|
226
|
+
})
|
|
227
|
+
continue
|
|
228
|
+
}
|
|
229
|
+
if (!options.run) {
|
|
230
|
+
results.push({
|
|
231
|
+
requestId: setup.requestId,
|
|
232
|
+
state: setup.state,
|
|
233
|
+
...(setup.chatId ? { chatId: setup.chatId } : {}),
|
|
234
|
+
})
|
|
235
|
+
continue
|
|
236
|
+
}
|
|
237
|
+
const nowMs = options.now ?? Date.now()
|
|
238
|
+
const hold = setupHoldUntil(
|
|
239
|
+
deps.holds,
|
|
240
|
+
setup.expectedUserId,
|
|
241
|
+
nowMs,
|
|
242
|
+
setupHoldGroups(setup),
|
|
243
|
+
)
|
|
244
|
+
if (hold) {
|
|
245
|
+
results.push(await reportHold(deps.cloud, setup, hold, nowMs))
|
|
246
|
+
continue
|
|
247
|
+
}
|
|
248
|
+
try {
|
|
249
|
+
const result = await deps.telegram(setup, (telegram) =>
|
|
250
|
+
provisionSetup({
|
|
251
|
+
setup,
|
|
252
|
+
telegram,
|
|
253
|
+
cloud: deps.cloud,
|
|
254
|
+
journal: deps.journal,
|
|
255
|
+
workerId: options.workerId,
|
|
256
|
+
}),
|
|
257
|
+
)
|
|
258
|
+
results.push({
|
|
259
|
+
requestId: setup.requestId,
|
|
260
|
+
state: result.state,
|
|
261
|
+
...("chatId" in result && result.chatId
|
|
262
|
+
? { chatId: result.chatId }
|
|
263
|
+
: {}),
|
|
264
|
+
})
|
|
265
|
+
} catch (error) {
|
|
266
|
+
// Get the actual state after an ambiguous response; never retry a creation in this tick.
|
|
267
|
+
const current = await deps.cloud.view(setup.requestId)
|
|
268
|
+
results.push({
|
|
269
|
+
requestId: setup.requestId,
|
|
270
|
+
state: current.state,
|
|
271
|
+
...(current.chatId ? { chatId: current.chatId } : {}),
|
|
272
|
+
retryAt: current.retryAt,
|
|
273
|
+
error: safeProvisionError(error),
|
|
274
|
+
})
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
return { dryRun: !options.run, results, nextAfter: page.nextAfter }
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* Waiting result for a held account. An incomplete setup this worker already
|
|
282
|
+
* holds the claim for gets `cloudRetryAt` of the hold's end as its cloud
|
|
283
|
+
* `retryAt`, through the
|
|
284
|
+
* same `progress` call a flood retry uses, so Deal Desk shows the time. Other
|
|
285
|
+
* setups are not claimed; Deal Desk cannot store a `retryAt` for them.
|
|
286
|
+
*/
|
|
287
|
+
async function reportHold(
|
|
288
|
+
cloud: ProvisionCloud,
|
|
289
|
+
setup: Setup,
|
|
290
|
+
hold: { untilMs: number; code: string },
|
|
291
|
+
nowMs: number,
|
|
292
|
+
): Promise<WorkerResult> {
|
|
293
|
+
// Same value provisioning saves for a hold, so the cloud time never ends
|
|
294
|
+
// before the local hold and the next tick does not report it again.
|
|
295
|
+
const retryAt =
|
|
296
|
+
cloudRetryAt(hold.untilMs, nowMs) ?? new Date(hold.untilMs).toISOString()
|
|
297
|
+
const result: WorkerResult = {
|
|
298
|
+
requestId: setup.requestId,
|
|
299
|
+
state: "waiting",
|
|
300
|
+
...(setup.chatId ? { chatId: setup.chatId } : {}),
|
|
301
|
+
retryAt,
|
|
302
|
+
error: hold.code,
|
|
303
|
+
}
|
|
304
|
+
if (setup.state !== "incomplete" || !setup.claimToken) return result
|
|
305
|
+
try {
|
|
306
|
+
await cloud.progress(setup.requestId, setup.claimToken, {
|
|
307
|
+
state: "incomplete",
|
|
308
|
+
...(setup.chatId ? { chatId: setup.chatId } : {}),
|
|
309
|
+
...(setup.binding ? { binding: setup.binding } : {}),
|
|
310
|
+
error: hold.code,
|
|
311
|
+
retryAt,
|
|
312
|
+
})
|
|
313
|
+
} catch (error) {
|
|
314
|
+
// The local hold still keeps the job waiting; the cloud may lag.
|
|
315
|
+
return { ...result, error: `${hold.code}:${safeProvisionError(error)}` }
|
|
316
|
+
}
|
|
317
|
+
return result
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
export async function runCustodyWorker(options: {
|
|
321
|
+
custodyProfile: string
|
|
322
|
+
once?: boolean
|
|
323
|
+
run?: boolean
|
|
324
|
+
limit?: number
|
|
325
|
+
intervalMs?: number
|
|
326
|
+
signal?: AbortSignal
|
|
327
|
+
/** Process run: worker.tick rows and per-job telemetry. */
|
|
328
|
+
telemetry?: Pick<TelemetryRun, "sink" | "context" | "client">
|
|
329
|
+
}): Promise<void> {
|
|
330
|
+
const interval = options.intervalMs ?? 5000
|
|
331
|
+
if (!Number.isSafeInteger(interval) || interval < 1000 || interval > 60000)
|
|
332
|
+
throw new Error("Worker interval must be 1000 through 60000ms")
|
|
333
|
+
if (!options.once && !options.run)
|
|
334
|
+
throw new Error(
|
|
335
|
+
"Daemon mode requires --run; use --once for read-only preview",
|
|
336
|
+
)
|
|
337
|
+
const root = configDir()
|
|
338
|
+
const workerId = await stableWorkerId(root)
|
|
339
|
+
const telemetry = options.telemetry
|
|
340
|
+
const clientTelemetry = telemetry?.client({
|
|
341
|
+
accountId: CUSTODY_USER_ID,
|
|
342
|
+
profile: options.custodyProfile,
|
|
343
|
+
})
|
|
344
|
+
// Each job's Telegram calls carry its setup ID as job_id.
|
|
345
|
+
const telegram = <T>(
|
|
346
|
+
setup: Setup,
|
|
347
|
+
operation: (adapter: TelegramProvisioner) => Promise<T>,
|
|
348
|
+
): Promise<T> =>
|
|
349
|
+
withTelemetryJob(setup.requestId, () =>
|
|
350
|
+
withProvisionTelegram(setup, operation, clientTelemetry, options.signal),
|
|
351
|
+
)
|
|
352
|
+
let lastTickRecordedMs: number | null = null
|
|
353
|
+
const recordTick = (tick: {
|
|
354
|
+
startedAt: number
|
|
355
|
+
result?: Awaited<ReturnType<typeof runWorkerOnce>>
|
|
356
|
+
error?: unknown
|
|
357
|
+
}) => {
|
|
358
|
+
if (!telemetry) return
|
|
359
|
+
const nowMs = Date.now()
|
|
360
|
+
if (
|
|
361
|
+
!shouldRecordTick({
|
|
362
|
+
...tick,
|
|
363
|
+
nowMs,
|
|
364
|
+
lastRecordedMs: lastTickRecordedMs,
|
|
365
|
+
})
|
|
366
|
+
) {
|
|
367
|
+
return
|
|
368
|
+
}
|
|
369
|
+
lastTickRecordedMs = nowMs
|
|
370
|
+
recordWorkerTick(telemetry, {
|
|
371
|
+
...tick,
|
|
372
|
+
custodyProfile: options.custodyProfile,
|
|
373
|
+
})
|
|
374
|
+
}
|
|
375
|
+
const execute = async () => {
|
|
376
|
+
const cloud = new CliProvisionCloud()
|
|
377
|
+
// Preview never reaches the hold check, so it opens no state file. A
|
|
378
|
+
// run shares the guard's store, which reopens after a failed open.
|
|
379
|
+
const holds: HoldStoreSource = options.run
|
|
380
|
+
? () => processHoldStore()
|
|
381
|
+
: HoldStore.unavailable("preview")
|
|
382
|
+
let after: string | undefined
|
|
383
|
+
try {
|
|
384
|
+
do {
|
|
385
|
+
const tickStartedAt = Date.now()
|
|
386
|
+
try {
|
|
387
|
+
const result = await runWorkerOnce(
|
|
388
|
+
{ ...options, workerId, after },
|
|
389
|
+
{
|
|
390
|
+
cloud,
|
|
391
|
+
journal: new FileProvisionJournal(root),
|
|
392
|
+
holds,
|
|
393
|
+
telegram,
|
|
394
|
+
},
|
|
395
|
+
)
|
|
396
|
+
after = result.nextAfter ?? undefined
|
|
397
|
+
const line = { ...result, observedAt: new Date().toISOString() }
|
|
398
|
+
console.log(JSON.stringify(line))
|
|
399
|
+
recordTick({ startedAt: tickStartedAt, result })
|
|
400
|
+
} catch (error) {
|
|
401
|
+
recordTick({ startedAt: tickStartedAt, error })
|
|
402
|
+
if (options.once) throw error
|
|
403
|
+
const line = {
|
|
404
|
+
state: "worker_tick_failed",
|
|
405
|
+
observedAt: new Date().toISOString(),
|
|
406
|
+
retryInMs: interval,
|
|
407
|
+
}
|
|
408
|
+
console.error(JSON.stringify(line))
|
|
409
|
+
}
|
|
410
|
+
if (options.once || options.signal?.aborted) break
|
|
411
|
+
await new Promise<void>((resolve) => {
|
|
412
|
+
const done = () => {
|
|
413
|
+
clearTimeout(timer)
|
|
414
|
+
options.signal?.removeEventListener("abort", done)
|
|
415
|
+
resolve()
|
|
416
|
+
}
|
|
417
|
+
const timer = setTimeout(done, interval)
|
|
418
|
+
options.signal?.addEventListener("abort", done, { once: true })
|
|
419
|
+
})
|
|
420
|
+
} while (!options.signal?.aborted)
|
|
421
|
+
} finally {
|
|
422
|
+
if (options.run) closeProcessHoldStores()
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
if (!options.run) return execute()
|
|
426
|
+
await withAccountLock(
|
|
427
|
+
join(root, "provisioning", "worker-service.lock"),
|
|
428
|
+
execute,
|
|
429
|
+
)
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
/** Counts results per state, e.g. `{ ready: 2, waiting: 1 }`. */
|
|
433
|
+
export function countStates(results: WorkerResult[]): Record<string, number> {
|
|
434
|
+
const counts: Record<string, number> = {}
|
|
435
|
+
for (const result of results) {
|
|
436
|
+
counts[result.state] = (counts[result.state] ?? 0) + 1
|
|
437
|
+
}
|
|
438
|
+
return counts
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
/** Idle ticks still write a `worker.tick` row this often, as a heartbeat. */
|
|
442
|
+
export const TICK_HEARTBEAT_MS = 5 * 60 * 1000
|
|
443
|
+
|
|
444
|
+
/** Job states that mean the tick did no work. */
|
|
445
|
+
const IDLE_STATES = new Set(["waiting", "blocked"])
|
|
446
|
+
|
|
447
|
+
/**
|
|
448
|
+
* Whether a tick earns a `worker.tick` row: it threw, a job did something
|
|
449
|
+
* (any result not `waiting`/`blocked`), it is the first tick, or the last
|
|
450
|
+
* recorded tick is `TICK_HEARTBEAT_MS` old. Idle polls are otherwise skipped
|
|
451
|
+
* so a 5s interval does not write ~17k rows a day.
|
|
452
|
+
*/
|
|
453
|
+
export function shouldRecordTick(tick: {
|
|
454
|
+
result?: Awaited<ReturnType<typeof runWorkerOnce>>
|
|
455
|
+
error?: unknown
|
|
456
|
+
nowMs: number
|
|
457
|
+
lastRecordedMs: number | null
|
|
458
|
+
}): boolean {
|
|
459
|
+
if (tick.error !== undefined) return true
|
|
460
|
+
if (tick.result?.results.some((r) => !IDLE_STATES.has(r.state))) return true
|
|
461
|
+
if (tick.lastRecordedMs === null) return true
|
|
462
|
+
return tick.nowMs - tick.lastRecordedMs >= TICK_HEARTBEAT_MS
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
/** Writes one `worker.tick` row; never throws. */
|
|
466
|
+
export function recordWorkerTick(
|
|
467
|
+
telemetry: Pick<TelemetryRun, "sink" | "context">,
|
|
468
|
+
tick: {
|
|
469
|
+
startedAt: number
|
|
470
|
+
custodyProfile: string
|
|
471
|
+
result?: Awaited<ReturnType<typeof runWorkerOnce>>
|
|
472
|
+
error?: unknown
|
|
473
|
+
},
|
|
474
|
+
): void {
|
|
475
|
+
try {
|
|
476
|
+
const { context } = telemetry
|
|
477
|
+
telemetry.sink.record({
|
|
478
|
+
runId: context.runId,
|
|
479
|
+
subsystem: context.subsystem,
|
|
480
|
+
command: context.command,
|
|
481
|
+
version: context.version,
|
|
482
|
+
accountId: CUSTODY_USER_ID,
|
|
483
|
+
profile: tick.custodyProfile,
|
|
484
|
+
event: WORKER_TICK_EVENT,
|
|
485
|
+
outcome: tick.error === undefined ? "ok" : "error",
|
|
486
|
+
durationMs: Date.now() - tick.startedAt,
|
|
487
|
+
err:
|
|
488
|
+
tick.error === undefined ? null : safeProvisionError(tick.error),
|
|
489
|
+
res: tick.result
|
|
490
|
+
? {
|
|
491
|
+
dryRun: tick.result.dryRun,
|
|
492
|
+
jobs: tick.result.results.length,
|
|
493
|
+
states: countStates(tick.result.results),
|
|
494
|
+
more: tick.result.nextAfter !== null,
|
|
495
|
+
}
|
|
496
|
+
: null,
|
|
497
|
+
})
|
|
498
|
+
} catch {
|
|
499
|
+
// Telemetry never stops the worker.
|
|
500
|
+
}
|
|
501
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import type { Command } from "commander"
|
|
2
|
+
import { runCustodyWorker } from "./custodyWorker"
|
|
3
|
+
import { activeTelemetryRun } from "./telemetry/run"
|
|
4
|
+
|
|
5
|
+
/** Root main.ts calls this once beside addProvisionCommands. No MCP is involved. */
|
|
6
|
+
export function addCustodyWorkerCommands(
|
|
7
|
+
program: Command,
|
|
8
|
+
run: typeof runCustodyWorker = runCustodyWorker,
|
|
9
|
+
): void {
|
|
10
|
+
program
|
|
11
|
+
.command("worker")
|
|
12
|
+
.description(
|
|
13
|
+
"Run the custody-account provisioning queue; defaults to a read-only one-page preview",
|
|
14
|
+
)
|
|
15
|
+
.requiredOption(
|
|
16
|
+
"--custody-profile <name>",
|
|
17
|
+
"Explicit saved custody profile for user 8274304175",
|
|
18
|
+
)
|
|
19
|
+
.option("--once", "Process one bounded page and exit")
|
|
20
|
+
.option("--run", "Execute approved queued Telegram setup jobs")
|
|
21
|
+
.option("--limit <count>", "Maximum jobs per tick, 1 through 50", "10")
|
|
22
|
+
.option(
|
|
23
|
+
"--interval-ms <ms>",
|
|
24
|
+
"Polling interval, 1000 through 60000",
|
|
25
|
+
"5000",
|
|
26
|
+
)
|
|
27
|
+
.action(
|
|
28
|
+
async (options: {
|
|
29
|
+
custodyProfile: string
|
|
30
|
+
once?: boolean
|
|
31
|
+
run?: boolean
|
|
32
|
+
limit: string
|
|
33
|
+
intervalMs: string
|
|
34
|
+
}) => {
|
|
35
|
+
const controller = new AbortController()
|
|
36
|
+
const telemetry = activeTelemetryRun() ?? undefined
|
|
37
|
+
const stop = () => {
|
|
38
|
+
telemetry?.markAborted()
|
|
39
|
+
controller.abort()
|
|
40
|
+
}
|
|
41
|
+
process.once("SIGTERM", stop)
|
|
42
|
+
process.once("SIGINT", stop)
|
|
43
|
+
try {
|
|
44
|
+
await run({
|
|
45
|
+
...options,
|
|
46
|
+
once: options.once || !options.run,
|
|
47
|
+
limit: Number(options.limit),
|
|
48
|
+
intervalMs: Number(options.intervalMs),
|
|
49
|
+
signal: controller.signal,
|
|
50
|
+
...(telemetry ? { telemetry } : {}),
|
|
51
|
+
})
|
|
52
|
+
} finally {
|
|
53
|
+
process.removeListener("SIGTERM", stop)
|
|
54
|
+
process.removeListener("SIGINT", stop)
|
|
55
|
+
}
|
|
56
|
+
},
|
|
57
|
+
)
|
|
58
|
+
}
|