@c4t4/heyamigo 0.9.3 → 0.9.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/db/schema.js +33 -0
- package/dist/memory/scheduler.js +24 -12
- package/dist/queue/cron-dispatch.js +82 -0
- package/dist/queue/cron-handlers.js +25 -0
- package/dist/queue/crons.js +205 -0
- package/dist/queue/orchestrator.js +17 -0
- package/migrations/0002_phase2_crons.sql +14 -0
- package/migrations/meta/0002_snapshot.json +468 -0
- package/migrations/meta/_journal.json +7 -0
- package/package.json +1 -1
package/dist/db/schema.js
CHANGED
|
@@ -95,3 +95,36 @@ export const outbound = sqliteTable('outbound', {
|
|
|
95
95
|
.on(t.idempotencyKey)
|
|
96
96
|
.where(sql `${t.idempotencyKey} IS NOT NULL`),
|
|
97
97
|
}));
|
|
98
|
+
// ──────────────────────────────────────────────────────────────────
|
|
99
|
+
// Cron table (Phase 2.2)
|
|
100
|
+
// ──────────────────────────────────────────────────────────────────
|
|
101
|
+
// Schedules. Not really a queue — a *table polled by the orchestrator*
|
|
102
|
+
// that fires inserts into the other queues when next_run_at <= now.
|
|
103
|
+
//
|
|
104
|
+
// recurrence:
|
|
105
|
+
// null → one-shot (deleted after firing)
|
|
106
|
+
// '@every <n><unit>' → 'every 30s', 'every 5m', 'every 3h', 'every 7d'
|
|
107
|
+
// '@daily HH:MM' → daily at owner-tz local time
|
|
108
|
+
// '@weekly DOW HH:MM' → weekly at owner-tz local time
|
|
109
|
+
// DOW = mon|tue|wed|thu|fri|sat|sun
|
|
110
|
+
// (Cron expressions intentionally NOT supported until we need them —
|
|
111
|
+
// own@every / @daily / @weekly covers every existing setInterval.)
|
|
112
|
+
export const crons = sqliteTable('crons', {
|
|
113
|
+
id: integer('id').primaryKey({ autoIncrement: true }),
|
|
114
|
+
name: text('name').notNull(), // human-readable; uniqueness enforced for non-oneshot
|
|
115
|
+
enqueueInto: text('enqueue_into').notNull(), // 'inbound'|'async'|'outbound'|'memory_writes'
|
|
116
|
+
payload: text('payload').notNull(), // JSON passed to the target queue
|
|
117
|
+
recurrence: text('recurrence'), // null = one-shot
|
|
118
|
+
nextRunAt: integer('next_run_at').notNull(),
|
|
119
|
+
lastRunAt: integer('last_run_at'),
|
|
120
|
+
enabled: integer('enabled').notNull().default(1), // SQLite bool = int
|
|
121
|
+
createdAt: integer('created_at').notNull(),
|
|
122
|
+
}, t => ({
|
|
123
|
+
byDue: index('crons_by_due').on(t.enabled, t.nextRunAt),
|
|
124
|
+
// Named recurring crons must be unique — prevents accidental
|
|
125
|
+
// duplicate schedules from setup wizard re-runs etc. One-shots
|
|
126
|
+
// (recurrence IS NULL) can have any name.
|
|
127
|
+
uniqName: uniqueIndex('crons_name_uq')
|
|
128
|
+
.on(t.name)
|
|
129
|
+
.where(sql `${t.recurrence} IS NOT NULL`),
|
|
130
|
+
}));
|
package/dist/memory/scheduler.js
CHANGED
|
@@ -2,6 +2,8 @@ import fastq from 'fastq';
|
|
|
2
2
|
import { config } from '../config.js';
|
|
3
3
|
import { logger } from '../logger.js';
|
|
4
4
|
import { prunePrompts } from '../promptlog.js';
|
|
5
|
+
import { registerInternalCronHandler } from '../queue/cron-handlers.js';
|
|
6
|
+
import { deleteCron, enqueueCron } from '../queue/crons.js';
|
|
5
7
|
import { pruneMedia } from '../store/media.js';
|
|
6
8
|
import { runDigest } from './digest.js';
|
|
7
9
|
import { ensureScaffold, getLastDigestedAt, jsonlMtimeFor, loadDigestState, } from './store.js';
|
|
@@ -74,7 +76,6 @@ async function sweep() {
|
|
|
74
76
|
}
|
|
75
77
|
}
|
|
76
78
|
let sweepTimer = null;
|
|
77
|
-
let nudgeTimer = null;
|
|
78
79
|
const NUDGE_TICK_MS = 5 * 60 * 1000; // 5 minutes
|
|
79
80
|
export function startScheduler() {
|
|
80
81
|
if (sweepTimer)
|
|
@@ -95,13 +96,17 @@ export function startScheduler() {
|
|
|
95
96
|
sweepTimer = setInterval(() => {
|
|
96
97
|
void sweep().catch((err) => logger.error({ err }, 'sweep failed'));
|
|
97
98
|
}, config.memory.sweepIntervalMs);
|
|
98
|
-
//
|
|
99
|
-
//
|
|
100
|
-
//
|
|
101
|
-
//
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
99
|
+
// Proactive journal nudges (check-ins, silent-nudges). Migrated from
|
|
100
|
+
// setInterval to a cron row → orchestrator. Same cadence, same body;
|
|
101
|
+
// benefits are: survives restarts, visible in `crons` table, can be
|
|
102
|
+
// paused via control row without code change.
|
|
103
|
+
registerInternalCronHandler('journal-nudge-tick', runNudgeTickSafe);
|
|
104
|
+
enqueueCron({
|
|
105
|
+
name: 'journal-nudge-tick',
|
|
106
|
+
enqueueInto: 'internal',
|
|
107
|
+
payload: { handler: 'journal-nudge-tick' },
|
|
108
|
+
recurrence: `@every ${Math.floor(NUDGE_TICK_MS / 1000)}s`,
|
|
109
|
+
});
|
|
105
110
|
logger.info({
|
|
106
111
|
intervalMs: config.memory.sweepIntervalMs,
|
|
107
112
|
nudgeTickMs: NUDGE_TICK_MS,
|
|
@@ -121,11 +126,18 @@ export function stopScheduler() {
|
|
|
121
126
|
clearInterval(sweepTimer);
|
|
122
127
|
sweepTimer = null;
|
|
123
128
|
}
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
129
|
+
// Nudge cron is owned by the crons table; orchestrator stops on its
|
|
130
|
+
// own. Deleting the cron row here would re-arm itself on next boot,
|
|
131
|
+
// so leave it alone — disabling via the `enabled` column is the
|
|
132
|
+
// user-facing knob.
|
|
128
133
|
for (const t of pendingTimers.values())
|
|
129
134
|
clearTimeout(t);
|
|
130
135
|
pendingTimers.clear();
|
|
131
136
|
}
|
|
137
|
+
// Exported for callers (CLI, /nudge command) that want to surgically
|
|
138
|
+
// disable nudges without editing config. Use `setCronEnabled` from
|
|
139
|
+
// crons.ts for the on/off switch; this is a hard delete (regenerated
|
|
140
|
+
// on next startScheduler call).
|
|
141
|
+
export function deleteNudgeCron() {
|
|
142
|
+
return deleteCron('journal-nudge-tick');
|
|
143
|
+
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
// Maps a fired cron row's payload into the right target queue.
|
|
2
|
+
// Called by the orchestrator each tick for due rows.
|
|
3
|
+
//
|
|
4
|
+
// Currently supports: outbound. inbound, async, memory_writes ride
|
|
5
|
+
// in on later phases — when those queues exist, add their dispatch
|
|
6
|
+
// here and a cron can fire into them with no other changes.
|
|
7
|
+
import { logger } from '../logger.js';
|
|
8
|
+
import { getInternalCronHandler } from './cron-handlers.js';
|
|
9
|
+
import { enqueueOutbound } from './outbound.js';
|
|
10
|
+
export function dispatchCron(row) {
|
|
11
|
+
let payload;
|
|
12
|
+
try {
|
|
13
|
+
payload = JSON.parse(row.payload);
|
|
14
|
+
}
|
|
15
|
+
catch (err) {
|
|
16
|
+
logger.error({ err, id: row.id, name: row.name }, 'cron payload not JSON');
|
|
17
|
+
return;
|
|
18
|
+
}
|
|
19
|
+
switch (row.enqueueInto) {
|
|
20
|
+
case 'outbound':
|
|
21
|
+
dispatchOutbound(row, payload);
|
|
22
|
+
return;
|
|
23
|
+
case 'internal':
|
|
24
|
+
dispatchInternal(row, payload);
|
|
25
|
+
return;
|
|
26
|
+
case 'inbound':
|
|
27
|
+
case 'async':
|
|
28
|
+
case 'memory_writes':
|
|
29
|
+
logger.warn({ name: row.name, target: row.enqueueInto }, 'cron target queue not yet wired (phase pending)');
|
|
30
|
+
return;
|
|
31
|
+
default:
|
|
32
|
+
logger.error({ name: row.name, target: row.enqueueInto }, 'cron has unknown target queue');
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
function dispatchInternal(row, payload) {
|
|
36
|
+
if (!payload || typeof payload !== 'object' || !('handler' in payload)) {
|
|
37
|
+
logger.error({ id: row.id, name: row.name }, "internal cron missing 'handler' in payload");
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
const handlerName = payload.handler;
|
|
41
|
+
if (typeof handlerName !== 'string') {
|
|
42
|
+
logger.error({ id: row.id, name: row.name }, 'internal cron handler name not a string');
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
const handler = getInternalCronHandler(handlerName);
|
|
46
|
+
if (!handler) {
|
|
47
|
+
logger.error({ id: row.id, name: row.name, handler: handlerName }, 'internal cron handler not registered');
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
50
|
+
// Fire-and-forget. Handler errors are caught and logged but the
|
|
51
|
+
// cron row still gets marked fired so we don't stack up retries.
|
|
52
|
+
try {
|
|
53
|
+
const result = handler();
|
|
54
|
+
if (result && typeof result.catch === 'function') {
|
|
55
|
+
;
|
|
56
|
+
result.catch((err) => logger.error({ err, handler: handlerName }, 'internal cron handler rejected'));
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
catch (err) {
|
|
60
|
+
logger.error({ err, handler: handlerName }, 'internal cron handler threw');
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
function dispatchOutbound(row, payload) {
|
|
64
|
+
if (!isOutboundPayload(payload)) {
|
|
65
|
+
logger.error({ id: row.id, payload }, 'cron outbound payload malformed');
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
enqueueOutbound({
|
|
69
|
+
...payload,
|
|
70
|
+
// Recurring crons MUST NOT collide with prior firings; embed the
|
|
71
|
+
// current tick in the idempotency key. One-shots can supply their
|
|
72
|
+
// own.
|
|
73
|
+
idempotencyKey: payload.idempotencyKey ??
|
|
74
|
+
`cron-${row.name}-${Math.floor(Date.now() / 1000)}`,
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
function isOutboundPayload(p) {
|
|
78
|
+
if (!p || typeof p !== 'object')
|
|
79
|
+
return false;
|
|
80
|
+
const o = p;
|
|
81
|
+
return typeof o.address === 'string' && typeof o.kind === 'string';
|
|
82
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
// In-process handler registry for the 'internal' cron target.
|
|
2
|
+
//
|
|
3
|
+
// Not every periodic task fits the queue→worker model. Things like
|
|
4
|
+
// "run the journal observer sweep" or "regenerate compressed memory"
|
|
5
|
+
// are pure in-process work — there's no queue to enqueue into, the
|
|
6
|
+
// orchestrator just needs to call a function.
|
|
7
|
+
//
|
|
8
|
+
// Mechanism: cron rows with enqueue_into='internal' carry a payload
|
|
9
|
+
// with `handler: <name>`. The dispatcher looks up the name in this
|
|
10
|
+
// registry and invokes the function. Registry is populated at boot,
|
|
11
|
+
// before the orchestrator starts polling.
|
|
12
|
+
import { logger } from '../logger.js';
|
|
13
|
+
const registry = new Map();
|
|
14
|
+
export function registerInternalCronHandler(name, handler) {
|
|
15
|
+
if (registry.has(name)) {
|
|
16
|
+
logger.warn({ name }, 'internal cron handler already registered; overwriting');
|
|
17
|
+
}
|
|
18
|
+
registry.set(name, handler);
|
|
19
|
+
}
|
|
20
|
+
export function getInternalCronHandler(name) {
|
|
21
|
+
return registry.get(name);
|
|
22
|
+
}
|
|
23
|
+
export function listInternalCronHandlers() {
|
|
24
|
+
return [...registry.keys()];
|
|
25
|
+
}
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
// Cron table helpers. Orchestrator polls `listDueCrons()` each tick;
|
|
2
|
+
// each due row gets its `payload` enqueued into its target queue,
|
|
3
|
+
// then `markCronFired()` updates `lastRunAt` and recomputes
|
|
4
|
+
// `nextRunAt` (or deletes if one-shot).
|
|
5
|
+
//
|
|
6
|
+
// Recurrence formats supported:
|
|
7
|
+
// '@every <n><unit>' n>0 integer; unit s|m|h|d
|
|
8
|
+
// '@daily HH:MM' owner-tz local
|
|
9
|
+
// '@weekly DOW HH:MM' owner-tz local; DOW = mon..sun
|
|
10
|
+
//
|
|
11
|
+
// No general cron expression parser — every existing setInterval in
|
|
12
|
+
// the bot maps to one of the three forms above. Adding cron parsing
|
|
13
|
+
// is straightforward later if we need it.
|
|
14
|
+
import { and, asc, eq, lte } from 'drizzle-orm';
|
|
15
|
+
import { config } from '../config.js';
|
|
16
|
+
import { getDb } from '../db/index.js';
|
|
17
|
+
import { logger } from '../logger.js';
|
|
18
|
+
import { crons } from '../db/schema.js';
|
|
19
|
+
// Idempotent for named recurring crons: re-inserting the same name
|
|
20
|
+
// updates the recurrence/payload but leaves nextRunAt alone (so we
|
|
21
|
+
// don't reset the firing schedule on every boot).
|
|
22
|
+
export function enqueueCron(input) {
|
|
23
|
+
const db = getDb();
|
|
24
|
+
const now = Math.floor(Date.now() / 1000);
|
|
25
|
+
const enabled = (input.enabled ?? true) ? 1 : 0;
|
|
26
|
+
if (input.recurrence) {
|
|
27
|
+
const existing = db
|
|
28
|
+
.select()
|
|
29
|
+
.from(crons)
|
|
30
|
+
.where(eq(crons.name, input.name))
|
|
31
|
+
.get();
|
|
32
|
+
if (existing) {
|
|
33
|
+
const updated = db
|
|
34
|
+
.update(crons)
|
|
35
|
+
.set({
|
|
36
|
+
enqueueInto: input.enqueueInto,
|
|
37
|
+
payload: JSON.stringify(input.payload),
|
|
38
|
+
recurrence: input.recurrence,
|
|
39
|
+
enabled,
|
|
40
|
+
})
|
|
41
|
+
.where(eq(crons.name, input.name))
|
|
42
|
+
.returning()
|
|
43
|
+
.get();
|
|
44
|
+
return updated;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
const firstRunAt = input.firstRunAt ?? computeNextRun(input.recurrence, now);
|
|
48
|
+
if (firstRunAt === null) {
|
|
49
|
+
throw new Error(`enqueueCron(${input.name}): one-shot requires firstRunAt`);
|
|
50
|
+
}
|
|
51
|
+
return db
|
|
52
|
+
.insert(crons)
|
|
53
|
+
.values({
|
|
54
|
+
name: input.name,
|
|
55
|
+
enqueueInto: input.enqueueInto,
|
|
56
|
+
payload: JSON.stringify(input.payload),
|
|
57
|
+
recurrence: input.recurrence,
|
|
58
|
+
nextRunAt: firstRunAt,
|
|
59
|
+
lastRunAt: null,
|
|
60
|
+
enabled,
|
|
61
|
+
createdAt: now,
|
|
62
|
+
})
|
|
63
|
+
.returning()
|
|
64
|
+
.get();
|
|
65
|
+
}
|
|
66
|
+
export function listDueCrons(asOf = Math.floor(Date.now() / 1000)) {
|
|
67
|
+
const db = getDb();
|
|
68
|
+
return db
|
|
69
|
+
.select()
|
|
70
|
+
.from(crons)
|
|
71
|
+
.where(and(eq(crons.enabled, 1), lte(crons.nextRunAt, asOf)))
|
|
72
|
+
.orderBy(asc(crons.nextRunAt))
|
|
73
|
+
.all();
|
|
74
|
+
}
|
|
75
|
+
// Called by orchestrator after the cron's payload has been enqueued
|
|
76
|
+
// into its target queue. Recurring crons get nextRunAt advanced;
|
|
77
|
+
// one-shots get deleted.
|
|
78
|
+
export function markCronFired(row) {
|
|
79
|
+
const db = getDb();
|
|
80
|
+
const now = Math.floor(Date.now() / 1000);
|
|
81
|
+
if (row.recurrence === null) {
|
|
82
|
+
db.delete(crons).where(eq(crons.id, row.id)).run();
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
const next = computeNextRun(row.recurrence, now);
|
|
86
|
+
if (next === null) {
|
|
87
|
+
logger.error({ id: row.id, name: row.name, recurrence: row.recurrence }, 'cron has unparseable recurrence after firing; disabling');
|
|
88
|
+
db.update(crons)
|
|
89
|
+
.set({ enabled: 0, lastRunAt: now })
|
|
90
|
+
.where(eq(crons.id, row.id))
|
|
91
|
+
.run();
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
db.update(crons)
|
|
95
|
+
.set({ lastRunAt: now, nextRunAt: next })
|
|
96
|
+
.where(eq(crons.id, row.id))
|
|
97
|
+
.run();
|
|
98
|
+
}
|
|
99
|
+
export function deleteCron(name) {
|
|
100
|
+
const db = getDb();
|
|
101
|
+
const result = db
|
|
102
|
+
.delete(crons)
|
|
103
|
+
.where(eq(crons.name, name))
|
|
104
|
+
.returning({ id: crons.id })
|
|
105
|
+
.all();
|
|
106
|
+
return result.length > 0;
|
|
107
|
+
}
|
|
108
|
+
export function setCronEnabled(name, enabled) {
|
|
109
|
+
const db = getDb();
|
|
110
|
+
const result = db
|
|
111
|
+
.update(crons)
|
|
112
|
+
.set({ enabled: enabled ? 1 : 0 })
|
|
113
|
+
.where(eq(crons.name, name))
|
|
114
|
+
.returning({ id: crons.id })
|
|
115
|
+
.all();
|
|
116
|
+
return result.length > 0;
|
|
117
|
+
}
|
|
118
|
+
// ──────────────────────────────────────────────────────────────────
|
|
119
|
+
// Recurrence parser
|
|
120
|
+
// ──────────────────────────────────────────────────────────────────
|
|
121
|
+
const EVERY_RE = /^@every\s+(\d+)\s*([smhd])$/;
|
|
122
|
+
const DAILY_RE = /^@daily\s+(\d{1,2}):(\d{2})$/;
|
|
123
|
+
const WEEKLY_RE = /^@weekly\s+(mon|tue|wed|thu|fri|sat|sun)\s+(\d{1,2}):(\d{2})$/i;
|
|
124
|
+
const UNIT_SECONDS = {
|
|
125
|
+
s: 1, m: 60, h: 3600, d: 86400,
|
|
126
|
+
};
|
|
127
|
+
const DOW_INDEX = {
|
|
128
|
+
sun: 0, mon: 1, tue: 2, wed: 3, thu: 4, fri: 5, sat: 6,
|
|
129
|
+
};
|
|
130
|
+
// Returns the next-run timestamp (unix seconds) for a recurrence, or
|
|
131
|
+
// null for an unparseable format.
|
|
132
|
+
export function computeNextRun(recurrence, nowSec) {
|
|
133
|
+
if (!recurrence)
|
|
134
|
+
return null;
|
|
135
|
+
const everyMatch = EVERY_RE.exec(recurrence);
|
|
136
|
+
if (everyMatch) {
|
|
137
|
+
const n = parseInt(everyMatch[1], 10);
|
|
138
|
+
const unit = everyMatch[2];
|
|
139
|
+
if (n <= 0)
|
|
140
|
+
return null;
|
|
141
|
+
return nowSec + n * (UNIT_SECONDS[unit] ?? 0);
|
|
142
|
+
}
|
|
143
|
+
const dailyMatch = DAILY_RE.exec(recurrence);
|
|
144
|
+
if (dailyMatch) {
|
|
145
|
+
return nextLocalHourMinute(nowSec, parseInt(dailyMatch[1], 10), parseInt(dailyMatch[2], 10), null);
|
|
146
|
+
}
|
|
147
|
+
const weeklyMatch = WEEKLY_RE.exec(recurrence);
|
|
148
|
+
if (weeklyMatch) {
|
|
149
|
+
const dow = DOW_INDEX[weeklyMatch[1].toLowerCase()];
|
|
150
|
+
return nextLocalHourMinute(nowSec, parseInt(weeklyMatch[2], 10), parseInt(weeklyMatch[3], 10), dow);
|
|
151
|
+
}
|
|
152
|
+
return null;
|
|
153
|
+
}
|
|
154
|
+
// Compute the next unix-seconds at HH:MM in the owner timezone,
|
|
155
|
+
// optionally constrained to a day-of-week. Always returns a moment
|
|
156
|
+
// strictly in the future (> nowSec).
|
|
157
|
+
function nextLocalHourMinute(nowSec, hour, minute, dayOfWeek) {
|
|
158
|
+
const tz = config.owner.timezone;
|
|
159
|
+
const fmt = new Intl.DateTimeFormat('en-CA', {
|
|
160
|
+
timeZone: tz,
|
|
161
|
+
year: 'numeric', month: '2-digit', day: '2-digit',
|
|
162
|
+
hour: '2-digit', minute: '2-digit', second: '2-digit',
|
|
163
|
+
weekday: 'short', hour12: false,
|
|
164
|
+
});
|
|
165
|
+
// Walk forward in 1-hour steps from now until we land on the right
|
|
166
|
+
// (day, dow) and the candidate moment is in the future. Cheaper than
|
|
167
|
+
// it sounds — at most 7*24 = 168 iterations.
|
|
168
|
+
for (let step = 0; step < 24 * 8; step++) {
|
|
169
|
+
const candidate = new Date((nowSec + step * 3600) * 1000);
|
|
170
|
+
const parts = fmt.formatToParts(candidate);
|
|
171
|
+
const get = (type) => parts.find((p) => p.type === type)?.value ?? '';
|
|
172
|
+
const cYear = parseInt(get('year'), 10);
|
|
173
|
+
const cMonth = parseInt(get('month'), 10) - 1;
|
|
174
|
+
const cDay = parseInt(get('day'), 10);
|
|
175
|
+
const wdName = get('weekday').toLowerCase();
|
|
176
|
+
const cDow = DOW_INDEX[wdName] ?? -1;
|
|
177
|
+
if (dayOfWeek !== null && cDow !== dayOfWeek)
|
|
178
|
+
continue;
|
|
179
|
+
// Build a UTC instant for HH:MM on (cYear,cMonth,cDay) in the tz.
|
|
180
|
+
// Easier path: format candidate at hour/minute and re-parse via
|
|
181
|
+
// the timezone-offset trick.
|
|
182
|
+
const candidateLocal = makeDateInTz(cYear, cMonth, cDay, hour, minute, tz);
|
|
183
|
+
if (candidateLocal > nowSec)
|
|
184
|
+
return candidateLocal;
|
|
185
|
+
}
|
|
186
|
+
// Fallback (shouldn't be reachable): an hour from now.
|
|
187
|
+
return nowSec + 3600;
|
|
188
|
+
}
|
|
189
|
+
// Build a unix-seconds for a given Y/M/D HH:MM in a named timezone.
|
|
190
|
+
// Done by guess-and-correct: assume the input is UTC, see how the tz
|
|
191
|
+
// renders that instant, take the delta, apply it.
|
|
192
|
+
function makeDateInTz(year, month, day, hour, minute, tz) {
|
|
193
|
+
const guessUtcMs = Date.UTC(year, month, day, hour, minute, 0);
|
|
194
|
+
const fmt = new Intl.DateTimeFormat('en-US', {
|
|
195
|
+
timeZone: tz,
|
|
196
|
+
year: 'numeric', month: '2-digit', day: '2-digit',
|
|
197
|
+
hour: '2-digit', minute: '2-digit', second: '2-digit',
|
|
198
|
+
hour12: false,
|
|
199
|
+
});
|
|
200
|
+
const parts = fmt.formatToParts(new Date(guessUtcMs));
|
|
201
|
+
const get = (type) => parts.find((p) => p.type === type)?.value ?? '0';
|
|
202
|
+
const renderedUtcMs = Date.UTC(parseInt(get('year'), 10), parseInt(get('month'), 10) - 1, parseInt(get('day'), 10), parseInt(get('hour'), 10), parseInt(get('minute'), 10), parseInt(get('second'), 10));
|
|
203
|
+
const offsetMs = guessUtcMs - renderedUtcMs;
|
|
204
|
+
return Math.floor((guessUtcMs + offsetMs) / 1000);
|
|
205
|
+
}
|
|
@@ -18,6 +18,8 @@ import { workers } from '../db/schema.js';
|
|
|
18
18
|
import { logger } from '../logger.js';
|
|
19
19
|
import { reclaimStuckOutbound } from './outbound.js';
|
|
20
20
|
import { clearControl, readControl, requestControl } from './control.js';
|
|
21
|
+
import { listDueCrons, markCronFired } from './crons.js';
|
|
22
|
+
import { dispatchCron } from './cron-dispatch.js';
|
|
21
23
|
const TICK_INTERVAL_MS = 500;
|
|
22
24
|
const HEARTBEAT_INTERVAL_MS = 5_000;
|
|
23
25
|
const WORKER_DEAD_AFTER_SECONDS = 30;
|
|
@@ -100,6 +102,21 @@ async function tick(id) {
|
|
|
100
102
|
if (reclaimed > 0) {
|
|
101
103
|
logger.info({ reclaimed }, 'reclaimed stuck outbound rows');
|
|
102
104
|
}
|
|
105
|
+
// Fire any due crons. Order: dispatch each in turn; if dispatch
|
|
106
|
+
// throws (it shouldn't — dispatch swallows), the cron is NOT
|
|
107
|
+
// marked fired and we'll retry on the next tick.
|
|
108
|
+
if (!draining) {
|
|
109
|
+
const due = listDueCrons();
|
|
110
|
+
for (const row of due) {
|
|
111
|
+
try {
|
|
112
|
+
dispatchCron(row);
|
|
113
|
+
markCronFired(row);
|
|
114
|
+
}
|
|
115
|
+
catch (err) {
|
|
116
|
+
logger.error({ err, name: row.name }, 'cron dispatch crashed');
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
103
120
|
markDeadWorkers();
|
|
104
121
|
if (draining) {
|
|
105
122
|
const busy = busyWorkerCount();
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
CREATE TABLE `crons` (
|
|
2
|
+
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
|
3
|
+
`name` text NOT NULL,
|
|
4
|
+
`enqueue_into` text NOT NULL,
|
|
5
|
+
`payload` text NOT NULL,
|
|
6
|
+
`recurrence` text,
|
|
7
|
+
`next_run_at` integer NOT NULL,
|
|
8
|
+
`last_run_at` integer,
|
|
9
|
+
`enabled` integer DEFAULT 1 NOT NULL,
|
|
10
|
+
`created_at` integer NOT NULL
|
|
11
|
+
);
|
|
12
|
+
--> statement-breakpoint
|
|
13
|
+
CREATE INDEX `crons_by_due` ON `crons` (`enabled`,`next_run_at`);--> statement-breakpoint
|
|
14
|
+
CREATE UNIQUE INDEX `crons_name_uq` ON `crons` (`name`) WHERE "crons"."recurrence" IS NOT NULL;
|
|
@@ -0,0 +1,468 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": "6",
|
|
3
|
+
"dialect": "sqlite",
|
|
4
|
+
"id": "417b2708-624e-416f-af59-bee5d5e38556",
|
|
5
|
+
"prevId": "56b9231e-919c-4b8d-9a77-68735c33cbb0",
|
|
6
|
+
"tables": {
|
|
7
|
+
"control": {
|
|
8
|
+
"name": "control",
|
|
9
|
+
"columns": {
|
|
10
|
+
"key": {
|
|
11
|
+
"name": "key",
|
|
12
|
+
"type": "text",
|
|
13
|
+
"primaryKey": true,
|
|
14
|
+
"notNull": true,
|
|
15
|
+
"autoincrement": false
|
|
16
|
+
},
|
|
17
|
+
"value": {
|
|
18
|
+
"name": "value",
|
|
19
|
+
"type": "text",
|
|
20
|
+
"primaryKey": false,
|
|
21
|
+
"notNull": false,
|
|
22
|
+
"autoincrement": false
|
|
23
|
+
},
|
|
24
|
+
"requested_by": {
|
|
25
|
+
"name": "requested_by",
|
|
26
|
+
"type": "text",
|
|
27
|
+
"primaryKey": false,
|
|
28
|
+
"notNull": false,
|
|
29
|
+
"autoincrement": false
|
|
30
|
+
},
|
|
31
|
+
"requested_at": {
|
|
32
|
+
"name": "requested_at",
|
|
33
|
+
"type": "integer",
|
|
34
|
+
"primaryKey": false,
|
|
35
|
+
"notNull": true,
|
|
36
|
+
"autoincrement": false
|
|
37
|
+
}
|
|
38
|
+
},
|
|
39
|
+
"indexes": {},
|
|
40
|
+
"foreignKeys": {},
|
|
41
|
+
"compositePrimaryKeys": {},
|
|
42
|
+
"uniqueConstraints": {},
|
|
43
|
+
"checkConstraints": {}
|
|
44
|
+
},
|
|
45
|
+
"crons": {
|
|
46
|
+
"name": "crons",
|
|
47
|
+
"columns": {
|
|
48
|
+
"id": {
|
|
49
|
+
"name": "id",
|
|
50
|
+
"type": "integer",
|
|
51
|
+
"primaryKey": true,
|
|
52
|
+
"notNull": true,
|
|
53
|
+
"autoincrement": true
|
|
54
|
+
},
|
|
55
|
+
"name": {
|
|
56
|
+
"name": "name",
|
|
57
|
+
"type": "text",
|
|
58
|
+
"primaryKey": false,
|
|
59
|
+
"notNull": true,
|
|
60
|
+
"autoincrement": false
|
|
61
|
+
},
|
|
62
|
+
"enqueue_into": {
|
|
63
|
+
"name": "enqueue_into",
|
|
64
|
+
"type": "text",
|
|
65
|
+
"primaryKey": false,
|
|
66
|
+
"notNull": true,
|
|
67
|
+
"autoincrement": false
|
|
68
|
+
},
|
|
69
|
+
"payload": {
|
|
70
|
+
"name": "payload",
|
|
71
|
+
"type": "text",
|
|
72
|
+
"primaryKey": false,
|
|
73
|
+
"notNull": true,
|
|
74
|
+
"autoincrement": false
|
|
75
|
+
},
|
|
76
|
+
"recurrence": {
|
|
77
|
+
"name": "recurrence",
|
|
78
|
+
"type": "text",
|
|
79
|
+
"primaryKey": false,
|
|
80
|
+
"notNull": false,
|
|
81
|
+
"autoincrement": false
|
|
82
|
+
},
|
|
83
|
+
"next_run_at": {
|
|
84
|
+
"name": "next_run_at",
|
|
85
|
+
"type": "integer",
|
|
86
|
+
"primaryKey": false,
|
|
87
|
+
"notNull": true,
|
|
88
|
+
"autoincrement": false
|
|
89
|
+
},
|
|
90
|
+
"last_run_at": {
|
|
91
|
+
"name": "last_run_at",
|
|
92
|
+
"type": "integer",
|
|
93
|
+
"primaryKey": false,
|
|
94
|
+
"notNull": false,
|
|
95
|
+
"autoincrement": false
|
|
96
|
+
},
|
|
97
|
+
"enabled": {
|
|
98
|
+
"name": "enabled",
|
|
99
|
+
"type": "integer",
|
|
100
|
+
"primaryKey": false,
|
|
101
|
+
"notNull": true,
|
|
102
|
+
"autoincrement": false,
|
|
103
|
+
"default": 1
|
|
104
|
+
},
|
|
105
|
+
"created_at": {
|
|
106
|
+
"name": "created_at",
|
|
107
|
+
"type": "integer",
|
|
108
|
+
"primaryKey": false,
|
|
109
|
+
"notNull": true,
|
|
110
|
+
"autoincrement": false
|
|
111
|
+
}
|
|
112
|
+
},
|
|
113
|
+
"indexes": {
|
|
114
|
+
"crons_by_due": {
|
|
115
|
+
"name": "crons_by_due",
|
|
116
|
+
"columns": [
|
|
117
|
+
"enabled",
|
|
118
|
+
"next_run_at"
|
|
119
|
+
],
|
|
120
|
+
"isUnique": false
|
|
121
|
+
},
|
|
122
|
+
"crons_name_uq": {
|
|
123
|
+
"name": "crons_name_uq",
|
|
124
|
+
"columns": [
|
|
125
|
+
"name"
|
|
126
|
+
],
|
|
127
|
+
"isUnique": true,
|
|
128
|
+
"where": "\"crons\".\"recurrence\" IS NOT NULL"
|
|
129
|
+
}
|
|
130
|
+
},
|
|
131
|
+
"foreignKeys": {},
|
|
132
|
+
"compositePrimaryKeys": {},
|
|
133
|
+
"uniqueConstraints": {},
|
|
134
|
+
"checkConstraints": {}
|
|
135
|
+
},
|
|
136
|
+
"identities": {
|
|
137
|
+
"name": "identities",
|
|
138
|
+
"columns": {
|
|
139
|
+
"person_id": {
|
|
140
|
+
"name": "person_id",
|
|
141
|
+
"type": "text",
|
|
142
|
+
"primaryKey": false,
|
|
143
|
+
"notNull": true,
|
|
144
|
+
"autoincrement": false
|
|
145
|
+
},
|
|
146
|
+
"address": {
|
|
147
|
+
"name": "address",
|
|
148
|
+
"type": "text",
|
|
149
|
+
"primaryKey": false,
|
|
150
|
+
"notNull": true,
|
|
151
|
+
"autoincrement": false
|
|
152
|
+
},
|
|
153
|
+
"added_at": {
|
|
154
|
+
"name": "added_at",
|
|
155
|
+
"type": "integer",
|
|
156
|
+
"primaryKey": false,
|
|
157
|
+
"notNull": true,
|
|
158
|
+
"autoincrement": false
|
|
159
|
+
}
|
|
160
|
+
},
|
|
161
|
+
"indexes": {
|
|
162
|
+
"identities_address_unique": {
|
|
163
|
+
"name": "identities_address_unique",
|
|
164
|
+
"columns": [
|
|
165
|
+
"address"
|
|
166
|
+
],
|
|
167
|
+
"isUnique": true
|
|
168
|
+
}
|
|
169
|
+
},
|
|
170
|
+
"foreignKeys": {
|
|
171
|
+
"identities_person_id_persons_id_fk": {
|
|
172
|
+
"name": "identities_person_id_persons_id_fk",
|
|
173
|
+
"tableFrom": "identities",
|
|
174
|
+
"tableTo": "persons",
|
|
175
|
+
"columnsFrom": [
|
|
176
|
+
"person_id"
|
|
177
|
+
],
|
|
178
|
+
"columnsTo": [
|
|
179
|
+
"id"
|
|
180
|
+
],
|
|
181
|
+
"onDelete": "no action",
|
|
182
|
+
"onUpdate": "no action"
|
|
183
|
+
}
|
|
184
|
+
},
|
|
185
|
+
"compositePrimaryKeys": {
|
|
186
|
+
"identities_person_id_address_pk": {
|
|
187
|
+
"columns": [
|
|
188
|
+
"person_id",
|
|
189
|
+
"address"
|
|
190
|
+
],
|
|
191
|
+
"name": "identities_person_id_address_pk"
|
|
192
|
+
}
|
|
193
|
+
},
|
|
194
|
+
"uniqueConstraints": {},
|
|
195
|
+
"checkConstraints": {}
|
|
196
|
+
},
|
|
197
|
+
"outbound": {
|
|
198
|
+
"name": "outbound",
|
|
199
|
+
"columns": {
|
|
200
|
+
"id": {
|
|
201
|
+
"name": "id",
|
|
202
|
+
"type": "integer",
|
|
203
|
+
"primaryKey": true,
|
|
204
|
+
"notNull": true,
|
|
205
|
+
"autoincrement": true
|
|
206
|
+
},
|
|
207
|
+
"address": {
|
|
208
|
+
"name": "address",
|
|
209
|
+
"type": "text",
|
|
210
|
+
"primaryKey": false,
|
|
211
|
+
"notNull": true,
|
|
212
|
+
"autoincrement": false
|
|
213
|
+
},
|
|
214
|
+
"kind": {
|
|
215
|
+
"name": "kind",
|
|
216
|
+
"type": "text",
|
|
217
|
+
"primaryKey": false,
|
|
218
|
+
"notNull": true,
|
|
219
|
+
"autoincrement": false
|
|
220
|
+
},
|
|
221
|
+
"text": {
|
|
222
|
+
"name": "text",
|
|
223
|
+
"type": "text",
|
|
224
|
+
"primaryKey": false,
|
|
225
|
+
"notNull": false,
|
|
226
|
+
"autoincrement": false
|
|
227
|
+
},
|
|
228
|
+
"media_path": {
|
|
229
|
+
"name": "media_path",
|
|
230
|
+
"type": "text",
|
|
231
|
+
"primaryKey": false,
|
|
232
|
+
"notNull": false,
|
|
233
|
+
"autoincrement": false
|
|
234
|
+
},
|
|
235
|
+
"media_mime": {
|
|
236
|
+
"name": "media_mime",
|
|
237
|
+
"type": "text",
|
|
238
|
+
"primaryKey": false,
|
|
239
|
+
"notNull": false,
|
|
240
|
+
"autoincrement": false
|
|
241
|
+
},
|
|
242
|
+
"media_bytes": {
|
|
243
|
+
"name": "media_bytes",
|
|
244
|
+
"type": "integer",
|
|
245
|
+
"primaryKey": false,
|
|
246
|
+
"notNull": false,
|
|
247
|
+
"autoincrement": false
|
|
248
|
+
},
|
|
249
|
+
"quote_msg_id": {
|
|
250
|
+
"name": "quote_msg_id",
|
|
251
|
+
"type": "text",
|
|
252
|
+
"primaryKey": false,
|
|
253
|
+
"notNull": false,
|
|
254
|
+
"autoincrement": false
|
|
255
|
+
},
|
|
256
|
+
"idempotency_key": {
|
|
257
|
+
"name": "idempotency_key",
|
|
258
|
+
"type": "text",
|
|
259
|
+
"primaryKey": false,
|
|
260
|
+
"notNull": false,
|
|
261
|
+
"autoincrement": false
|
|
262
|
+
},
|
|
263
|
+
"status": {
|
|
264
|
+
"name": "status",
|
|
265
|
+
"type": "text",
|
|
266
|
+
"primaryKey": false,
|
|
267
|
+
"notNull": true,
|
|
268
|
+
"autoincrement": false
|
|
269
|
+
},
|
|
270
|
+
"attempts": {
|
|
271
|
+
"name": "attempts",
|
|
272
|
+
"type": "integer",
|
|
273
|
+
"primaryKey": false,
|
|
274
|
+
"notNull": true,
|
|
275
|
+
"autoincrement": false,
|
|
276
|
+
"default": 0
|
|
277
|
+
},
|
|
278
|
+
"next_attempt_at": {
|
|
279
|
+
"name": "next_attempt_at",
|
|
280
|
+
"type": "integer",
|
|
281
|
+
"primaryKey": false,
|
|
282
|
+
"notNull": false,
|
|
283
|
+
"autoincrement": false
|
|
284
|
+
},
|
|
285
|
+
"last_error": {
|
|
286
|
+
"name": "last_error",
|
|
287
|
+
"type": "text",
|
|
288
|
+
"primaryKey": false,
|
|
289
|
+
"notNull": false,
|
|
290
|
+
"autoincrement": false
|
|
291
|
+
},
|
|
292
|
+
"claimed_by": {
|
|
293
|
+
"name": "claimed_by",
|
|
294
|
+
"type": "text",
|
|
295
|
+
"primaryKey": false,
|
|
296
|
+
"notNull": false,
|
|
297
|
+
"autoincrement": false
|
|
298
|
+
},
|
|
299
|
+
"claimed_at": {
|
|
300
|
+
"name": "claimed_at",
|
|
301
|
+
"type": "integer",
|
|
302
|
+
"primaryKey": false,
|
|
303
|
+
"notNull": false,
|
|
304
|
+
"autoincrement": false
|
|
305
|
+
},
|
|
306
|
+
"created_at": {
|
|
307
|
+
"name": "created_at",
|
|
308
|
+
"type": "integer",
|
|
309
|
+
"primaryKey": false,
|
|
310
|
+
"notNull": true,
|
|
311
|
+
"autoincrement": false
|
|
312
|
+
},
|
|
313
|
+
"updated_at": {
|
|
314
|
+
"name": "updated_at",
|
|
315
|
+
"type": "integer",
|
|
316
|
+
"primaryKey": false,
|
|
317
|
+
"notNull": true,
|
|
318
|
+
"autoincrement": false
|
|
319
|
+
}
|
|
320
|
+
},
|
|
321
|
+
"indexes": {
|
|
322
|
+
"outbound_by_status_next": {
|
|
323
|
+
"name": "outbound_by_status_next",
|
|
324
|
+
"columns": [
|
|
325
|
+
"status",
|
|
326
|
+
"next_attempt_at"
|
|
327
|
+
],
|
|
328
|
+
"isUnique": false
|
|
329
|
+
},
|
|
330
|
+
"outbound_by_address": {
|
|
331
|
+
"name": "outbound_by_address",
|
|
332
|
+
"columns": [
|
|
333
|
+
"address"
|
|
334
|
+
],
|
|
335
|
+
"isUnique": false
|
|
336
|
+
},
|
|
337
|
+
"outbound_idempotency_key_uq": {
|
|
338
|
+
"name": "outbound_idempotency_key_uq",
|
|
339
|
+
"columns": [
|
|
340
|
+
"idempotency_key"
|
|
341
|
+
],
|
|
342
|
+
"isUnique": true,
|
|
343
|
+
"where": "\"outbound\".\"idempotency_key\" IS NOT NULL"
|
|
344
|
+
}
|
|
345
|
+
},
|
|
346
|
+
"foreignKeys": {},
|
|
347
|
+
"compositePrimaryKeys": {},
|
|
348
|
+
"uniqueConstraints": {},
|
|
349
|
+
"checkConstraints": {}
|
|
350
|
+
},
|
|
351
|
+
"persons": {
|
|
352
|
+
"name": "persons",
|
|
353
|
+
"columns": {
|
|
354
|
+
"id": {
|
|
355
|
+
"name": "id",
|
|
356
|
+
"type": "text",
|
|
357
|
+
"primaryKey": true,
|
|
358
|
+
"notNull": true,
|
|
359
|
+
"autoincrement": false
|
|
360
|
+
},
|
|
361
|
+
"display_name": {
|
|
362
|
+
"name": "display_name",
|
|
363
|
+
"type": "text",
|
|
364
|
+
"primaryKey": false,
|
|
365
|
+
"notNull": false,
|
|
366
|
+
"autoincrement": false
|
|
367
|
+
},
|
|
368
|
+
"timezone": {
|
|
369
|
+
"name": "timezone",
|
|
370
|
+
"type": "text",
|
|
371
|
+
"primaryKey": false,
|
|
372
|
+
"notNull": false,
|
|
373
|
+
"autoincrement": false
|
|
374
|
+
},
|
|
375
|
+
"created_at": {
|
|
376
|
+
"name": "created_at",
|
|
377
|
+
"type": "integer",
|
|
378
|
+
"primaryKey": false,
|
|
379
|
+
"notNull": true,
|
|
380
|
+
"autoincrement": false
|
|
381
|
+
}
|
|
382
|
+
},
|
|
383
|
+
"indexes": {},
|
|
384
|
+
"foreignKeys": {},
|
|
385
|
+
"compositePrimaryKeys": {},
|
|
386
|
+
"uniqueConstraints": {},
|
|
387
|
+
"checkConstraints": {}
|
|
388
|
+
},
|
|
389
|
+
"workers": {
|
|
390
|
+
"name": "workers",
|
|
391
|
+
"columns": {
|
|
392
|
+
"id": {
|
|
393
|
+
"name": "id",
|
|
394
|
+
"type": "text",
|
|
395
|
+
"primaryKey": true,
|
|
396
|
+
"notNull": true,
|
|
397
|
+
"autoincrement": false
|
|
398
|
+
},
|
|
399
|
+
"kind": {
|
|
400
|
+
"name": "kind",
|
|
401
|
+
"type": "text",
|
|
402
|
+
"primaryKey": false,
|
|
403
|
+
"notNull": true,
|
|
404
|
+
"autoincrement": false
|
|
405
|
+
},
|
|
406
|
+
"status": {
|
|
407
|
+
"name": "status",
|
|
408
|
+
"type": "text",
|
|
409
|
+
"primaryKey": false,
|
|
410
|
+
"notNull": true,
|
|
411
|
+
"autoincrement": false
|
|
412
|
+
},
|
|
413
|
+
"current_job": {
|
|
414
|
+
"name": "current_job",
|
|
415
|
+
"type": "text",
|
|
416
|
+
"primaryKey": false,
|
|
417
|
+
"notNull": false,
|
|
418
|
+
"autoincrement": false
|
|
419
|
+
},
|
|
420
|
+
"last_seen": {
|
|
421
|
+
"name": "last_seen",
|
|
422
|
+
"type": "integer",
|
|
423
|
+
"primaryKey": false,
|
|
424
|
+
"notNull": true,
|
|
425
|
+
"autoincrement": false
|
|
426
|
+
},
|
|
427
|
+
"started_at": {
|
|
428
|
+
"name": "started_at",
|
|
429
|
+
"type": "integer",
|
|
430
|
+
"primaryKey": false,
|
|
431
|
+
"notNull": true,
|
|
432
|
+
"autoincrement": false
|
|
433
|
+
}
|
|
434
|
+
},
|
|
435
|
+
"indexes": {
|
|
436
|
+
"workers_by_kind_status": {
|
|
437
|
+
"name": "workers_by_kind_status",
|
|
438
|
+
"columns": [
|
|
439
|
+
"kind",
|
|
440
|
+
"status"
|
|
441
|
+
],
|
|
442
|
+
"isUnique": false
|
|
443
|
+
},
|
|
444
|
+
"workers_by_last_seen": {
|
|
445
|
+
"name": "workers_by_last_seen",
|
|
446
|
+
"columns": [
|
|
447
|
+
"last_seen"
|
|
448
|
+
],
|
|
449
|
+
"isUnique": false
|
|
450
|
+
}
|
|
451
|
+
},
|
|
452
|
+
"foreignKeys": {},
|
|
453
|
+
"compositePrimaryKeys": {},
|
|
454
|
+
"uniqueConstraints": {},
|
|
455
|
+
"checkConstraints": {}
|
|
456
|
+
}
|
|
457
|
+
},
|
|
458
|
+
"views": {},
|
|
459
|
+
"enums": {},
|
|
460
|
+
"_meta": {
|
|
461
|
+
"schemas": {},
|
|
462
|
+
"tables": {},
|
|
463
|
+
"columns": {}
|
|
464
|
+
},
|
|
465
|
+
"internal": {
|
|
466
|
+
"indexes": {}
|
|
467
|
+
}
|
|
468
|
+
}
|