doc2vec 2.10.5 → 2.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,139 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || (function () {
19
+ var ownKeys = function(o) {
20
+ ownKeys = Object.getOwnPropertyNames || function (o) {
21
+ var ar = [];
22
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
+ return ar;
24
+ };
25
+ return ownKeys(o);
26
+ };
27
+ return function (mod) {
28
+ if (mod && mod.__esModule) return mod;
29
+ var result = {};
30
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
+ __setModuleDefault(result, mod);
32
+ return result;
33
+ };
34
+ })();
35
+ Object.defineProperty(exports, "__esModule", { value: true });
36
+ exports.startController = startController;
37
+ const fs = __importStar(require("fs"));
38
+ const path = __importStar(require("path"));
39
+ const logger_1 = require("../logger");
40
+ const config_registry_1 = require("./config-registry");
41
+ const events_1 = require("./events");
42
+ const job_runner_1 = require("./job-runner");
43
+ const notifier_1 = require("./notifier");
44
+ const scheduler_1 = require("./scheduler");
45
+ const server_1 = require("./server");
46
+ const store_1 = require("./store");
47
+ const LOG_PRUNE_INTERVAL_MS = 24 * 60 * 60 * 1000;
48
+ async function startController(opts) {
49
+ const logger = new logger_1.Logger('Controller', {
50
+ level: logger_1.LogLevel.INFO,
51
+ useTimestamp: true,
52
+ useColor: true,
53
+ prettyPrint: true,
54
+ });
55
+ if (!opts.databaseUrl) {
56
+ throw new Error('controller mode requires Postgres: pass --database-url or set DATABASE_URL');
57
+ }
58
+ if (opts.readWrite && !opts.configDir) {
59
+ throw new Error('--read-write requires --config-dir (where new configs are written)');
60
+ }
61
+ if (opts.configArgs.length === 0 && !(opts.readWrite && opts.configDir)) {
62
+ throw new Error('provide at least one config file or directory');
63
+ }
64
+ if (opts.configDir) {
65
+ fs.mkdirSync(path.resolve(opts.configDir), { recursive: true });
66
+ }
67
+ logger.section('DOC2VEC CONTROLLER');
68
+ logger.info(`Mode: ${opts.readWrite ? 'read-write' : 'read-only'}, max parallel jobs: ${opts.maxParallel}`);
69
+ const store = new store_1.ControllerStore(opts.databaseUrl, logger.child('store'));
70
+ await store.init();
71
+ const events = new events_1.ControllerEvents();
72
+ const runner = new job_runner_1.JobRunner(store, events, { maxParallel: Math.max(opts.maxParallel, 1) }, logger.child('runner'));
73
+ const registry = new config_registry_1.ConfigRegistry({
74
+ configArgs: opts.configArgs,
75
+ configDir: opts.configDir,
76
+ readWrite: opts.readWrite,
77
+ reloadIntervalSec: opts.reloadIntervalSec,
78
+ }, store, events, logger.child('registry'));
79
+ const scheduler = new scheduler_1.Scheduler(configId => {
80
+ const config = registry.get(configId);
81
+ if (!config)
82
+ return;
83
+ runner.enqueue(config, 'scheduled').catch(err => logger.error(`Failed to enqueue scheduled run for '${config.name}':`, err));
84
+ }, logger.child('scheduler'));
85
+ events.on('config:update', () => scheduler.sync(registry.list()));
86
+ if (opts.slackWebhookUrl) {
87
+ new notifier_1.SlackNotifier({
88
+ webhookUrl: opts.slackWebhookUrl,
89
+ notify: opts.slackNotify ?? 'all',
90
+ publicUrl: opts.publicUrl,
91
+ }, store, logger.child('slack')).attach(events);
92
+ }
93
+ await registry.start();
94
+ scheduler.sync(registry.list());
95
+ const app = (0, server_1.createServer)({
96
+ store,
97
+ registry,
98
+ scheduler,
99
+ runner,
100
+ events,
101
+ readWrite: opts.readWrite,
102
+ logger: logger.child('api'),
103
+ });
104
+ const server = app.listen(opts.port, () => {
105
+ logger.info(`API and UI listening on http://localhost:${opts.port}`);
106
+ });
107
+ const pruneLogs = () => {
108
+ store.pruneOldLogs(opts.logRetentionDays)
109
+ .then(count => { if (count > 0)
110
+ logger.info(`Pruned ${count} log row(s) older than ${opts.logRetentionDays} days`); })
111
+ .catch(err => logger.error('Log retention pruning failed:', err));
112
+ };
113
+ pruneLogs();
114
+ const pruneTimer = setInterval(pruneLogs, LOG_PRUNE_INTERVAL_MS);
115
+ pruneTimer.unref();
116
+ let shuttingDown = false;
117
+ const shutdown = (signal) => {
118
+ if (shuttingDown)
119
+ return;
120
+ shuttingDown = true;
121
+ logger.info(`Received ${signal}, shutting down...`);
122
+ clearInterval(pruneTimer);
123
+ scheduler.stop();
124
+ registry.stop();
125
+ server.close();
126
+ runner.shutdown()
127
+ .then(() => store.close())
128
+ .then(() => {
129
+ logger.info('Shutdown complete');
130
+ process.exit(0);
131
+ })
132
+ .catch(err => {
133
+ logger.error('Error during shutdown:', err);
134
+ process.exit(1);
135
+ });
136
+ };
137
+ process.on('SIGTERM', () => shutdown('SIGTERM'));
138
+ process.on('SIGINT', () => shutdown('SIGINT'));
139
+ }
@@ -0,0 +1,314 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || (function () {
19
+ var ownKeys = function(o) {
20
+ ownKeys = Object.getOwnPropertyNames || function (o) {
21
+ var ar = [];
22
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
+ return ar;
24
+ };
25
+ return ownKeys(o);
26
+ };
27
+ return function (mod) {
28
+ if (mod && mod.__esModule) return mod;
29
+ var result = {};
30
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
+ __setModuleDefault(result, mod);
32
+ return result;
33
+ };
34
+ })();
35
+ Object.defineProperty(exports, "__esModule", { value: true });
36
+ exports.JobRunner = void 0;
37
+ const child_process_1 = require("child_process");
38
+ const path = __importStar(require("path"));
39
+ const readline = __importStar(require("readline"));
40
+ const types_1 = require("./types");
41
+ const MAX_LOG_MESSAGE_BYTES = 8192;
42
+ const LOG_FLUSH_LINES = 100;
43
+ const LOG_FLUSH_MS = 500;
44
+ const CANCEL_KILL_GRACE_MS = 10000;
45
+ const SHUTDOWN_GRACE_MS = 25000;
46
+ /**
47
+ * Runs sync jobs as child processes with a global concurrency cap and a
48
+ * per-config lock (a config never has two simultaneous runs). Captures the
49
+ * child's structured log stream into Postgres and fans it out over SSE.
50
+ */
51
+ class JobRunner {
52
+ constructor(store, events, opts, logger) {
53
+ this.store = store;
54
+ this.events = events;
55
+ this.opts = opts;
56
+ this.logger = logger;
57
+ this.queue = [];
58
+ this.active = new Map(); // by runId
59
+ this.queuedConfigs = new Set();
60
+ this.activeConfigs = new Set();
61
+ this.shuttingDown = false;
62
+ }
63
+ isBusy(configId) {
64
+ return this.queuedConfigs.has(configId) || this.activeConfigs.has(configId);
65
+ }
66
+ runningCount() {
67
+ return this.active.size;
68
+ }
69
+ async enqueue(config, trigger) {
70
+ if (this.shuttingDown) {
71
+ throw new types_1.ConflictError('controller is shutting down');
72
+ }
73
+ if (this.isBusy(config.id)) {
74
+ if (trigger === 'scheduled') {
75
+ // Overlapping scheduled run: record it as skipped so the history shows why nothing happened
76
+ const run = await this.store.createRun(config.id, config.content_hash, trigger, 'skipped', 'previous run still queued or running');
77
+ this.events.emitRunUpdate(run);
78
+ this.logger.warn(`Skipped scheduled run for '${config.name}': previous run still in progress`);
79
+ return run;
80
+ }
81
+ throw new types_1.ConflictError('a run for this config is already queued or running');
82
+ }
83
+ if (config.parse_error) {
84
+ throw new types_1.ConflictError(`config is invalid: ${config.parse_error}`);
85
+ }
86
+ const run = await this.store.createRun(config.id, config.content_hash, trigger, 'queued');
87
+ this.queuedConfigs.add(config.id);
88
+ this.queue.push({ runId: run.id, configId: config.id, configPath: config.path });
89
+ this.events.emitRunUpdate(run);
90
+ this.pump();
91
+ return run;
92
+ }
93
+ pump() {
94
+ while (!this.shuttingDown && this.active.size < this.opts.maxParallel && this.queue.length > 0) {
95
+ const job = this.queue.shift();
96
+ this.queuedConfigs.delete(job.configId);
97
+ this.activeConfigs.add(job.configId);
98
+ this.start(job).catch(async (err) => {
99
+ this.logger.error(`Failed to start run ${job.runId}:`, err);
100
+ this.activeConfigs.delete(job.configId);
101
+ this.active.delete(job.runId);
102
+ const run = await this.store.finishRun(job.runId, {
103
+ status: 'failed',
104
+ error: `failed to spawn job: ${err instanceof Error ? err.message : String(err)}`,
105
+ });
106
+ this.events.emitRunUpdate(run);
107
+ this.pump();
108
+ });
109
+ }
110
+ }
111
+ commandFor(configPath) {
112
+ if (this.opts.commandFor)
113
+ return this.opts.commandFor(configPath);
114
+ // dist/controller/job-runner.js → dist/doc2vec.js
115
+ const scriptPath = path.resolve(__dirname, '..', 'doc2vec.js');
116
+ return { cmd: process.execPath, args: [scriptPath, 'run', configPath] };
117
+ }
118
+ async start(job) {
119
+ const { cmd, args } = this.commandFor(job.configPath);
120
+ this.logger.info(`Starting run ${job.runId}: ${cmd} ${args.join(' ')}`);
121
+ const child = (0, child_process_1.spawn)(cmd, args, {
122
+ env: { ...process.env, DOC2VEC_STRUCTURED_LOGS: '1' },
123
+ stdio: ['ignore', 'pipe', 'pipe'],
124
+ });
125
+ const activeJob = {
126
+ runId: job.runId,
127
+ configId: job.configId,
128
+ child,
129
+ seq: 0,
130
+ buffer: [],
131
+ flushTimer: null,
132
+ warnCount: 0,
133
+ errorCount: 0,
134
+ sources: null,
135
+ cancelReason: null,
136
+ killTimer: null,
137
+ };
138
+ this.active.set(job.runId, activeJob);
139
+ const run = await this.store.markRunStarted(job.runId, child.pid);
140
+ this.events.emitRunUpdate(run);
141
+ readline.createInterface({ input: child.stdout }).on('line', line => this.handleLine(activeJob, line, false));
142
+ readline.createInterface({ input: child.stderr }).on('line', line => this.handleLine(activeJob, line, true));
143
+ child.on('error', err => {
144
+ this.appendLog(activeJob, 'error', null, `spawn error: ${err.message}`);
145
+ });
146
+ child.on('close', (code, signal) => {
147
+ this.finish(activeJob, code, signal).catch(err => this.logger.error(`Failed to finalize run ${job.runId}:`, err));
148
+ });
149
+ }
150
+ handleLine(aj, raw, fromStderr) {
151
+ if (!raw.trim())
152
+ return;
153
+ let level = fromStderr ? 'warn' : 'info';
154
+ let module = null;
155
+ let message = raw;
156
+ let ts = new Date().toISOString();
157
+ try {
158
+ const obj = JSON.parse(raw);
159
+ if (obj && typeof obj === 'object') {
160
+ ts = obj.ts || ts;
161
+ module = obj.module ?? null;
162
+ if (obj.event === 'run-summary') {
163
+ aj.sources = Array.isArray(obj.sources) ? obj.sources : null;
164
+ return;
165
+ }
166
+ else if (obj.event === 'progress') {
167
+ message = `${obj.title}: ${obj.current}/${obj.total}${obj.message ? ` — ${obj.message}` : ''}`;
168
+ level = 'info';
169
+ }
170
+ else if (obj.event === 'section') {
171
+ message = `═══ ${obj.title} ═══`;
172
+ level = 'info';
173
+ }
174
+ else if (obj.msg !== undefined) {
175
+ message = String(obj.msg);
176
+ level = String(obj.level || level);
177
+ }
178
+ }
179
+ }
180
+ catch {
181
+ // Not JSON (e.g. Chromium noise on stderr) — keep the raw line
182
+ }
183
+ if (level === 'warn')
184
+ aj.warnCount++;
185
+ if (level === 'error')
186
+ aj.errorCount++;
187
+ this.appendLog(aj, level, module, message, ts);
188
+ }
189
+ appendLog(aj, level, module, message, ts = new Date().toISOString()) {
190
+ aj.buffer.push({
191
+ seq: ++aj.seq,
192
+ ts,
193
+ level,
194
+ module,
195
+ message: Buffer.byteLength(message) > MAX_LOG_MESSAGE_BYTES
196
+ ? message.slice(0, MAX_LOG_MESSAGE_BYTES) + '… [truncated]'
197
+ : message,
198
+ });
199
+ if (aj.buffer.length >= LOG_FLUSH_LINES) {
200
+ void this.flush(aj);
201
+ }
202
+ else if (!aj.flushTimer) {
203
+ aj.flushTimer = setTimeout(() => void this.flush(aj), LOG_FLUSH_MS);
204
+ }
205
+ }
206
+ async flush(aj) {
207
+ if (aj.flushTimer) {
208
+ clearTimeout(aj.flushTimer);
209
+ aj.flushTimer = null;
210
+ }
211
+ if (aj.buffer.length === 0)
212
+ return;
213
+ const rows = aj.buffer.splice(0, aj.buffer.length);
214
+ this.events.emitRunLogs(aj.runId, rows);
215
+ try {
216
+ await this.store.insertLogs(aj.runId, rows);
217
+ }
218
+ catch (err) {
219
+ this.logger.error(`Failed to persist ${rows.length} log line(s) for run ${aj.runId}:`, err);
220
+ }
221
+ }
222
+ async finish(aj, code, signal) {
223
+ if (aj.killTimer)
224
+ clearTimeout(aj.killTimer);
225
+ await this.flush(aj);
226
+ let status;
227
+ let error = null;
228
+ if (aj.cancelReason === 'user') {
229
+ status = 'canceled';
230
+ }
231
+ else if (aj.cancelReason === 'shutdown') {
232
+ status = 'failed';
233
+ error = 'controller shutdown';
234
+ }
235
+ else if (code === 0) {
236
+ status = 'succeeded';
237
+ }
238
+ else {
239
+ status = 'failed';
240
+ error = signal ? `terminated by signal ${signal}` : `exited with code ${code}`;
241
+ const failedSources = aj.sources?.filter(s => !s.ok) ?? [];
242
+ if (failedSources.length > 0) {
243
+ error = `${failedSources.length} source(s) failed: ${failedSources.map(s => s.product_name).join(', ')}`;
244
+ }
245
+ }
246
+ const run = await this.store.finishRun(aj.runId, {
247
+ status,
248
+ exitCode: code,
249
+ error,
250
+ stats: {
251
+ ...(aj.sources && { sources: aj.sources }),
252
+ warn_count: aj.warnCount,
253
+ error_count: aj.errorCount,
254
+ },
255
+ });
256
+ this.active.delete(aj.runId);
257
+ this.activeConfigs.delete(aj.configId);
258
+ this.logger.info(`Run ${aj.runId} finished: ${status}${error ? ` (${error})` : ''}`);
259
+ this.events.emitRunUpdate(run);
260
+ this.pump();
261
+ }
262
+ async cancel(runId) {
263
+ const queuedIndex = this.queue.findIndex(job => job.runId === runId);
264
+ if (queuedIndex >= 0) {
265
+ const [job] = this.queue.splice(queuedIndex, 1);
266
+ this.queuedConfigs.delete(job.configId);
267
+ const run = await this.store.finishRun(runId, { status: 'canceled', error: 'canceled while queued' });
268
+ this.events.emitRunUpdate(run);
269
+ return run;
270
+ }
271
+ const aj = this.active.get(runId);
272
+ if (!aj) {
273
+ throw new types_1.ConflictError('run is not queued or running');
274
+ }
275
+ aj.cancelReason = 'user';
276
+ aj.child.kill('SIGTERM');
277
+ aj.killTimer = setTimeout(() => {
278
+ if (this.active.has(runId))
279
+ aj.child.kill('SIGKILL');
280
+ }, CANCEL_KILL_GRACE_MS);
281
+ aj.killTimer.unref();
282
+ const run = await this.store.getRun(runId);
283
+ return run;
284
+ }
285
+ async shutdown() {
286
+ this.shuttingDown = true;
287
+ const queued = this.queue.splice(0, this.queue.length);
288
+ this.queuedConfigs.clear();
289
+ for (const job of queued) {
290
+ const run = await this.store.finishRun(job.runId, { status: 'canceled', error: 'controller shutdown' });
291
+ this.events.emitRunUpdate(run);
292
+ }
293
+ if (this.active.size === 0)
294
+ return;
295
+ this.logger.info(`Waiting for ${this.active.size} running job(s) to terminate...`);
296
+ for (const aj of this.active.values()) {
297
+ aj.cancelReason = aj.cancelReason ?? 'shutdown';
298
+ aj.child.kill('SIGTERM');
299
+ }
300
+ const deadline = Date.now() + SHUTDOWN_GRACE_MS;
301
+ while (this.active.size > 0 && Date.now() < deadline) {
302
+ await new Promise(resolve => setTimeout(resolve, 250));
303
+ }
304
+ for (const aj of this.active.values()) {
305
+ this.logger.warn(`Run ${aj.runId} did not exit in time, sending SIGKILL`);
306
+ aj.child.kill('SIGKILL');
307
+ }
308
+ const killDeadline = Date.now() + 3000;
309
+ while (this.active.size > 0 && Date.now() < killDeadline) {
310
+ await new Promise(resolve => setTimeout(resolve, 100));
311
+ }
312
+ }
313
+ }
314
+ exports.JobRunner = JobRunner;
@@ -0,0 +1,55 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.MIGRATIONS = void 0;
4
+ /**
5
+ * Ordered schema migrations for the controller's Postgres database.
6
+ * Applied versions are tracked in d2v_schema_migrations; never edit an entry
7
+ * after it has shipped — append a new one instead.
8
+ */
9
+ exports.MIGRATIONS = [
10
+ // 1: initial schema
11
+ `
12
+ CREATE TABLE IF NOT EXISTS d2v_configs (
13
+ id SERIAL PRIMARY KEY,
14
+ path TEXT NOT NULL UNIQUE,
15
+ name TEXT NOT NULL,
16
+ content TEXT NOT NULL,
17
+ content_hash TEXT NOT NULL,
18
+ schedule TEXT,
19
+ source_summary JSONB NOT NULL DEFAULT '[]',
20
+ parse_error TEXT,
21
+ enabled BOOLEAN NOT NULL DEFAULT TRUE,
22
+ deleted_at TIMESTAMPTZ,
23
+ created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
24
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
25
+ );
26
+
27
+ CREATE TABLE IF NOT EXISTS d2v_runs (
28
+ id SERIAL PRIMARY KEY,
29
+ config_id INT NOT NULL REFERENCES d2v_configs(id),
30
+ config_hash TEXT NOT NULL,
31
+ trigger TEXT NOT NULL CHECK (trigger IN ('scheduled','manual')),
32
+ status TEXT NOT NULL CHECK (status IN ('queued','running','succeeded','failed','skipped','canceled')),
33
+ pid INT,
34
+ exit_code INT,
35
+ error TEXT,
36
+ stats JSONB NOT NULL DEFAULT '{}',
37
+ queued_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
38
+ started_at TIMESTAMPTZ,
39
+ finished_at TIMESTAMPTZ
40
+ );
41
+
42
+ CREATE INDEX IF NOT EXISTS d2v_runs_config_queued_idx ON d2v_runs (config_id, queued_at DESC);
43
+ CREATE INDEX IF NOT EXISTS d2v_runs_status_idx ON d2v_runs (status);
44
+
45
+ CREATE TABLE IF NOT EXISTS d2v_run_logs (
46
+ run_id INT NOT NULL REFERENCES d2v_runs(id) ON DELETE CASCADE,
47
+ seq BIGINT NOT NULL,
48
+ ts TIMESTAMPTZ NOT NULL,
49
+ level TEXT NOT NULL,
50
+ module TEXT,
51
+ message TEXT NOT NULL,
52
+ PRIMARY KEY (run_id, seq)
53
+ );
54
+ `,
55
+ ];
@@ -0,0 +1,106 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.SlackNotifier = void 0;
4
+ exports.buildRunMessage = buildRunMessage;
5
+ // Terminal statuses worth a notification. 'skipped' (overlapping schedule)
6
+ // is deliberately excluded — it would be pure noise on busy schedules.
7
+ const NOTIFIED_STATUSES = ['succeeded', 'failed', 'canceled'];
8
+ const STATUS_DECOR = {
9
+ succeeded: { emoji: '✅', verb: 'succeeded' },
10
+ failed: { emoji: '❌', verb: 'failed' },
11
+ canceled: { emoji: '⚠️', verb: 'was canceled' },
12
+ };
13
+ function formatDuration(run) {
14
+ if (!run.started_at || !run.finished_at)
15
+ return null;
16
+ const seconds = (new Date(run.finished_at).getTime() - new Date(run.started_at).getTime()) / 1000;
17
+ if (seconds < 60)
18
+ return `${Math.round(seconds)}s`;
19
+ const minutes = Math.floor(seconds / 60);
20
+ if (minutes < 60)
21
+ return `${minutes}m ${Math.round(seconds % 60)}s`;
22
+ return `${Math.floor(minutes / 60)}h ${minutes % 60}m`;
23
+ }
24
+ /** Builds the Slack webhook payload (Block Kit) for a finished run. */
25
+ function buildRunMessage(run, configName, publicUrl) {
26
+ const decor = STATUS_DECOR[run.status] ?? { emoji: 'ℹ️', verb: run.status };
27
+ const sources = run.stats?.sources ?? [];
28
+ const failedSources = sources.filter(s => !s.ok);
29
+ let headline = `${decor.emoji} doc2vec sync *${configName}* ${decor.verb}`;
30
+ if (publicUrl) {
31
+ headline += ` — <${publicUrl.replace(/\/$/, '')}/runs/${run.id}|view run #${run.id}>`;
32
+ }
33
+ else {
34
+ headline += ` (run #${run.id})`;
35
+ }
36
+ const lines = [];
37
+ if (sources.length > 0) {
38
+ lines.push(`${sources.length - failedSources.length}/${sources.length} sources ok`);
39
+ }
40
+ if (failedSources.length > 0) {
41
+ const shown = failedSources.slice(0, 5)
42
+ .map(s => `• *${s.product_name}*: ${s.error ?? 'failed'}`);
43
+ if (failedSources.length > 5)
44
+ shown.push(`• …and ${failedSources.length - 5} more`);
45
+ lines.push(shown.join('\n'));
46
+ }
47
+ if (run.error && failedSources.length === 0) {
48
+ lines.push(run.error);
49
+ }
50
+ const meta = [`trigger: ${run.trigger}`];
51
+ const duration = formatDuration(run);
52
+ if (duration)
53
+ meta.push(`duration: ${duration}`);
54
+ if (run.stats?.warn_count)
55
+ meta.push(`warnings: ${run.stats.warn_count}`);
56
+ if (run.stats?.error_count)
57
+ meta.push(`errors: ${run.stats.error_count}`);
58
+ return {
59
+ text: `doc2vec sync ${configName} ${decor.verb}`, // fallback for notifications
60
+ blocks: [
61
+ {
62
+ type: 'section',
63
+ text: { type: 'mrkdwn', text: [headline, ...lines].join('\n') },
64
+ },
65
+ {
66
+ type: 'context',
67
+ elements: [{ type: 'mrkdwn', text: meta.join(' · ') }],
68
+ },
69
+ ],
70
+ };
71
+ }
72
+ /**
73
+ * Posts a Slack message (incoming webhook) whenever a run reaches a terminal
74
+ * status. Subscribes to the same event bus that feeds the UI, so every path
75
+ * that finishes a run — success, failure, cancel — is covered.
76
+ */
77
+ class SlackNotifier {
78
+ constructor(opts, store, logger) {
79
+ this.opts = opts;
80
+ this.store = store;
81
+ this.logger = logger;
82
+ }
83
+ attach(events) {
84
+ events.on('run:update', (run) => {
85
+ this.maybeNotify(run).catch(err => this.logger.error(`Failed to send Slack notification for run ${run.id}:`, err));
86
+ });
87
+ this.logger.info(`Slack notifications enabled (${this.opts.notify})`);
88
+ }
89
+ async maybeNotify(run) {
90
+ if (!NOTIFIED_STATUSES.includes(run.status))
91
+ return;
92
+ if (this.opts.notify === 'failures' && run.status === 'succeeded')
93
+ return;
94
+ const config = await this.store.getConfig(run.config_id);
95
+ const payload = buildRunMessage(run, config?.name ?? `config ${run.config_id}`, this.opts.publicUrl);
96
+ const response = await fetch(this.opts.webhookUrl, {
97
+ method: 'POST',
98
+ headers: { 'Content-Type': 'application/json' },
99
+ body: JSON.stringify(payload),
100
+ });
101
+ if (!response.ok) {
102
+ this.logger.warn(`Slack webhook returned ${response.status}: ${await response.text().catch(() => '')}`);
103
+ }
104
+ }
105
+ }
106
+ exports.SlackNotifier = SlackNotifier;
@@ -0,0 +1,54 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.Scheduler = void 0;
4
+ const croner_1 = require("croner");
5
+ /**
6
+ * One croner Cron per config that has a valid schedule. Reconciled against the
7
+ * registry's current config list whenever configs change on disk.
8
+ */
9
+ class Scheduler {
10
+ constructor(onTrigger, logger) {
11
+ this.onTrigger = onTrigger;
12
+ this.logger = logger;
13
+ this.crons = new Map();
14
+ }
15
+ sync(configs) {
16
+ const seen = new Set();
17
+ for (const config of configs) {
18
+ if (!config.schedule || config.parse_error || !config.enabled || config.deleted_at)
19
+ continue;
20
+ seen.add(config.id);
21
+ const existing = this.crons.get(config.id);
22
+ if (existing && existing.getPattern() === config.schedule)
23
+ continue;
24
+ existing?.stop();
25
+ try {
26
+ const cron = new croner_1.Cron(config.schedule, { name: `config-${config.id}` }, () => {
27
+ this.onTrigger(config.id);
28
+ });
29
+ this.crons.set(config.id, cron);
30
+ this.logger.info(`Scheduled '${config.name}' (${config.schedule}), next run ${cron.nextRun()?.toISOString() ?? 'never'}`);
31
+ }
32
+ catch (err) {
33
+ this.crons.delete(config.id);
34
+ this.logger.warn(`Failed to schedule '${config.name}' with '${config.schedule}':`, err);
35
+ }
36
+ }
37
+ for (const [id, cron] of this.crons) {
38
+ if (!seen.has(id)) {
39
+ cron.stop();
40
+ this.crons.delete(id);
41
+ this.logger.info(`Unscheduled config ${id}`);
42
+ }
43
+ }
44
+ }
45
+ nextRun(configId) {
46
+ return this.crons.get(configId)?.nextRun()?.toISOString() ?? null;
47
+ }
48
+ stop() {
49
+ for (const cron of this.crons.values())
50
+ cron.stop();
51
+ this.crons.clear();
52
+ }
53
+ }
54
+ exports.Scheduler = Scheduler;