doc2vec 2.10.5 → 2.11.0

package/dist/controller/server.js

@@ -0,0 +1,288 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createServer = createServer;
+const express_1 = __importDefault(require("express"));
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const config_registry_1 = require("./config-registry");
+const types_1 = require("./types");
+const SSE_HEARTBEAT_MS = 15000;
+function getVersion() {
+    try {
+        return require('../../package.json').version;
+    }
+    catch {
+        return 'unknown';
+    }
+}
+function sseInit(res) {
+    res.writeHead(200, {
+        'Content-Type': 'text/event-stream',
+        'Cache-Control': 'no-cache',
+        'Connection': 'keep-alive',
+        'X-Accel-Buffering': 'no',
+    });
+    res.write(':ok\n\n');
+}
+function sseSend(res, event, data) {
+    res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
+}
+function createServer(deps) {
+    const { store, registry, scheduler, runner, events, readWrite, logger } = deps;
+    const app = (0, express_1.default)();
+    app.use(express_1.default.json({ limit: '2mb' }));
+    const requireReadWrite = (_req, res, next) => {
+        if (!readWrite) {
+            res.status(403).json({ error: 'controller is running in read-only mode' });
+            return;
+        }
+        next();
+    };
+    const parseId = (raw) => {
+        const id = Number(raw);
+        if (!Number.isInteger(id) || id <= 0)
+            throw new types_1.ValidationError('invalid id');
+        return id;
+    };
+    const enrichConfig = (config, lastRuns) => ({
+        ...config,
+        next_run: scheduler.nextRun(config.id),
+        last_run: lastRuns.get(config.id) ?? null,
+        busy: runner.isBusy(config.id),
+    });
+    // ------------------------------------------------------------------ meta
+    app.get('/api/health', (_req, res) => {
+        res.json({ status: 'ok', mode: readWrite ? 'rw' : 'ro', version: getVersion() });
+    });
+    // ------------------------------------------------------------------ configs
+    app.get('/api/configs', async (_req, res) => {
+        const lastRuns = await store.getLastRuns();
+        res.json(registry.list().map(c => enrichConfig(c, lastRuns)));
+    });
+    app.get('/api/configs/:id', async (req, res) => {
+        const config = registry.get(parseId(String(req.params.id)));
+        if (!config)
+            throw new types_1.NotFoundError('config not found');
+        const lastRuns = await store.getLastRuns();
+        res.json(enrichConfig(config, lastRuns));
+    });
+    app.post('/api/configs', requireReadWrite, async (req, res) => {
+        const { filename, content } = req.body ?? {};
+        if (typeof filename !== 'string' || typeof content !== 'string') {
+            throw new types_1.ValidationError('body must contain filename and content strings');
+        }
+        const record = await registry.create(filename, content);
+        res.status(201).json(record);
+    });
+    app.put('/api/configs/:id', requireReadWrite, async (req, res) => {
+        const { content, baseHash } = req.body ?? {};
+        if (typeof content !== 'string' || typeof baseHash !== 'string') {
+            throw new types_1.ValidationError('body must contain content and baseHash strings');
+        }
+        const record = await registry.update(parseId(String(req.params.id)), content, baseHash);
+        res.json(record);
+    });
+    app.delete('/api/configs/:id', requireReadWrite, async (req, res) => {
+        await registry.delete(parseId(String(req.params.id)));
+        res.status(204).end();
+    });
+    app.post('/api/configs/validate', (req, res) => {
+        const { content } = req.body ?? {};
+        if (typeof content !== 'string') {
+            throw new types_1.ValidationError('body must contain a content string');
+        }
+        const meta = (0, config_registry_1.parseConfigMeta)(content, 'config.yaml');
+        res.json({
+            valid: !meta.parseError,
+            error: meta.parseError,
+            name: meta.name,
+            schedule: meta.schedule,
+            schedule_valid: meta.schedule ? (0, config_registry_1.isValidCron)(meta.schedule) : null,
+            sources: meta.sourceSummary,
+        });
+    });
+    app.post('/api/configs/:id/run', async (req, res) => {
+        const config = registry.get(parseId(String(req.params.id)));
+        if (!config)
+            throw new types_1.NotFoundError('config not found');
+        const run = await runner.enqueue(config, 'manual');
+        res.status(202).json(run);
+    });
+    app.get('/api/configs/:id/stats', async (req, res) => {
+        const id = parseId(String(req.params.id));
+        if (!registry.get(id))
+            throw new types_1.NotFoundError('config not found');
+        const days = Number(req.query.days) || 30;
+        res.json(await store.getConfigStats(id, days));
+    });
+    // ------------------------------------------------------------------ runs
+    app.get('/api/runs', async (req, res) => {
+        res.json(await store.listRuns({
+            configId: req.query.configId !== undefined ? parseId(String(req.query.configId)) : undefined,
+            status: req.query.status !== undefined ? String(req.query.status) : undefined,
+            limit: req.query.limit !== undefined ? Number(req.query.limit) : undefined,
+            before: req.query.before !== undefined ? Number(req.query.before) : undefined,
+        }));
+    });
+    app.get('/api/runs/:id', async (req, res) => {
+        const run = await store.getRun(parseId(String(req.params.id)));
+        if (!run)
+            throw new types_1.NotFoundError('run not found');
+        const config = registry.get(run.config_id) ?? await store.getConfig(run.config_id);
+        res.json({ ...run, config_name: config?.name ?? null });
+    });
+    app.get('/api/runs/:id/logs', async (req, res) => {
+        const id = parseId(String(req.params.id));
+        if (!await store.getRun(id))
+            throw new types_1.NotFoundError('run not found');
+        const afterSeq = Number(req.query.afterSeq) || 0;
+        const limit = Number(req.query.limit) || 1000;
+        res.json(await store.getLogs(id, afterSeq, limit));
+    });
+    app.post('/api/runs/:id/cancel', async (req, res) => {
+        res.json(await runner.cancel(parseId(String(req.params.id))));
+    });
+    // ------------------------------------------------------------------ SSE
+    app.get('/api/events', (req, res) => {
+        sseInit(res);
+        const onRunUpdate = (run) => sseSend(res, 'run:update', run);
+        const onConfigUpdate = () => sseSend(res, 'config:update', {});
+        events.on('run:update', onRunUpdate);
+        events.on('config:update', onConfigUpdate);
+        const heartbeat = setInterval(() => res.write(':hb\n\n'), SSE_HEARTBEAT_MS);
+        req.on('close', () => {
+            clearInterval(heartbeat);
+            events.off('run:update', onRunUpdate);
+            events.off('config:update', onConfigUpdate);
+        });
+    });
+    app.get('/api/runs/:id/logs/stream', async (req, res) => {
+        const id = parseId(String(req.params.id));
+        const run = await store.getRun(id);
+        if (!run)
+            throw new types_1.NotFoundError('run not found');
+        sseInit(res);
+        let lastSeq = Number(req.query.afterSeq) || 0;
+        let replayDone = false;
+        const pending = [];
+        const sendRow = (row) => {
+            if (row.seq > lastSeq) {
+                lastSeq = row.seq;
+                sseSend(res, 'log', row);
+            }
+        };
+        const onLog = (payload) => {
+            if (payload.runId !== id)
+                return;
+            if (!replayDone) {
+                pending.push(...payload.lines);
+                return;
+            }
+            payload.lines.forEach(sendRow);
+        };
+        const onRunUpdate = (updated) => {
+            if (updated.id !== id)
+                return;
+            sseSend(res, 'run:update', updated);
+            if (['succeeded', 'failed', 'canceled', 'skipped'].includes(updated.status)) {
+                sseSend(res, 'end', { status: updated.status });
+            }
+        };
+        events.on('run:log', onLog);
+        events.on('run:update', onRunUpdate);
+        // Replay history from the DB, then flush anything that streamed in meanwhile
+        try {
+            let batch;
+            do {
+                batch = await store.getLogs(id, lastSeq, 5000);
+                batch.forEach(sendRow);
+            } while (batch.length === 5000);
+        }
+        finally {
+            replayDone = true;
+            pending.forEach(sendRow);
+            pending.length = 0;
+        }
+        const current = await store.getRun(id);
+        if (current && ['succeeded', 'failed', 'canceled', 'skipped'].includes(current.status)) {
+            sseSend(res, 'end', { status: current.status });
+        }
+        const heartbeat = setInterval(() => res.write(':hb\n\n'), SSE_HEARTBEAT_MS);
+        req.on('close', () => {
+            clearInterval(heartbeat);
+            events.off('run:log', onLog);
+            events.off('run:update', onRunUpdate);
+        });
+    });
+    // ------------------------------------------------------------------ static UI
+    const uiDir = path.resolve(__dirname, '..', 'ui');
+    if (fs.existsSync(path.join(uiDir, 'index.html'))) {
+        app.use(express_1.default.static(uiDir));
+        // SPA fallback for client-side routes (anything that's not /api)
+        app.use((req, res, next) => {
+            if (req.method === 'GET' && !req.path.startsWith('/api')) {
+                res.sendFile(path.join(uiDir, 'index.html'));
+                return;
+            }
+            next();
+        });
+    }
+    else {
+        logger.warn(`UI assets not found at ${uiDir} — API only (build them with 'npm run build:ui')`);
+    }
+    // ------------------------------------------------------------------ errors
+    app.use((err, _req, res, _next) => {
+        if (res.headersSent)
+            return;
+        if (err instanceof types_1.ValidationError) {
+            res.status(400).json({ error: err.message });
+        }
+        else if (err instanceof types_1.NotFoundError) {
+            res.status(404).json({ error: err.message });
+        }
+        else if (err instanceof types_1.ConflictError) {
+            res.status(409).json({ error: err.message });
+        }
+        else {
+            logger.error('Unhandled API error:', err);
+            res.status(500).json({ error: 'internal error' });
+        }
+    });
+    return app;
+}

package/dist/controller/store.js

@@ -0,0 +1,186 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ControllerStore = void 0;
+const pg_1 = require("pg");
+const migrations_1 = require("./migrations");
+/**
+ * All Postgres access for controller mode: configs, runs, and run logs.
+ */
+class ControllerStore {
+    constructor(databaseUrl, logger) {
+        this.pool = new pg_1.Pool({ connectionString: databaseUrl, max: 10 });
+        this.logger = logger;
+    }
+    async init() {
+        await this.migrate();
+        await this.markOrphanedRuns();
+    }
+    async close() {
+        await this.pool.end();
+    }
+    async migrate() {
+        const client = await this.pool.connect();
+        try {
+            await client.query(`CREATE TABLE IF NOT EXISTS d2v_schema_migrations (
+                version INT PRIMARY KEY,
+                applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+            )`);
+            const { rows } = await client.query('SELECT COALESCE(MAX(version), 0) AS version FROM d2v_schema_migrations');
+            const current = Number(rows[0].version);
+            for (let i = current; i < migrations_1.MIGRATIONS.length; i++) {
+                const version = i + 1;
+                this.logger.info(`Applying migration ${version}`);
+                await client.query('BEGIN');
+                try {
+                    await client.query(migrations_1.MIGRATIONS[i]);
+                    await client.query('INSERT INTO d2v_schema_migrations (version) VALUES ($1)', [version]);
+                    await client.query('COMMIT');
+                }
+                catch (err) {
+                    await client.query('ROLLBACK');
+                    throw err;
+                }
+            }
+        }
+        finally {
+            client.release();
+        }
+    }
+    /** Runs left 'queued'/'running' by a previous controller process can never finish — fail them. */
+    async markOrphanedRuns() {
+        const { rowCount } = await this.pool.query(`UPDATE d2v_runs
+             SET status = 'failed', error = 'controller restarted while run was in progress', finished_at = NOW()
+             WHERE status IN ('queued', 'running')`);
+        if (rowCount) {
+            this.logger.warn(`Marked ${rowCount} orphaned run(s) from a previous controller process as failed`);
+        }
+    }
+    // ------------------------------------------------------------------ configs
+    async upsertConfig(input) {
+        const { rows } = await this.pool.query(`INSERT INTO d2v_configs (path, name, content, content_hash, schedule, source_summary, parse_error)
+             VALUES ($1, $2, $3, $4, $5, $6, $7)
+             ON CONFLICT (path) DO UPDATE SET
+                name = EXCLUDED.name,
+                content = EXCLUDED.content,
+                content_hash = EXCLUDED.content_hash,
+                schedule = EXCLUDED.schedule,
+                source_summary = EXCLUDED.source_summary,
+                parse_error = EXCLUDED.parse_error,
+                deleted_at = NULL,
+                updated_at = NOW()
+             RETURNING *`, [input.path, input.name, input.content, input.contentHash, input.schedule,
+            JSON.stringify(input.sourceSummary), input.parseError]);
+        return rows[0];
+    }
+    async markConfigDeleted(path) {
+        await this.pool.query('UPDATE d2v_configs SET deleted_at = NOW(), updated_at = NOW() WHERE path = $1', [path]);
+    }
+    async listConfigs(includeDeleted = false) {
+        const { rows } = await this.pool.query(`SELECT * FROM d2v_configs ${includeDeleted ? '' : 'WHERE deleted_at IS NULL'} ORDER BY name`);
+        return rows;
+    }
+    async getConfig(id) {
+        const { rows } = await this.pool.query('SELECT * FROM d2v_configs WHERE id = $1', [id]);
+        return rows[0] ?? null;
+    }
+    // ------------------------------------------------------------------ runs
+    async createRun(configId, configHash, trigger, status, error) {
+        const terminal = status === 'skipped';
+        const { rows } = await this.pool.query(`INSERT INTO d2v_runs (config_id, config_hash, trigger, status, error, finished_at)
+             VALUES ($1, $2, $3, $4, $5, ${terminal ? 'NOW()' : 'NULL'})
+             RETURNING *`, [configId, configHash, trigger, status, error ?? null]);
+        return rows[0];
+    }
+    async markRunStarted(runId, pid) {
+        const { rows } = await this.pool.query(`UPDATE d2v_runs SET status = 'running', pid = $2, started_at = NOW() WHERE id = $1 RETURNING *`, [runId, pid ?? null]);
+        return rows[0];
+    }
+    async finishRun(runId, outcome) {
+        const { rows } = await this.pool.query(`UPDATE d2v_runs
+             SET status = $2, exit_code = $3, error = $4, stats = $5, finished_at = NOW()
+             WHERE id = $1 RETURNING *`, [runId, outcome.status, outcome.exitCode ?? null, outcome.error ?? null, JSON.stringify(outcome.stats ?? {})]);
+        return rows[0];
+    }
+    async getRun(runId) {
+        const { rows } = await this.pool.query('SELECT * FROM d2v_runs WHERE id = $1', [runId]);
+        return rows[0] ?? null;
+    }
+    async listRuns(filter = {}) {
+        const conditions = [];
+        const params = [];
+        if (filter.configId !== undefined) {
+            params.push(filter.configId);
+            conditions.push(`config_id = $${params.length}`);
+        }
+        if (filter.status !== undefined) {
+            params.push(filter.status);
+            conditions.push(`status = $${params.length}`);
+        }
+        if (filter.before !== undefined) {
+            params.push(filter.before);
+            conditions.push(`id < $${params.length}`);
+        }
+        params.push(Math.min(filter.limit ?? 50, 500));
+        const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : '';
+        const { rows } = await this.pool.query(`SELECT * FROM d2v_runs ${where} ORDER BY id DESC LIMIT $${params.length}`, params);
+        return rows;
+    }
+    /** Latest run per config, for the dashboard list. */
+    async getLastRuns() {
+        const { rows } = await this.pool.query(`SELECT DISTINCT ON (config_id) * FROM d2v_runs ORDER BY config_id, id DESC`);
+        return new Map(rows.map((r) => [r.config_id, r]));
+    }
+    // ------------------------------------------------------------------ logs
+    async insertLogs(runId, rows) {
+        if (rows.length === 0)
+            return;
+        const values = [];
+        const placeholders = rows.map((row, i) => {
+            const base = i * 6;
+            values.push(runId, row.seq, row.ts, row.level, row.module, row.message);
+            return `($${base + 1}, $${base + 2}, $${base + 3}, $${base + 4}, $${base + 5}, $${base + 6})`;
+        });
+        await this.pool.query(`INSERT INTO d2v_run_logs (run_id, seq, ts, level, module, message) VALUES ${placeholders.join(', ')}
+             ON CONFLICT DO NOTHING`, values);
+    }
+    async getLogs(runId, afterSeq = 0, limit = 1000) {
+        const { rows } = await this.pool.query(`SELECT seq, ts, level, module, message FROM d2v_run_logs
+             WHERE run_id = $1 AND seq > $2 ORDER BY seq LIMIT $3`, [runId, afterSeq, Math.min(limit, 5000)]);
+        return rows.map((r) => ({ ...r, seq: Number(r.seq) }));
+    }
+    async pruneOldLogs(retentionDays) {
+        const { rowCount } = await this.pool.query(`DELETE FROM d2v_run_logs USING d2v_runs
+             WHERE d2v_run_logs.run_id = d2v_runs.id
+               AND d2v_runs.finished_at < NOW() - make_interval(days => $1)`, [retentionDays]);
+        return rowCount ?? 0;
+    }
+    // ------------------------------------------------------------------ stats
+    /** Daily run counts by status + duration stats, for the config stats page. */
+    async getConfigStats(configId, days) {
+        const boundedDays = Math.min(Math.max(days, 1), 365);
+        const [daily, recent, totals] = await Promise.all([
+            this.pool.query(`SELECT date_trunc('day', queued_at)::date::text AS day, status, COUNT(*)::int AS count,
+                        AVG(EXTRACT(EPOCH FROM (finished_at - started_at)))::float AS avg_duration_s
+                 FROM d2v_runs
+                 WHERE config_id = $1 AND queued_at > NOW() - make_interval(days => $2)
+                 GROUP BY 1, 2 ORDER BY 1`, [configId, boundedDays]),
+            this.pool.query(`SELECT id, status, started_at, EXTRACT(EPOCH FROM (finished_at - started_at))::float AS duration_s
+                 FROM d2v_runs
+                 WHERE config_id = $1 AND status IN ('succeeded', 'failed') AND started_at IS NOT NULL AND finished_at IS NOT NULL
+                 ORDER BY id DESC LIMIT 50`, [configId]),
+            this.pool.query(`SELECT COUNT(*)::int AS total,
+                        COUNT(*) FILTER (WHERE status = 'succeeded')::int AS succeeded,
+                        COUNT(*) FILTER (WHERE status = 'failed')::int AS failed,
+                        COUNT(*) FILTER (WHERE status = 'skipped')::int AS skipped,
+                        COUNT(*) FILTER (WHERE status = 'canceled')::int AS canceled
+                 FROM d2v_runs
+                 WHERE config_id = $1 AND queued_at > NOW() - make_interval(days => $2)`, [configId, boundedDays]),
+        ]);
+        return {
+            daily: daily.rows,
+            recentDurations: recent.rows.reverse(),
+            totals: totals.rows[0],
+        };
+    }
+}
+exports.ControllerStore = ControllerStore;

package/dist/controller/types.js

@@ -0,0 +1,25 @@
+"use strict";
+// Shared types for controller mode
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ValidationError = exports.NotFoundError = exports.ConflictError = void 0;
+class ConflictError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'ConflictError';
+    }
+}
+exports.ConflictError = ConflictError;
+class NotFoundError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'NotFoundError';
+    }
+}
+exports.NotFoundError = NotFoundError;
+class ValidationError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'ValidationError';
+    }
+}
+exports.ValidationError = ValidationError;