doc2vec 2.10.5 → 2.10.6

package/dist/controller/config-registry.js

@@ -0,0 +1,277 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ConfigRegistry = void 0;
+exports.parseConfigMeta = parseConfigMeta;
+exports.isValidCron = isValidCron;
+exports.hashContent = hashContent;
+const crypto = __importStar(require("crypto"));
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const yaml = __importStar(require("js-yaml"));
+const croner_1 = require("croner");
+const types_1 = require("./types");
+/**
+ * Parse raw config YAML WITHOUT ${ENV} substitution (secrets stay as placeholders)
+ * and without Doc2Vec.loadConfig()'s process.exit() behavior — invalid files are
+ * reported, never fatal.
+ */
+function parseConfigMeta(content, filePath) {
+    const fallbackName = path.basename(filePath).replace(/\.ya?ml$/i, '');
+    try {
+        const parsed = yaml.load(content);
+        if (!parsed || typeof parsed !== 'object') {
+            return { name: fallbackName, schedule: null, sourceSummary: [], parseError: 'config is empty or not a YAML mapping' };
+        }
+        if (!Array.isArray(parsed.sources) || parsed.sources.length === 0) {
+            return { name: parsed.name || fallbackName, schedule: null, sourceSummary: [], parseError: 'config has no sources' };
+        }
+        const sourceSummary = parsed.sources.map((s) => ({
+            type: String(s?.type ?? 'unknown'),
+            product_name: String(s?.product_name ?? 'unknown'),
+            ...(s?.version !== undefined && { version: String(s.version) }),
+        }));
+        let schedule = null;
+        let parseError = null;
+        if (parsed.schedule !== undefined && parsed.schedule !== null) {
+            schedule = String(parsed.schedule);
+            if (!isValidCron(schedule)) {
+                parseError = `invalid cron expression: ${schedule}`;
+                schedule = null;
+            }
+        }
+        return { name: String(parsed.name || fallbackName), schedule, sourceSummary, parseError };
+    }
+    catch (err) {
+        return {
+            name: fallbackName,
+            schedule: null,
+            sourceSummary: [],
+            parseError: err instanceof Error ? err.message : String(err),
+        };
+    }
+}
+function isValidCron(expr) {
+    try {
+        new croner_1.Cron(expr, { paused: true }).stop();
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function hashContent(content) {
+    return crypto.createHash('sha256').update(content).digest('hex');
+}
+/**
+ * Watches the config files/directories passed on the CLI (plus --config-dir in
+ * read-write mode), keeps the d2v_configs table in sync, and — in read-write mode —
+ * persists UI edits back to YAML files on disk. Disk is the source of truth.
+ *
+ * Change detection is content-hash polling rather than fs.watch: Kubernetes
+ * ConfigMap updates arrive as atomic symlink swaps that fs.watch misses.
+ */
+class ConfigRegistry {
+    constructor(opts, store, events, logger) {
+        this.opts = opts;
+        this.store = store;
+        this.events = events;
+        this.logger = logger;
+        this.byPath = new Map();
+        this.timer = null;
+        this.syncing = false;
+    }
+    async start() {
+        await this.sync();
+        this.timer = setInterval(() => {
+            this.sync().catch(err => this.logger.error('Config re-sync failed:', err));
+        }, Math.max(this.opts.reloadIntervalSec, 5) * 1000);
+        this.timer.unref();
+    }
+    stop() {
+        if (this.timer)
+            clearInterval(this.timer);
+        this.timer = null;
+    }
+    list() {
+        return [...this.byPath.values()].sort((a, b) => a.name.localeCompare(b.name));
+    }
+    get(id) {
+        return [...this.byPath.values()].find(c => c.id === id);
+    }
+    /** Expand CLI args (files and directories) into the current set of config files. */
+    discoverFiles() {
+        const files = new Set();
+        const addDir = (dir) => {
+            for (const entry of fs.readdirSync(dir)) {
+                if (/\.ya?ml$/i.test(entry) && !entry.startsWith('.')) {
+                    const full = path.join(dir, entry);
+                    if (fs.statSync(full).isFile())
+                        files.add(full);
+                }
+            }
+        };
+        for (const arg of this.opts.configArgs) {
+            const abs = path.resolve(arg);
+            if (!fs.existsSync(abs)) {
+                this.logger.warn(`Config path does not exist: ${abs}`);
+                continue;
+            }
+            if (fs.statSync(abs).isDirectory()) {
+                addDir(abs);
+            }
+            else {
+                files.add(abs);
+            }
+        }
+        if (this.opts.readWrite && this.opts.configDir && fs.existsSync(this.opts.configDir)) {
+            addDir(path.resolve(this.opts.configDir));
+        }
+        return [...files];
+    }
+    async sync() {
+        if (this.syncing)
+            return;
+        this.syncing = true;
+        try {
+            const files = this.discoverFiles();
+            const seen = new Set();
+            let changed = false;
+            for (const file of files) {
+                seen.add(file);
+                let content;
+                try {
+                    content = fs.readFileSync(file, 'utf8');
+                }
+                catch (err) {
+                    this.logger.warn(`Failed to read config ${file}:`, err);
+                    continue;
+                }
+                const contentHash = hashContent(content);
+                const existing = this.byPath.get(file);
+                if (existing && existing.content_hash === contentHash)
+                    continue;
+                const meta = parseConfigMeta(content, file);
+                if (meta.parseError) {
+                    this.logger.warn(`Config ${file} has a problem: ${meta.parseError}`);
+                }
+                const record = await this.store.upsertConfig({
+                    path: file,
+                    name: meta.name,
+                    content,
+                    contentHash,
+                    schedule: meta.schedule,
+                    sourceSummary: meta.sourceSummary,
+                    parseError: meta.parseError,
+                });
+                this.byPath.set(file, record);
+                changed = true;
+                this.logger.info(`${existing ? 'Reloaded' : 'Loaded'} config '${record.name}' from ${file}` +
+                    (record.schedule ? ` (schedule: ${record.schedule})` : ' (manual runs only)'));
+            }
+            for (const [file] of this.byPath) {
+                if (!seen.has(file)) {
+                    this.logger.info(`Config file removed: ${file}`);
+                    await this.store.markConfigDeleted(file);
+                    this.byPath.delete(file);
+                    changed = true;
+                }
+            }
+            if (changed)
+                this.events.emitConfigUpdate();
+        }
+        finally {
+            this.syncing = false;
+        }
+    }
+    // ------------------------------------------------------------- read-write mode
+    /** Validate content for create/update: must parse and declare sources; cron must be valid. */
+    validateForWrite(content, filePath) {
+        const meta = parseConfigMeta(content, filePath);
+        if (meta.parseError) {
+            throw new types_1.ValidationError(meta.parseError);
+        }
+        return meta;
+    }
+    atomicWrite(target, content) {
+        const tmp = `${target}.tmp-${process.pid}`;
+        fs.writeFileSync(tmp, content, 'utf8');
+        fs.renameSync(tmp, target);
+    }
+    async create(filename, content) {
+        if (!this.opts.configDir) {
+            throw new types_1.ValidationError('no --config-dir configured');
+        }
+        if (!/^[A-Za-z0-9][A-Za-z0-9._-]*\.ya?ml$/.test(filename)) {
+            throw new types_1.ValidationError('filename must be a plain name ending in .yaml or .yml');
+        }
+        const target = path.join(path.resolve(this.opts.configDir), filename);
+        if (fs.existsSync(target)) {
+            throw new types_1.ConflictError(`config file ${filename} already exists`);
+        }
+        this.validateForWrite(content, target);
+        this.atomicWrite(target, content);
+        await this.sync();
+        const record = this.byPath.get(target);
+        if (!record)
+            throw new Error('config was written but failed to load');
+        return record;
+    }
+    async update(id, content, baseHash) {
+        const existing = this.get(id);
+        if (!existing)
+            throw new types_1.NotFoundError(`config ${id} not found`);
+        if (existing.content_hash !== baseHash) {
+            throw new types_1.ConflictError('config changed on disk since you loaded it — reload and reapply your edits');
+        }
+        this.validateForWrite(content, existing.path);
+        this.atomicWrite(existing.path, content);
+        await this.sync();
+        const record = this.byPath.get(existing.path);
+        if (!record)
+            throw new Error('config was written but failed to load');
+        return record;
+    }
+    async delete(id) {
+        const existing = this.get(id);
+        if (!existing)
+            throw new types_1.NotFoundError(`config ${id} not found`);
+        fs.unlinkSync(existing.path);
+        await this.store.markConfigDeleted(existing.path);
+        this.byPath.delete(existing.path);
+        this.events.emitConfigUpdate();
+    }
+}
+exports.ConfigRegistry = ConfigRegistry;

package/dist/controller/events.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ControllerEvents = void 0;
+const events_1 = require("events");
+/**
+ * In-process event bus connecting the job runner / config registry to SSE clients.
+ *
+ * Events:
+ *  - 'run:update'    (run: RunRecord)                     — a run was created or changed status
+ *  - 'run:log'       ({ runId: number, lines: LogRow[] }) — new log lines for a running job
+ *  - 'config:update' ()                                   — the set of configs changed on disk
+ */
+class ControllerEvents extends events_1.EventEmitter {
+    constructor() {
+        super();
+        // Every SSE client adds a listener per event; don't warn at the default 10
+        this.setMaxListeners(1000);
+    }
+    emitRunUpdate(run) {
+        this.emit('run:update', run);
+    }
+    emitRunLogs(runId, lines) {
+        this.emit('run:log', { runId, lines });
+    }
+    emitConfigUpdate() {
+        this.emit('config:update');
+    }
+}
+exports.ControllerEvents = ControllerEvents;

package/dist/controller/index.js

@@ -0,0 +1,139 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.startController = startController;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const logger_1 = require("../logger");
+const config_registry_1 = require("./config-registry");
+const events_1 = require("./events");
+const job_runner_1 = require("./job-runner");
+const notifier_1 = require("./notifier");
+const scheduler_1 = require("./scheduler");
+const server_1 = require("./server");
+const store_1 = require("./store");
+const LOG_PRUNE_INTERVAL_MS = 24 * 60 * 60 * 1000;
+async function startController(opts) {
+    const logger = new logger_1.Logger('Controller', {
+        level: logger_1.LogLevel.INFO,
+        useTimestamp: true,
+        useColor: true,
+        prettyPrint: true,
+    });
+    if (!opts.databaseUrl) {
+        throw new Error('controller mode requires Postgres: pass --database-url or set DATABASE_URL');
+    }
+    if (opts.readWrite && !opts.configDir) {
+        throw new Error('--read-write requires --config-dir (where new configs are written)');
+    }
+    if (opts.configArgs.length === 0 && !(opts.readWrite && opts.configDir)) {
+        throw new Error('provide at least one config file or directory');
+    }
+    if (opts.configDir) {
+        fs.mkdirSync(path.resolve(opts.configDir), { recursive: true });
+    }
+    logger.section('DOC2VEC CONTROLLER');
+    logger.info(`Mode: ${opts.readWrite ? 'read-write' : 'read-only'}, max parallel jobs: ${opts.maxParallel}`);
+    const store = new store_1.ControllerStore(opts.databaseUrl, logger.child('store'));
+    await store.init();
+    const events = new events_1.ControllerEvents();
+    const runner = new job_runner_1.JobRunner(store, events, { maxParallel: Math.max(opts.maxParallel, 1) }, logger.child('runner'));
+    const registry = new config_registry_1.ConfigRegistry({
+        configArgs: opts.configArgs,
+        configDir: opts.configDir,
+        readWrite: opts.readWrite,
+        reloadIntervalSec: opts.reloadIntervalSec,
+    }, store, events, logger.child('registry'));
+    const scheduler = new scheduler_1.Scheduler(configId => {
+        const config = registry.get(configId);
+        if (!config)
+            return;
+        runner.enqueue(config, 'scheduled').catch(err => logger.error(`Failed to enqueue scheduled run for '${config.name}':`, err));
+    }, logger.child('scheduler'));
+    events.on('config:update', () => scheduler.sync(registry.list()));
+    if (opts.slackWebhookUrl) {
+        new notifier_1.SlackNotifier({
+            webhookUrl: opts.slackWebhookUrl,
+            notify: opts.slackNotify ?? 'all',
+            publicUrl: opts.publicUrl,
+        }, store, logger.child('slack')).attach(events);
+    }
+    await registry.start();
+    scheduler.sync(registry.list());
+    const app = (0, server_1.createServer)({
+        store,
+        registry,
+        scheduler,
+        runner,
+        events,
+        readWrite: opts.readWrite,
+        logger: logger.child('api'),
+    });
+    const server = app.listen(opts.port, () => {
+        logger.info(`API and UI listening on http://localhost:${opts.port}`);
+    });
+    const pruneLogs = () => {
+        store.pruneOldLogs(opts.logRetentionDays)
+            .then(count => { if (count > 0)
+            logger.info(`Pruned ${count} log row(s) older than ${opts.logRetentionDays} days`); })
+            .catch(err => logger.error('Log retention pruning failed:', err));
+    };
+    pruneLogs();
+    const pruneTimer = setInterval(pruneLogs, LOG_PRUNE_INTERVAL_MS);
+    pruneTimer.unref();
+    let shuttingDown = false;
+    const shutdown = (signal) => {
+        if (shuttingDown)
+            return;
+        shuttingDown = true;
+        logger.info(`Received ${signal}, shutting down...`);
+        clearInterval(pruneTimer);
+        scheduler.stop();
+        registry.stop();
+        server.close();
+        runner.shutdown()
+            .then(() => store.close())
+            .then(() => {
+            logger.info('Shutdown complete');
+            process.exit(0);
+        })
+            .catch(err => {
+            logger.error('Error during shutdown:', err);
+            process.exit(1);
+        });
+    };
+    process.on('SIGTERM', () => shutdown('SIGTERM'));
+    process.on('SIGINT', () => shutdown('SIGINT'));
+}