doc2vec 2.10.4 → 2.10.6

package/dist/controller/server.js

@@ -0,0 +1,288 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createServer = createServer;
+const express_1 = __importDefault(require("express"));
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const config_registry_1 = require("./config-registry");
+const types_1 = require("./types");
+const SSE_HEARTBEAT_MS = 15000;
+function getVersion() {
+    try {
+        return require('../../package.json').version;
+    }
+    catch {
+        return 'unknown';
+    }
+}
+function sseInit(res) {
+    res.writeHead(200, {
+        'Content-Type': 'text/event-stream',
+        'Cache-Control': 'no-cache',
+        'Connection': 'keep-alive',
+        'X-Accel-Buffering': 'no',
+    });
+    res.write(':ok\n\n');
+}
+function sseSend(res, event, data) {
+    res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
+}
+function createServer(deps) {
+    const { store, registry, scheduler, runner, events, readWrite, logger } = deps;
+    const app = (0, express_1.default)();
+    app.use(express_1.default.json({ limit: '2mb' }));
+    const requireReadWrite = (_req, res, next) => {
+        if (!readWrite) {
+            res.status(403).json({ error: 'controller is running in read-only mode' });
+            return;
+        }
+        next();
+    };
+    const parseId = (raw) => {
+        const id = Number(raw);
+        if (!Number.isInteger(id) || id <= 0)
+            throw new types_1.ValidationError('invalid id');
+        return id;
+    };
+    const enrichConfig = (config, lastRuns) => ({
+        ...config,
+        next_run: scheduler.nextRun(config.id),
+        last_run: lastRuns.get(config.id) ?? null,
+        busy: runner.isBusy(config.id),
+    });
+    // ------------------------------------------------------------------ meta
+    app.get('/api/health', (_req, res) => {
+        res.json({ status: 'ok', mode: readWrite ? 'rw' : 'ro', version: getVersion() });
+    });
+    // ------------------------------------------------------------------ configs
+    app.get('/api/configs', async (_req, res) => {
+        const lastRuns = await store.getLastRuns();
+        res.json(registry.list().map(c => enrichConfig(c, lastRuns)));
+    });
+    app.get('/api/configs/:id', async (req, res) => {
+        const config = registry.get(parseId(String(req.params.id)));
+        if (!config)
+            throw new types_1.NotFoundError('config not found');
+        const lastRuns = await store.getLastRuns();
+        res.json(enrichConfig(config, lastRuns));
+    });
+    app.post('/api/configs', requireReadWrite, async (req, res) => {
+        const { filename, content } = req.body ?? {};
+        if (typeof filename !== 'string' || typeof content !== 'string') {
+            throw new types_1.ValidationError('body must contain filename and content strings');
+        }
+        const record = await registry.create(filename, content);
+        res.status(201).json(record);
+    });
+    app.put('/api/configs/:id', requireReadWrite, async (req, res) => {
+        const { content, baseHash } = req.body ?? {};
+        if (typeof content !== 'string' || typeof baseHash !== 'string') {
+            throw new types_1.ValidationError('body must contain content and baseHash strings');
+        }
+        const record = await registry.update(parseId(String(req.params.id)), content, baseHash);
+        res.json(record);
+    });
+    app.delete('/api/configs/:id', requireReadWrite, async (req, res) => {
+        await registry.delete(parseId(String(req.params.id)));
+        res.status(204).end();
+    });
+    app.post('/api/configs/validate', (req, res) => {
+        const { content } = req.body ?? {};
+        if (typeof content !== 'string') {
+            throw new types_1.ValidationError('body must contain a content string');
+        }
+        const meta = (0, config_registry_1.parseConfigMeta)(content, 'config.yaml');
+        res.json({
+            valid: !meta.parseError,
+            error: meta.parseError,
+            name: meta.name,
+            schedule: meta.schedule,
+            schedule_valid: meta.schedule ? (0, config_registry_1.isValidCron)(meta.schedule) : null,
+            sources: meta.sourceSummary,
+        });
+    });
+    app.post('/api/configs/:id/run', async (req, res) => {
+        const config = registry.get(parseId(String(req.params.id)));
+        if (!config)
+            throw new types_1.NotFoundError('config not found');
+        const run = await runner.enqueue(config, 'manual');
+        res.status(202).json(run);
+    });
+    app.get('/api/configs/:id/stats', async (req, res) => {
+        const id = parseId(String(req.params.id));
+        if (!registry.get(id))
+            throw new types_1.NotFoundError('config not found');
+        const days = Number(req.query.days) || 30;
+        res.json(await store.getConfigStats(id, days));
+    });
+    // ------------------------------------------------------------------ runs
+    app.get('/api/runs', async (req, res) => {
+        res.json(await store.listRuns({
+            configId: req.query.configId !== undefined ? parseId(String(req.query.configId)) : undefined,
+            status: req.query.status !== undefined ? String(req.query.status) : undefined,
+            limit: req.query.limit !== undefined ? Number(req.query.limit) : undefined,
+            before: req.query.before !== undefined ? Number(req.query.before) : undefined,
+        }));
+    });
+    app.get('/api/runs/:id', async (req, res) => {
+        const run = await store.getRun(parseId(String(req.params.id)));
+        if (!run)
+            throw new types_1.NotFoundError('run not found');
+        const config = registry.get(run.config_id) ?? await store.getConfig(run.config_id);
+        res.json({ ...run, config_name: config?.name ?? null });
+    });
+    app.get('/api/runs/:id/logs', async (req, res) => {
+        const id = parseId(String(req.params.id));
+        if (!await store.getRun(id))
+            throw new types_1.NotFoundError('run not found');
+        const afterSeq = Number(req.query.afterSeq) || 0;
+        const limit = Number(req.query.limit) || 1000;
+        res.json(await store.getLogs(id, afterSeq, limit));
+    });
+    app.post('/api/runs/:id/cancel', async (req, res) => {
+        res.json(await runner.cancel(parseId(String(req.params.id))));
+    });
+    // ------------------------------------------------------------------ SSE
+    app.get('/api/events', (req, res) => {
+        sseInit(res);
+        const onRunUpdate = (run) => sseSend(res, 'run:update', run);
+        const onConfigUpdate = () => sseSend(res, 'config:update', {});
+        events.on('run:update', onRunUpdate);
+        events.on('config:update', onConfigUpdate);
+        const heartbeat = setInterval(() => res.write(':hb\n\n'), SSE_HEARTBEAT_MS);
+        req.on('close', () => {
+            clearInterval(heartbeat);
+            events.off('run:update', onRunUpdate);
+            events.off('config:update', onConfigUpdate);
+        });
+    });
+    app.get('/api/runs/:id/logs/stream', async (req, res) => {
+        const id = parseId(String(req.params.id));
+        const run = await store.getRun(id);
+        if (!run)
+            throw new types_1.NotFoundError('run not found');
+        sseInit(res);
+        let lastSeq = Number(req.query.afterSeq) || 0;
+        let replayDone = false;
+        const pending = [];
+        const sendRow = (row) => {
+            if (row.seq > lastSeq) {
+                lastSeq = row.seq;
+                sseSend(res, 'log', row);
+            }
+        };
+        const onLog = (payload) => {
+            if (payload.runId !== id)
+                return;
+            if (!replayDone) {
+                pending.push(...payload.lines);
+                return;
+            }
+            payload.lines.forEach(sendRow);
+        };
+        const onRunUpdate = (updated) => {
+            if (updated.id !== id)
+                return;
+            sseSend(res, 'run:update', updated);
+            if (['succeeded', 'failed', 'canceled', 'skipped'].includes(updated.status)) {
+                sseSend(res, 'end', { status: updated.status });
+            }
+        };
+        events.on('run:log', onLog);
+        events.on('run:update', onRunUpdate);
+        // Replay history from the DB, then flush anything that streamed in meanwhile
+        try {
+            let batch;
+            do {
+                batch = await store.getLogs(id, lastSeq, 5000);
+                batch.forEach(sendRow);
+            } while (batch.length === 5000);
+        }
+        finally {
+            replayDone = true;
+            pending.forEach(sendRow);
+            pending.length = 0;
+        }
+        const current = await store.getRun(id);
+        if (current && ['succeeded', 'failed', 'canceled', 'skipped'].includes(current.status)) {
+            sseSend(res, 'end', { status: current.status });
+        }
+        const heartbeat = setInterval(() => res.write(':hb\n\n'), SSE_HEARTBEAT_MS);
+        req.on('close', () => {
+            clearInterval(heartbeat);
+            events.off('run:log', onLog);
+            events.off('run:update', onRunUpdate);
+        });
+    });
+    // ------------------------------------------------------------------ static UI
+    const uiDir = path.resolve(__dirname, '..', 'ui');
+    if (fs.existsSync(path.join(uiDir, 'index.html'))) {
+        app.use(express_1.default.static(uiDir));
+        // SPA fallback for client-side routes (anything that's not /api)
+        app.use((req, res, next) => {
+            if (req.method === 'GET' && !req.path.startsWith('/api')) {
+                res.sendFile(path.join(uiDir, 'index.html'));
+                return;
+            }
+            next();
+        });
+    }
+    else {
+        logger.warn(`UI assets not found at ${uiDir} — API only (build them with 'npm run build:ui')`);
+    }
+    // ------------------------------------------------------------------ errors
+    app.use((err, _req, res, _next) => {
+        if (res.headersSent)
+            return;
+        if (err instanceof types_1.ValidationError) {
+            res.status(400).json({ error: err.message });
+        }
+        else if (err instanceof types_1.NotFoundError) {
+            res.status(404).json({ error: err.message });
+        }
+        else if (err instanceof types_1.ConflictError) {
+            res.status(409).json({ error: err.message });
+        }
+        else {
+            logger.error('Unhandled API error:', err);
+            res.status(500).json({ error: 'internal error' });
+        }
+    });
+    return app;
+}

package/dist/controller/store.js

@@ -0,0 +1,186 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ControllerStore = void 0;
+const pg_1 = require("pg");
+const migrations_1 = require("./migrations");
+/**
+ * All Postgres access for controller mode: configs, runs, and run logs.
+ */
+class ControllerStore {
+    constructor(databaseUrl, logger) {
+        this.pool = new pg_1.Pool({ connectionString: databaseUrl, max: 10 });
+        this.logger = logger;
+    }
+    async init() {
+        await this.migrate();
+        await this.markOrphanedRuns();
+    }
+    async close() {
+        await this.pool.end();
+    }
+    async migrate() {
+        const client = await this.pool.connect();
+        try {
+            await client.query(`CREATE TABLE IF NOT EXISTS d2v_schema_migrations (
+                version INT PRIMARY KEY,
+                applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+            )`);
+            const { rows } = await client.query('SELECT COALESCE(MAX(version), 0) AS version FROM d2v_schema_migrations');
+            const current = Number(rows[0].version);
+            for (let i = current; i < migrations_1.MIGRATIONS.length; i++) {
+                const version = i + 1;
+                this.logger.info(`Applying migration ${version}`);
+                await client.query('BEGIN');
+                try {
+                    await client.query(migrations_1.MIGRATIONS[i]);
+                    await client.query('INSERT INTO d2v_schema_migrations (version) VALUES ($1)', [version]);
+                    await client.query('COMMIT');
+                }
+                catch (err) {
+                    await client.query('ROLLBACK');
+                    throw err;
+                }
+            }
+        }
+        finally {
+            client.release();
+        }
+    }
+    /** Runs left 'queued'/'running' by a previous controller process can never finish — fail them. */
+    async markOrphanedRuns() {
+        const { rowCount } = await this.pool.query(`UPDATE d2v_runs
+             SET status = 'failed', error = 'controller restarted while run was in progress', finished_at = NOW()
+             WHERE status IN ('queued', 'running')`);
+        if (rowCount) {
+            this.logger.warn(`Marked ${rowCount} orphaned run(s) from a previous controller process as failed`);
+        }
+    }
+    // ------------------------------------------------------------------ configs
+    async upsertConfig(input) {
+        const { rows } = await this.pool.query(`INSERT INTO d2v_configs (path, name, content, content_hash, schedule, source_summary, parse_error)
+             VALUES ($1, $2, $3, $4, $5, $6, $7)
+             ON CONFLICT (path) DO UPDATE SET
+                name = EXCLUDED.name,
+                content = EXCLUDED.content,
+                content_hash = EXCLUDED.content_hash,
+                schedule = EXCLUDED.schedule,
+                source_summary = EXCLUDED.source_summary,
+                parse_error = EXCLUDED.parse_error,
+                deleted_at = NULL,
+                updated_at = NOW()
+             RETURNING *`, [input.path, input.name, input.content, input.contentHash, input.schedule,
+            JSON.stringify(input.sourceSummary), input.parseError]);
+        return rows[0];
+    }
+    async markConfigDeleted(path) {
+        await this.pool.query('UPDATE d2v_configs SET deleted_at = NOW(), updated_at = NOW() WHERE path = $1', [path]);
+    }
+    async listConfigs(includeDeleted = false) {
+        const { rows } = await this.pool.query(`SELECT * FROM d2v_configs ${includeDeleted ? '' : 'WHERE deleted_at IS NULL'} ORDER BY name`);
+        return rows;
+    }
+    async getConfig(id) {
+        const { rows } = await this.pool.query('SELECT * FROM d2v_configs WHERE id = $1', [id]);
+        return rows[0] ?? null;
+    }
+    // ------------------------------------------------------------------ runs
+    async createRun(configId, configHash, trigger, status, error) {
+        const terminal = status === 'skipped';
+        const { rows } = await this.pool.query(`INSERT INTO d2v_runs (config_id, config_hash, trigger, status, error, finished_at)
+             VALUES ($1, $2, $3, $4, $5, ${terminal ? 'NOW()' : 'NULL'})
+             RETURNING *`, [configId, configHash, trigger, status, error ?? null]);
+        return rows[0];
+    }
+    async markRunStarted(runId, pid) {
+        const { rows } = await this.pool.query(`UPDATE d2v_runs SET status = 'running', pid = $2, started_at = NOW() WHERE id = $1 RETURNING *`, [runId, pid ?? null]);
+        return rows[0];
+    }
+    async finishRun(runId, outcome) {
+        const { rows } = await this.pool.query(`UPDATE d2v_runs
+             SET status = $2, exit_code = $3, error = $4, stats = $5, finished_at = NOW()
+             WHERE id = $1 RETURNING *`, [runId, outcome.status, outcome.exitCode ?? null, outcome.error ?? null, JSON.stringify(outcome.stats ?? {})]);
+        return rows[0];
+    }
+    async getRun(runId) {
+        const { rows } = await this.pool.query('SELECT * FROM d2v_runs WHERE id = $1', [runId]);
+        return rows[0] ?? null;
+    }
+    async listRuns(filter = {}) {
+        const conditions = [];
+        const params = [];
+        if (filter.configId !== undefined) {
+            params.push(filter.configId);
+            conditions.push(`config_id = $${params.length}`);
+        }
+        if (filter.status !== undefined) {
+            params.push(filter.status);
+            conditions.push(`status = $${params.length}`);
+        }
+        if (filter.before !== undefined) {
+            params.push(filter.before);
+            conditions.push(`id < $${params.length}`);
+        }
+        params.push(Math.min(filter.limit ?? 50, 500));
+        const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : '';
+        const { rows } = await this.pool.query(`SELECT * FROM d2v_runs ${where} ORDER BY id DESC LIMIT $${params.length}`, params);
+        return rows;
+    }
+    /** Latest run per config, for the dashboard list. */
+    async getLastRuns() {
+        const { rows } = await this.pool.query(`SELECT DISTINCT ON (config_id) * FROM d2v_runs ORDER BY config_id, id DESC`);
+        return new Map(rows.map((r) => [r.config_id, r]));
+    }
+    // ------------------------------------------------------------------ logs
+    async insertLogs(runId, rows) {
+        if (rows.length === 0)
+            return;
+        const values = [];
+        const placeholders = rows.map((row, i) => {
+            const base = i * 6;
+            values.push(runId, row.seq, row.ts, row.level, row.module, row.message);
+            return `($${base + 1}, $${base + 2}, $${base + 3}, $${base + 4}, $${base + 5}, $${base + 6})`;
+        });
+        await this.pool.query(`INSERT INTO d2v_run_logs (run_id, seq, ts, level, module, message) VALUES ${placeholders.join(', ')}
+             ON CONFLICT DO NOTHING`, values);
+    }
+    async getLogs(runId, afterSeq = 0, limit = 1000) {
+        const { rows } = await this.pool.query(`SELECT seq, ts, level, module, message FROM d2v_run_logs
+             WHERE run_id = $1 AND seq > $2 ORDER BY seq LIMIT $3`, [runId, afterSeq, Math.min(limit, 5000)]);
+        return rows.map((r) => ({ ...r, seq: Number(r.seq) }));
+    }
+    async pruneOldLogs(retentionDays) {
+        const { rowCount } = await this.pool.query(`DELETE FROM d2v_run_logs USING d2v_runs
+             WHERE d2v_run_logs.run_id = d2v_runs.id
+               AND d2v_runs.finished_at < NOW() - make_interval(days => $1)`, [retentionDays]);
+        return rowCount ?? 0;
+    }
+    // ------------------------------------------------------------------ stats
+    /** Daily run counts by status + duration stats, for the config stats page. */
+    async getConfigStats(configId, days) {
+        const boundedDays = Math.min(Math.max(days, 1), 365);
+        const [daily, recent, totals] = await Promise.all([
+            this.pool.query(`SELECT date_trunc('day', queued_at)::date::text AS day, status, COUNT(*)::int AS count,
+                        AVG(EXTRACT(EPOCH FROM (finished_at - started_at)))::float AS avg_duration_s
+                 FROM d2v_runs
+                 WHERE config_id = $1 AND queued_at > NOW() - make_interval(days => $2)
+                 GROUP BY 1, 2 ORDER BY 1`, [configId, boundedDays]),
+            this.pool.query(`SELECT id, status, started_at, EXTRACT(EPOCH FROM (finished_at - started_at))::float AS duration_s
+                 FROM d2v_runs
+                 WHERE config_id = $1 AND status IN ('succeeded', 'failed') AND started_at IS NOT NULL AND finished_at IS NOT NULL
+                 ORDER BY id DESC LIMIT 50`, [configId]),
+            this.pool.query(`SELECT COUNT(*)::int AS total,
+                        COUNT(*) FILTER (WHERE status = 'succeeded')::int AS succeeded,
+                        COUNT(*) FILTER (WHERE status = 'failed')::int AS failed,
+                        COUNT(*) FILTER (WHERE status = 'skipped')::int AS skipped,
+                        COUNT(*) FILTER (WHERE status = 'canceled')::int AS canceled
+                 FROM d2v_runs
+                 WHERE config_id = $1 AND queued_at > NOW() - make_interval(days => $2)`, [configId, boundedDays]),
+        ]);
+        return {
+            daily: daily.rows,
+            recentDurations: recent.rows.reverse(),
+            totals: totals.rows[0],
+        };
+    }
+}
+exports.ControllerStore = ControllerStore;

package/dist/controller/types.js

@@ -0,0 +1,25 @@
+"use strict";
+// Shared types for controller mode
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ValidationError = exports.NotFoundError = exports.ConflictError = void 0;
+class ConflictError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'ConflictError';
+    }
+}
+exports.ConflictError = ConflictError;
+class NotFoundError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'NotFoundError';
+    }
+}
+exports.NotFoundError = NotFoundError;
+class ValidationError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'ValidationError';
+    }
+}
+exports.ValidationError = ValidationError;

package/dist/database.js

@@ -215,7 +215,13 @@ class DatabaseManager {
             }
         }
         catch (error) {
-            logger.error(`Failed to update metadata value for ${key}:`, error);
+            // Best-effort by design: a failed metadata write errs toward the
+            // safe direction (redundant reprocessing next run — etag/lastmod/
+            // watermark stay behind, never skip ahead), so we warn rather than
+            // fail the job. Genuine disk/connection failures surface earlier at
+            // the chunk-write stage (storeChunkInQdrant), which shares this
+            // collection and throws.
+            logger.warn(`Failed to update metadata value for ${key} (continuing; will retry next run):`, error);
         }
     }
     static async getLastRunDate(dbConnection, repo, defaultDate, logger) {
@@ -436,7 +442,11 @@ class DatabaseManager {
             });
         }
         catch (error) {
-            console.error("Error storing chunk in Qdrant:", error);
+            // Rethrow so the failure propagates to the top-level handler and the
+            // job exits non-zero. Swallowing here previously let a sync "succeed"
+            // while chunks silently failed to store (e.g. Qdrant disk full).
+            console.error(`Error storing chunk in Qdrant (collection ${collectionName}, url ${chunk.metadata.url}):`, error);
+            throw error;
         }
     }
     static removeObsoleteChunksSQLite(db, visitedUrls, urlPrefix, logger) {
@@ -518,7 +528,11 @@ class DatabaseManager {
             }
         }
         catch (error) {
+            // Rethrow so a failed cleanup fails the job instead of silently
+            // leaving orphans behind. This is idempotent and recomputed each
+            // run, so a rerun retries it cleanly.
             logger.error(`Error removing obsolete chunks from Qdrant:`, error);
+            throw error;
         }
     }
     static hasColumn(db, columnName) {
@@ -561,7 +575,12 @@ class DatabaseManager {
             logger.info(`Deleted chunks from Qdrant for URL ${url}`);
         }
         catch (error) {
+            // Rethrow so the failure propagates and the job exits non-zero.
+            // A swallowed delete leaves stale chunks behind while the sync
+            // reports success. Callers that treat deletion as best-effort
+            // (e.g. 404 cleanup) wrap this call in their own try/catch.
             logger.error(`Error deleting chunks from Qdrant for URL ${url}:`, error);
+            throw error;
         }
     }
     /**
@@ -726,53 +745,64 @@ class DatabaseManager {
                 urlPrefix = `file://${cleanedDirPrefix}`;
             }
             logger.debug(`Checking for obsolete chunks with URL prefix: ${urlPrefix}`);
-            const response = await client.scroll(collectionName, {
-                limit: 10000,
-                with_payload: true,
-                with_vector: false,
-                filter: {
-                    must: [
-                        {
-                            key: "url",
-                            match: {
-                                text: urlPrefix
-                            }
+            // Scroll through ALL matching points via next_page_offset. A single
+            // capped scroll would only examine the first page, so obsolete
+            // chunks beyond it (in collections larger than the page limit) would
+            // never be removed.
+            const filter = {
+                must: [
+                    {
+                        key: "url",
+                        match: {
+                            text: urlPrefix
                         }
-                    ],
-                    must_not: [
-                        {
-                            key: "is_metadata",
-                            match: {
-                                value: true
-                            }
+                    }
+                ],
+                must_not: [
+                    {
+                        key: "is_metadata",
+                        match: {
+                            value: true
                         }
-                    ]
-                }
-            });
-            const obsoletePointIds = response.points
-                .filter((point) => {
-                const url = point.payload?.url;
-                // Double check it's not a metadata record
-                if (point.payload?.is_metadata === true) {
-                    return false;
-                }
-                if (!url || !url.startsWith(urlPrefix)) {
-                    return false;
-                }
-                let filePath;
-                if (isRewriteMode) {
-                    // URL rewrite mode: extract relative path and construct full file path
-                    const config = pathConfig;
-                    const relativePath = url.substring(urlPrefix.length + 1); // +1 for the '/'
-                    filePath = path.join(config.path, relativePath);
-                }
-                else {
-                    // Direct file path mode: remove file:// prefix to match with processedFiles
-                    filePath = url.startsWith('file://') ? url.substring(7) : '';
+                    }
+                ]
+            };
+            const obsoletePointIds = [];
+            let offset = undefined;
+            do {
+                const response = await client.scroll(collectionName, {
+                    limit: 1000,
+                    offset,
+                    with_payload: true,
+                    with_vector: false,
+                    filter
+                });
+                for (const point of response.points) {
+                    const url = point.payload?.url;
+                    // Double check it's not a metadata record
+                    if (point.payload?.is_metadata === true) {
+                        continue;
+                    }
+                    if (!url || !url.startsWith(urlPrefix)) {
+                        continue;
+                    }
+                    let filePath;
+                    if (isRewriteMode) {
+                        // URL rewrite mode: extract relative path and construct full file path
+                        const config = pathConfig;
+                        const relativePath = url.substring(urlPrefix.length + 1); // +1 for the '/'
+                        filePath = path.join(config.path, relativePath);
+                    }
+                    else {
+                        // Direct file path mode: remove file:// prefix to match with processedFiles
+                        filePath = url.startsWith('file://') ? url.substring(7) : '';
+                    }
+                    if (filePath && !processedFiles.has(filePath)) {
+                        obsoletePointIds.push(point.id);
+                    }
                 }
-                return filePath && !processedFiles.has(filePath);
-            })
-                .map((point) => point.id);
+                offset = response.next_page_offset;
+            } while (offset !== null && offset !== undefined);
             if (obsoletePointIds.length > 0) {
                 await client.delete(collectionName, {
                     points: obsoletePointIds,
@@ -784,7 +814,10 @@ class DatabaseManager {
             }
         }
         catch (error) {
+            // Rethrow so a failed cleanup fails the job instead of silently
+            // leaving orphans behind (idempotent — retried cleanly next run).
             logger.error(`Error removing obsolete chunks from Qdrant:`, error);
+            throw error;
         }
     }
 }