doc2vec 2.10.5 → 2.10.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,288 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || (function () {
19
+ var ownKeys = function(o) {
20
+ ownKeys = Object.getOwnPropertyNames || function (o) {
21
+ var ar = [];
22
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
+ return ar;
24
+ };
25
+ return ownKeys(o);
26
+ };
27
+ return function (mod) {
28
+ if (mod && mod.__esModule) return mod;
29
+ var result = {};
30
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
+ __setModuleDefault(result, mod);
32
+ return result;
33
+ };
34
+ })();
35
+ var __importDefault = (this && this.__importDefault) || function (mod) {
36
+ return (mod && mod.__esModule) ? mod : { "default": mod };
37
+ };
38
+ Object.defineProperty(exports, "__esModule", { value: true });
39
+ exports.createServer = createServer;
40
+ const express_1 = __importDefault(require("express"));
41
+ const fs = __importStar(require("fs"));
42
+ const path = __importStar(require("path"));
43
+ const config_registry_1 = require("./config-registry");
44
+ const types_1 = require("./types");
45
+ const SSE_HEARTBEAT_MS = 15000;
46
+ function getVersion() {
47
+ try {
48
+ return require('../../package.json').version;
49
+ }
50
+ catch {
51
+ return 'unknown';
52
+ }
53
+ }
54
+ function sseInit(res) {
55
+ res.writeHead(200, {
56
+ 'Content-Type': 'text/event-stream',
57
+ 'Cache-Control': 'no-cache',
58
+ 'Connection': 'keep-alive',
59
+ 'X-Accel-Buffering': 'no',
60
+ });
61
+ res.write(':ok\n\n');
62
+ }
63
+ function sseSend(res, event, data) {
64
+ res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
65
+ }
66
+ function createServer(deps) {
67
+ const { store, registry, scheduler, runner, events, readWrite, logger } = deps;
68
+ const app = (0, express_1.default)();
69
+ app.use(express_1.default.json({ limit: '2mb' }));
70
+ const requireReadWrite = (_req, res, next) => {
71
+ if (!readWrite) {
72
+ res.status(403).json({ error: 'controller is running in read-only mode' });
73
+ return;
74
+ }
75
+ next();
76
+ };
77
+ const parseId = (raw) => {
78
+ const id = Number(raw);
79
+ if (!Number.isInteger(id) || id <= 0)
80
+ throw new types_1.ValidationError('invalid id');
81
+ return id;
82
+ };
83
+ const enrichConfig = (config, lastRuns) => ({
84
+ ...config,
85
+ next_run: scheduler.nextRun(config.id),
86
+ last_run: lastRuns.get(config.id) ?? null,
87
+ busy: runner.isBusy(config.id),
88
+ });
89
+ // ------------------------------------------------------------------ meta
90
+ app.get('/api/health', (_req, res) => {
91
+ res.json({ status: 'ok', mode: readWrite ? 'rw' : 'ro', version: getVersion() });
92
+ });
93
+ // ------------------------------------------------------------------ configs
94
+ app.get('/api/configs', async (_req, res) => {
95
+ const lastRuns = await store.getLastRuns();
96
+ res.json(registry.list().map(c => enrichConfig(c, lastRuns)));
97
+ });
98
+ app.get('/api/configs/:id', async (req, res) => {
99
+ const config = registry.get(parseId(String(req.params.id)));
100
+ if (!config)
101
+ throw new types_1.NotFoundError('config not found');
102
+ const lastRuns = await store.getLastRuns();
103
+ res.json(enrichConfig(config, lastRuns));
104
+ });
105
+ app.post('/api/configs', requireReadWrite, async (req, res) => {
106
+ const { filename, content } = req.body ?? {};
107
+ if (typeof filename !== 'string' || typeof content !== 'string') {
108
+ throw new types_1.ValidationError('body must contain filename and content strings');
109
+ }
110
+ const record = await registry.create(filename, content);
111
+ res.status(201).json(record);
112
+ });
113
+ app.put('/api/configs/:id', requireReadWrite, async (req, res) => {
114
+ const { content, baseHash } = req.body ?? {};
115
+ if (typeof content !== 'string' || typeof baseHash !== 'string') {
116
+ throw new types_1.ValidationError('body must contain content and baseHash strings');
117
+ }
118
+ const record = await registry.update(parseId(String(req.params.id)), content, baseHash);
119
+ res.json(record);
120
+ });
121
+ app.delete('/api/configs/:id', requireReadWrite, async (req, res) => {
122
+ await registry.delete(parseId(String(req.params.id)));
123
+ res.status(204).end();
124
+ });
125
+ app.post('/api/configs/validate', (req, res) => {
126
+ const { content } = req.body ?? {};
127
+ if (typeof content !== 'string') {
128
+ throw new types_1.ValidationError('body must contain a content string');
129
+ }
130
+ const meta = (0, config_registry_1.parseConfigMeta)(content, 'config.yaml');
131
+ res.json({
132
+ valid: !meta.parseError,
133
+ error: meta.parseError,
134
+ name: meta.name,
135
+ schedule: meta.schedule,
136
+ schedule_valid: meta.schedule ? (0, config_registry_1.isValidCron)(meta.schedule) : null,
137
+ sources: meta.sourceSummary,
138
+ });
139
+ });
140
+ app.post('/api/configs/:id/run', async (req, res) => {
141
+ const config = registry.get(parseId(String(req.params.id)));
142
+ if (!config)
143
+ throw new types_1.NotFoundError('config not found');
144
+ const run = await runner.enqueue(config, 'manual');
145
+ res.status(202).json(run);
146
+ });
147
+ app.get('/api/configs/:id/stats', async (req, res) => {
148
+ const id = parseId(String(req.params.id));
149
+ if (!registry.get(id))
150
+ throw new types_1.NotFoundError('config not found');
151
+ const days = Number(req.query.days) || 30;
152
+ res.json(await store.getConfigStats(id, days));
153
+ });
154
+ // ------------------------------------------------------------------ runs
155
+ app.get('/api/runs', async (req, res) => {
156
+ res.json(await store.listRuns({
157
+ configId: req.query.configId !== undefined ? parseId(String(req.query.configId)) : undefined,
158
+ status: req.query.status !== undefined ? String(req.query.status) : undefined,
159
+ limit: req.query.limit !== undefined ? Number(req.query.limit) : undefined,
160
+ before: req.query.before !== undefined ? Number(req.query.before) : undefined,
161
+ }));
162
+ });
163
+ app.get('/api/runs/:id', async (req, res) => {
164
+ const run = await store.getRun(parseId(String(req.params.id)));
165
+ if (!run)
166
+ throw new types_1.NotFoundError('run not found');
167
+ const config = registry.get(run.config_id) ?? await store.getConfig(run.config_id);
168
+ res.json({ ...run, config_name: config?.name ?? null });
169
+ });
170
+ app.get('/api/runs/:id/logs', async (req, res) => {
171
+ const id = parseId(String(req.params.id));
172
+ if (!await store.getRun(id))
173
+ throw new types_1.NotFoundError('run not found');
174
+ const afterSeq = Number(req.query.afterSeq) || 0;
175
+ const limit = Number(req.query.limit) || 1000;
176
+ res.json(await store.getLogs(id, afterSeq, limit));
177
+ });
178
+ app.post('/api/runs/:id/cancel', async (req, res) => {
179
+ res.json(await runner.cancel(parseId(String(req.params.id))));
180
+ });
181
+ // ------------------------------------------------------------------ SSE
182
+ app.get('/api/events', (req, res) => {
183
+ sseInit(res);
184
+ const onRunUpdate = (run) => sseSend(res, 'run:update', run);
185
+ const onConfigUpdate = () => sseSend(res, 'config:update', {});
186
+ events.on('run:update', onRunUpdate);
187
+ events.on('config:update', onConfigUpdate);
188
+ const heartbeat = setInterval(() => res.write(':hb\n\n'), SSE_HEARTBEAT_MS);
189
+ req.on('close', () => {
190
+ clearInterval(heartbeat);
191
+ events.off('run:update', onRunUpdate);
192
+ events.off('config:update', onConfigUpdate);
193
+ });
194
+ });
195
+ app.get('/api/runs/:id/logs/stream', async (req, res) => {
196
+ const id = parseId(String(req.params.id));
197
+ const run = await store.getRun(id);
198
+ if (!run)
199
+ throw new types_1.NotFoundError('run not found');
200
+ sseInit(res);
201
+ let lastSeq = Number(req.query.afterSeq) || 0;
202
+ let replayDone = false;
203
+ const pending = [];
204
+ const sendRow = (row) => {
205
+ if (row.seq > lastSeq) {
206
+ lastSeq = row.seq;
207
+ sseSend(res, 'log', row);
208
+ }
209
+ };
210
+ const onLog = (payload) => {
211
+ if (payload.runId !== id)
212
+ return;
213
+ if (!replayDone) {
214
+ pending.push(...payload.lines);
215
+ return;
216
+ }
217
+ payload.lines.forEach(sendRow);
218
+ };
219
+ const onRunUpdate = (updated) => {
220
+ if (updated.id !== id)
221
+ return;
222
+ sseSend(res, 'run:update', updated);
223
+ if (['succeeded', 'failed', 'canceled', 'skipped'].includes(updated.status)) {
224
+ sseSend(res, 'end', { status: updated.status });
225
+ }
226
+ };
227
+ events.on('run:log', onLog);
228
+ events.on('run:update', onRunUpdate);
229
+ // Replay history from the DB, then flush anything that streamed in meanwhile
230
+ try {
231
+ let batch;
232
+ do {
233
+ batch = await store.getLogs(id, lastSeq, 5000);
234
+ batch.forEach(sendRow);
235
+ } while (batch.length === 5000);
236
+ }
237
+ finally {
238
+ replayDone = true;
239
+ pending.forEach(sendRow);
240
+ pending.length = 0;
241
+ }
242
+ const current = await store.getRun(id);
243
+ if (current && ['succeeded', 'failed', 'canceled', 'skipped'].includes(current.status)) {
244
+ sseSend(res, 'end', { status: current.status });
245
+ }
246
+ const heartbeat = setInterval(() => res.write(':hb\n\n'), SSE_HEARTBEAT_MS);
247
+ req.on('close', () => {
248
+ clearInterval(heartbeat);
249
+ events.off('run:log', onLog);
250
+ events.off('run:update', onRunUpdate);
251
+ });
252
+ });
253
+ // ------------------------------------------------------------------ static UI
254
+ const uiDir = path.resolve(__dirname, '..', 'ui');
255
+ if (fs.existsSync(path.join(uiDir, 'index.html'))) {
256
+ app.use(express_1.default.static(uiDir));
257
+ // SPA fallback for client-side routes (anything that's not /api)
258
+ app.use((req, res, next) => {
259
+ if (req.method === 'GET' && !req.path.startsWith('/api')) {
260
+ res.sendFile(path.join(uiDir, 'index.html'));
261
+ return;
262
+ }
263
+ next();
264
+ });
265
+ }
266
+ else {
267
+ logger.warn(`UI assets not found at ${uiDir} — API only (build them with 'npm run build:ui')`);
268
+ }
269
+ // ------------------------------------------------------------------ errors
270
+ app.use((err, _req, res, _next) => {
271
+ if (res.headersSent)
272
+ return;
273
+ if (err instanceof types_1.ValidationError) {
274
+ res.status(400).json({ error: err.message });
275
+ }
276
+ else if (err instanceof types_1.NotFoundError) {
277
+ res.status(404).json({ error: err.message });
278
+ }
279
+ else if (err instanceof types_1.ConflictError) {
280
+ res.status(409).json({ error: err.message });
281
+ }
282
+ else {
283
+ logger.error('Unhandled API error:', err);
284
+ res.status(500).json({ error: 'internal error' });
285
+ }
286
+ });
287
+ return app;
288
+ }
@@ -0,0 +1,186 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.ControllerStore = void 0;
4
+ const pg_1 = require("pg");
5
+ const migrations_1 = require("./migrations");
6
+ /**
7
+ * All Postgres access for controller mode: configs, runs, and run logs.
8
+ */
9
+ class ControllerStore {
10
+ constructor(databaseUrl, logger) {
11
+ this.pool = new pg_1.Pool({ connectionString: databaseUrl, max: 10 });
12
+ this.logger = logger;
13
+ }
14
+ async init() {
15
+ await this.migrate();
16
+ await this.markOrphanedRuns();
17
+ }
18
+ async close() {
19
+ await this.pool.end();
20
+ }
21
+ async migrate() {
22
+ const client = await this.pool.connect();
23
+ try {
24
+ await client.query(`CREATE TABLE IF NOT EXISTS d2v_schema_migrations (
25
+ version INT PRIMARY KEY,
26
+ applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
27
+ )`);
28
+ const { rows } = await client.query('SELECT COALESCE(MAX(version), 0) AS version FROM d2v_schema_migrations');
29
+ const current = Number(rows[0].version);
30
+ for (let i = current; i < migrations_1.MIGRATIONS.length; i++) {
31
+ const version = i + 1;
32
+ this.logger.info(`Applying migration ${version}`);
33
+ await client.query('BEGIN');
34
+ try {
35
+ await client.query(migrations_1.MIGRATIONS[i]);
36
+ await client.query('INSERT INTO d2v_schema_migrations (version) VALUES ($1)', [version]);
37
+ await client.query('COMMIT');
38
+ }
39
+ catch (err) {
40
+ await client.query('ROLLBACK');
41
+ throw err;
42
+ }
43
+ }
44
+ }
45
+ finally {
46
+ client.release();
47
+ }
48
+ }
49
+ /** Runs left 'queued'/'running' by a previous controller process can never finish — fail them. */
50
+ async markOrphanedRuns() {
51
+ const { rowCount } = await this.pool.query(`UPDATE d2v_runs
52
+ SET status = 'failed', error = 'controller restarted while run was in progress', finished_at = NOW()
53
+ WHERE status IN ('queued', 'running')`);
54
+ if (rowCount) {
55
+ this.logger.warn(`Marked ${rowCount} orphaned run(s) from a previous controller process as failed`);
56
+ }
57
+ }
58
+ // ------------------------------------------------------------------ configs
59
+ async upsertConfig(input) {
60
+ const { rows } = await this.pool.query(`INSERT INTO d2v_configs (path, name, content, content_hash, schedule, source_summary, parse_error)
61
+ VALUES ($1, $2, $3, $4, $5, $6, $7)
62
+ ON CONFLICT (path) DO UPDATE SET
63
+ name = EXCLUDED.name,
64
+ content = EXCLUDED.content,
65
+ content_hash = EXCLUDED.content_hash,
66
+ schedule = EXCLUDED.schedule,
67
+ source_summary = EXCLUDED.source_summary,
68
+ parse_error = EXCLUDED.parse_error,
69
+ deleted_at = NULL,
70
+ updated_at = NOW()
71
+ RETURNING *`, [input.path, input.name, input.content, input.contentHash, input.schedule,
72
+ JSON.stringify(input.sourceSummary), input.parseError]);
73
+ return rows[0];
74
+ }
75
+ async markConfigDeleted(path) {
76
+ await this.pool.query('UPDATE d2v_configs SET deleted_at = NOW(), updated_at = NOW() WHERE path = $1', [path]);
77
+ }
78
+ async listConfigs(includeDeleted = false) {
79
+ const { rows } = await this.pool.query(`SELECT * FROM d2v_configs ${includeDeleted ? '' : 'WHERE deleted_at IS NULL'} ORDER BY name`);
80
+ return rows;
81
+ }
82
+ async getConfig(id) {
83
+ const { rows } = await this.pool.query('SELECT * FROM d2v_configs WHERE id = $1', [id]);
84
+ return rows[0] ?? null;
85
+ }
86
+ // ------------------------------------------------------------------ runs
87
+ async createRun(configId, configHash, trigger, status, error) {
88
+ const terminal = status === 'skipped';
89
+ const { rows } = await this.pool.query(`INSERT INTO d2v_runs (config_id, config_hash, trigger, status, error, finished_at)
90
+ VALUES ($1, $2, $3, $4, $5, ${terminal ? 'NOW()' : 'NULL'})
91
+ RETURNING *`, [configId, configHash, trigger, status, error ?? null]);
92
+ return rows[0];
93
+ }
94
+ async markRunStarted(runId, pid) {
95
+ const { rows } = await this.pool.query(`UPDATE d2v_runs SET status = 'running', pid = $2, started_at = NOW() WHERE id = $1 RETURNING *`, [runId, pid ?? null]);
96
+ return rows[0];
97
+ }
98
+ async finishRun(runId, outcome) {
99
+ const { rows } = await this.pool.query(`UPDATE d2v_runs
100
+ SET status = $2, exit_code = $3, error = $4, stats = $5, finished_at = NOW()
101
+ WHERE id = $1 RETURNING *`, [runId, outcome.status, outcome.exitCode ?? null, outcome.error ?? null, JSON.stringify(outcome.stats ?? {})]);
102
+ return rows[0];
103
+ }
104
+ async getRun(runId) {
105
+ const { rows } = await this.pool.query('SELECT * FROM d2v_runs WHERE id = $1', [runId]);
106
+ return rows[0] ?? null;
107
+ }
108
+ async listRuns(filter = {}) {
109
+ const conditions = [];
110
+ const params = [];
111
+ if (filter.configId !== undefined) {
112
+ params.push(filter.configId);
113
+ conditions.push(`config_id = $${params.length}`);
114
+ }
115
+ if (filter.status !== undefined) {
116
+ params.push(filter.status);
117
+ conditions.push(`status = $${params.length}`);
118
+ }
119
+ if (filter.before !== undefined) {
120
+ params.push(filter.before);
121
+ conditions.push(`id < $${params.length}`);
122
+ }
123
+ params.push(Math.min(filter.limit ?? 50, 500));
124
+ const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : '';
125
+ const { rows } = await this.pool.query(`SELECT * FROM d2v_runs ${where} ORDER BY id DESC LIMIT $${params.length}`, params);
126
+ return rows;
127
+ }
128
+ /** Latest run per config, for the dashboard list. */
129
+ async getLastRuns() {
130
+ const { rows } = await this.pool.query(`SELECT DISTINCT ON (config_id) * FROM d2v_runs ORDER BY config_id, id DESC`);
131
+ return new Map(rows.map((r) => [r.config_id, r]));
132
+ }
133
+ // ------------------------------------------------------------------ logs
134
+ async insertLogs(runId, rows) {
135
+ if (rows.length === 0)
136
+ return;
137
+ const values = [];
138
+ const placeholders = rows.map((row, i) => {
139
+ const base = i * 6;
140
+ values.push(runId, row.seq, row.ts, row.level, row.module, row.message);
141
+ return `($${base + 1}, $${base + 2}, $${base + 3}, $${base + 4}, $${base + 5}, $${base + 6})`;
142
+ });
143
+ await this.pool.query(`INSERT INTO d2v_run_logs (run_id, seq, ts, level, module, message) VALUES ${placeholders.join(', ')}
144
+ ON CONFLICT DO NOTHING`, values);
145
+ }
146
+ async getLogs(runId, afterSeq = 0, limit = 1000) {
147
+ const { rows } = await this.pool.query(`SELECT seq, ts, level, module, message FROM d2v_run_logs
148
+ WHERE run_id = $1 AND seq > $2 ORDER BY seq LIMIT $3`, [runId, afterSeq, Math.min(limit, 5000)]);
149
+ return rows.map((r) => ({ ...r, seq: Number(r.seq) }));
150
+ }
151
+ async pruneOldLogs(retentionDays) {
152
+ const { rowCount } = await this.pool.query(`DELETE FROM d2v_run_logs USING d2v_runs
153
+ WHERE d2v_run_logs.run_id = d2v_runs.id
154
+ AND d2v_runs.finished_at < NOW() - make_interval(days => $1)`, [retentionDays]);
155
+ return rowCount ?? 0;
156
+ }
157
+ // ------------------------------------------------------------------ stats
158
+ /** Daily run counts by status + duration stats, for the config stats page. */
159
+ async getConfigStats(configId, days) {
160
+ const boundedDays = Math.min(Math.max(days, 1), 365);
161
+ const [daily, recent, totals] = await Promise.all([
162
+ this.pool.query(`SELECT date_trunc('day', queued_at)::date::text AS day, status, COUNT(*)::int AS count,
163
+ AVG(EXTRACT(EPOCH FROM (finished_at - started_at)))::float AS avg_duration_s
164
+ FROM d2v_runs
165
+ WHERE config_id = $1 AND queued_at > NOW() - make_interval(days => $2)
166
+ GROUP BY 1, 2 ORDER BY 1`, [configId, boundedDays]),
167
+ this.pool.query(`SELECT id, status, started_at, EXTRACT(EPOCH FROM (finished_at - started_at))::float AS duration_s
168
+ FROM d2v_runs
169
+ WHERE config_id = $1 AND status IN ('succeeded', 'failed') AND started_at IS NOT NULL AND finished_at IS NOT NULL
170
+ ORDER BY id DESC LIMIT 50`, [configId]),
171
+ this.pool.query(`SELECT COUNT(*)::int AS total,
172
+ COUNT(*) FILTER (WHERE status = 'succeeded')::int AS succeeded,
173
+ COUNT(*) FILTER (WHERE status = 'failed')::int AS failed,
174
+ COUNT(*) FILTER (WHERE status = 'skipped')::int AS skipped,
175
+ COUNT(*) FILTER (WHERE status = 'canceled')::int AS canceled
176
+ FROM d2v_runs
177
+ WHERE config_id = $1 AND queued_at > NOW() - make_interval(days => $2)`, [configId, boundedDays]),
178
+ ]);
179
+ return {
180
+ daily: daily.rows,
181
+ recentDurations: recent.rows.reverse(),
182
+ totals: totals.rows[0],
183
+ };
184
+ }
185
+ }
186
+ exports.ControllerStore = ControllerStore;
@@ -0,0 +1,25 @@
1
+ "use strict";
2
+ // Shared types for controller mode
3
+ Object.defineProperty(exports, "__esModule", { value: true });
4
+ exports.ValidationError = exports.NotFoundError = exports.ConflictError = void 0;
5
+ class ConflictError extends Error {
6
+ constructor(message) {
7
+ super(message);
8
+ this.name = 'ConflictError';
9
+ }
10
+ }
11
+ exports.ConflictError = ConflictError;
12
+ class NotFoundError extends Error {
13
+ constructor(message) {
14
+ super(message);
15
+ this.name = 'NotFoundError';
16
+ }
17
+ }
18
+ exports.NotFoundError = NotFoundError;
19
+ class ValidationError extends Error {
20
+ constructor(message) {
21
+ super(message);
22
+ this.name = 'ValidationError';
23
+ }
24
+ }
25
+ exports.ValidationError = ValidationError;
package/dist/doc2vec.js CHANGED
@@ -287,7 +287,6 @@ class Doc2Vec {
287
287
  let issues = [];
288
288
  let page = 1;
289
289
  const perPage = 100;
290
- const sinceTimestamp = new Date(sinceDate);
291
290
  while (true) {
292
291
  // Log progress every 10 pages to reduce noise
293
292
  if (page === 1 || page % 10 === 0) {
@@ -301,9 +300,14 @@ class Doc2Vec {
301
300
  });
302
301
  if (data.length === 0)
303
302
  break;
304
- const filtered = data.filter((issue) => new Date(issue.created_at) >= sinceTimestamp);
305
- issues = issues.concat(filtered);
306
- if (filtered.length < data.length)
303
+ // The `since` param already scopes results to issues whose
304
+ // updated_at is after sinceDate — i.e. anything created, closed,
305
+ // reopened, edited, or newly commented since the last run. Do NOT
306
+ // re-filter by created_at: an old issue that was just closed or
307
+ // received a new comment has an old created_at and would be
308
+ // dropped, leaving its status and comments permanently stale.
309
+ issues = issues.concat(data);
310
+ if (data.length < perPage)
307
311
  break;
308
312
  page++;
309
313
  }
@@ -347,6 +351,18 @@ class Doc2Vec {
347
351
  };
348
352
  const chunks = await this.contentProcessor.chunkMarkdown(markdown, issueConfig, url);
349
353
  logger.info(`Issue #${issueNumber}: Created ${chunks.length} chunks`);
354
+ // Purge the issue's existing chunks before inserting the fresh set.
355
+ // chunk_id is a content hash, so when an issue changes (closed/reopened,
356
+ // edited, new comments) the regenerated chunks get new ids and the old
357
+ // ones would otherwise linger — leaving stale state ("open") and missing
358
+ // the latest comments in search results. All chunks for an issue share
359
+ // its unique url, so delete-by-url reconciles them exactly.
360
+ if (dbConnection.type === 'sqlite') {
361
+ database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
362
+ }
363
+ else if (dbConnection.type === 'qdrant') {
364
+ await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
365
+ }
350
366
  // Process and store each chunk immediately
351
367
  for (const chunk of chunks) {
352
368
  const chunkHash = utils_1.Utils.generateHash(chunk.content);
@@ -0,0 +1 @@
1
+ @layer properties{@supports (((-webkit-hyphens:none)) and (not (margin-trim:inline))) or ((-moz-orient:inline) and (not (color:rgb(from red r g b)))){*,:before,:after,::backdrop{--tw-space-y-reverse:0;--tw-border-style:solid;--tw-leading:initial;--tw-font-weight:initial;--tw-tracking:initial;--tw-shadow:0 0 #0000;--tw-shadow-color:initial;--tw-shadow-alpha:100%;--tw-inset-shadow:0 0 #0000;--tw-inset-shadow-color:initial;--tw-inset-shadow-alpha:100%;--tw-ring-color:initial;--tw-ring-shadow:0 0 #0000;--tw-inset-ring-color:initial;--tw-inset-ring-shadow:0 0 #0000;--tw-ring-inset:initial;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-offset-shadow:0 0 #0000}}}@layer theme{:root,:host{--font-mono:ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;--color-white:#fff;--spacing:.25rem;--container-md:28rem;--container-6xl:72rem;--text-xs:.75rem;--text-xs--line-height:calc(1 / .75);--text-sm:.875rem;--text-sm--line-height:calc(1.25 / .875);--text-lg:1.125rem;--text-lg--line-height:calc(1.75 / 1.125);--text-xl:1.25rem;--text-xl--line-height:calc(1.75 / 1.25);--text-2xl:1.5rem;--text-2xl--line-height:calc(2 / 1.5);--font-weight-normal:400;--font-weight-medium:500;--font-weight-semibold:600;--tracking-tight:-.025em;--tracking-wide:.025em;--radius-md:.375rem;--radius-lg:.5rem;--animate-pulse:pulse 2s cubic-bezier(.4, 0, .6, 1) infinite;--default-transition-duration:.15s;--default-transition-timing-function:cubic-bezier(.4, 0, .2, 1);--default-font-family:system-ui, -apple-system, "Segoe UI", sans-serif;--default-mono-font-family:var(--font-mono)}}@layer base{*,:after,:before,::backdrop{box-sizing:border-box;border:0 solid;margin:0;padding:0}::file-selector-button{box-sizing:border-box;border:0 solid;margin:0;padding:0}html,:host{-webkit-text-size-adjust:100%;tab-size:4;line-height:1.5;font-family:var(--default-font-family,ui-sans-serif, system-ui, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji");font-feature-settings:var(--default-font-feature-settings,normal);font-variation-settings:var(--default-font-variation-settings,normal);-webkit-tap-highlight-color:transparent}hr{height:0;color:inherit;border-top-width:1px}abbr:where([title]){-webkit-text-decoration:underline dotted;text-decoration:underline dotted}h1,h2,h3,h4,h5,h6{font-size:inherit;font-weight:inherit}a{color:inherit;-webkit-text-decoration:inherit;text-decoration:inherit}b,strong{font-weight:bolder}code,kbd,samp,pre{font-family:var(--default-mono-font-family,ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace);font-feature-settings:var(--default-mono-font-feature-settings,normal);font-variation-settings:var(--default-mono-font-variation-settings,normal);font-size:1em}small{font-size:80%}sub,sup{vertical-align:baseline;font-size:75%;line-height:0;position:relative}sub{bottom:-.25em}sup{top:-.5em}table{text-indent:0;border-color:inherit;border-collapse:collapse}:-moz-focusring{outline:auto}progress{vertical-align:baseline}summary{display:list-item}ol,ul,menu{list-style:none}img,svg,video,canvas,audio,iframe,embed,object{vertical-align:middle;display:block}img,video{max-width:100%;height:auto}button,input,select,optgroup,textarea{font:inherit;font-feature-settings:inherit;font-variation-settings:inherit;letter-spacing:inherit;color:inherit;opacity:1;background-color:#0000;border-radius:0}::file-selector-button{font:inherit;font-feature-settings:inherit;font-variation-settings:inherit;letter-spacing:inherit;color:inherit;opacity:1;background-color:#0000;border-radius:0}:where(select:is([multiple],[size])) optgroup{font-weight:bolder}:where(select:is([multiple],[size])) optgroup option{padding-inline-start:20px}::file-selector-button{margin-inline-end:4px}::placeholder{opacity:1}@supports (not ((-webkit-appearance:-apple-pay-button))) or (contain-intrinsic-size:1px){::placeholder{color:currentColor}@supports (color:color-mix(in lab,red,red)){::placeholder{color:color-mix(in oklab,currentcolor 50%,transparent)}}}textarea{resize:vertical}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-date-and-time-value{min-height:1lh;text-align:inherit}::-webkit-datetime-edit{display:inline-flex}::-webkit-datetime-edit-fields-wrapper{padding:0}::-webkit-datetime-edit{padding-block:0}::-webkit-datetime-edit-year-field{padding-block:0}::-webkit-datetime-edit-month-field{padding-block:0}::-webkit-datetime-edit-day-field{padding-block:0}::-webkit-datetime-edit-hour-field{padding-block:0}::-webkit-datetime-edit-minute-field{padding-block:0}::-webkit-datetime-edit-second-field{padding-block:0}::-webkit-datetime-edit-millisecond-field{padding-block:0}::-webkit-datetime-edit-meridiem-field{padding-block:0}::-webkit-calendar-picker-indicator{line-height:1}:-moz-ui-invalid{box-shadow:none}button,input:where([type=button],[type=reset],[type=submit]){appearance:button}::file-selector-button{appearance:button}::-webkit-inner-spin-button{height:auto}::-webkit-outer-spin-button{height:auto}[hidden]:where(:not([hidden=until-found])){display:none!important}}@layer components;@layer utilities{.mx-auto{margin-inline:auto}.mt-0\.5{margin-top:calc(var(--spacing) * .5)}.mt-1{margin-top:var(--spacing)}.mt-1\.5{margin-top:calc(var(--spacing) * 1.5)}.mt-2{margin-top:calc(var(--spacing) * 2)}.mt-3{margin-top:calc(var(--spacing) * 3)}.-mb-px{margin-bottom:-1px}.mb-1{margin-bottom:var(--spacing)}.mb-2{margin-bottom:calc(var(--spacing) * 2)}.mb-3{margin-bottom:calc(var(--spacing) * 3)}.mb-4{margin-bottom:calc(var(--spacing) * 4)}.ml-1{margin-left:var(--spacing)}.ml-2{margin-left:calc(var(--spacing) * 2)}.ml-auto{margin-left:auto}.block{display:block}.flex{display:flex}.grid{display:grid}.hidden{display:none}.inline-block{display:inline-block}.inline-flex{display:inline-flex}.h-2{height:calc(var(--spacing) * 2)}.h-4{height:calc(var(--spacing) * 4)}.max-h-\[560px\]{max-height:560px}.min-h-screen{min-height:100vh}.w-2{width:calc(var(--spacing) * 2)}.w-4{width:calc(var(--spacing) * 4)}.w-10{width:calc(var(--spacing) * 10)}.w-44{width:calc(var(--spacing) * 44)}.w-72{width:calc(var(--spacing) * 72)}.w-full{width:100%}.max-w-6xl{max-width:var(--container-6xl)}.max-w-md{max-width:var(--container-md)}.min-w-48{min-width:calc(var(--spacing) * 48)}.flex-1{flex:1}.shrink-0{flex-shrink:0}.animate-pulse{animation:var(--animate-pulse)}.grid-cols-2{grid-template-columns:repeat(2,minmax(0,1fr))}.flex-wrap{flex-wrap:wrap}.items-baseline{align-items:baseline}.items-center{align-items:center}.items-start{align-items:flex-start}.justify-between{justify-content:space-between}.gap-1{gap:var(--spacing)}.gap-1\.5{gap:calc(var(--spacing) * 1.5)}.gap-2{gap:calc(var(--spacing) * 2)}.gap-3{gap:calc(var(--spacing) * 3)}.gap-4{gap:calc(var(--spacing) * 4)}.gap-6{gap:calc(var(--spacing) * 6)}:where(.space-y-3>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 3) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 3) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-4>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 4) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 4) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-6>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 6) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 6) * calc(1 - var(--tw-space-y-reverse)))}.gap-x-4{column-gap:calc(var(--spacing) * 4)}.gap-x-6{column-gap:calc(var(--spacing) * 6)}.gap-x-8{column-gap:calc(var(--spacing) * 8)}.gap-y-2{row-gap:calc(var(--spacing) * 2)}.gap-y-3{row-gap:calc(var(--spacing) * 3)}.self-center{align-self:center}.truncate{text-overflow:ellipsis;white-space:nowrap;overflow:hidden}.overflow-auto{overflow:auto}.overflow-hidden{overflow:hidden}.overflow-x-auto{overflow-x:auto}.rounded{border-radius:.25rem}.rounded-full{border-radius:3.40282e38px}.rounded-lg{border-radius:var(--radius-lg)}.rounded-md{border-radius:var(--radius-md)}.border{border-style:var(--tw-border-style);border-width:1px}.border-b{border-bottom-style:var(--tw-border-style);border-bottom-width:1px}.border-b-2{border-bottom-style:var(--tw-border-style);border-bottom-width:2px}.border-dashed{--tw-border-style:dashed;border-style:dashed}.border-accent,.border-accent\/30{border-color:var(--accent)}@supports (color:color-mix(in lab,red,red)){.border-accent\/30{border-color:color-mix(in oklab,var(--accent) 30%,transparent)}}.border-accent\/40{border-color:var(--accent)}@supports (color:color-mix(in lab,red,red)){.border-accent\/40{border-color:color-mix(in oklab,var(--accent) 40%,transparent)}}.border-critical\/40{border-color:var(--status-critical)}@supports (color:color-mix(in lab,red,red)){.border-critical\/40{border-color:color-mix(in oklab,var(--status-critical) 40%,transparent)}}.border-critical\/50{border-color:var(--status-critical)}@supports (color:color-mix(in lab,red,red)){.border-critical\/50{border-color:color-mix(in oklab,var(--status-critical) 50%,transparent)}}.border-current{border-color:currentColor}.border-edge,.border-edge\/60{border-color:var(--border)}@supports (color:color-mix(in lab,red,red)){.border-edge\/60{border-color:color-mix(in oklab,var(--border) 60%,transparent)}}.border-good\/40{border-color:var(--status-good)}@supports (color:color-mix(in lab,red,red)){.border-good\/40{border-color:color-mix(in oklab,var(--status-good) 40%,transparent)}}.border-transparent{border-color:#0000}.border-warning\/40{border-color:var(--status-warning)}@supports (color:color-mix(in lab,red,red)){.border-warning\/40{border-color:color-mix(in oklab,var(--status-warning) 40%,transparent)}}.border-warning\/50{border-color:var(--status-warning)}@supports (color:color-mix(in lab,red,red)){.border-warning\/50{border-color:color-mix(in oklab,var(--status-warning) 50%,transparent)}}.bg-accent{background-color:var(--accent)}.bg-page{background-color:var(--page)}.bg-surface{background-color:var(--surface-1)}.p-4{padding:calc(var(--spacing) * 4)}.px-1{padding-inline:var(--spacing)}.px-2{padding-inline:calc(var(--spacing) * 2)}.px-2\.5{padding-inline:calc(var(--spacing) * 2.5)}.px-3{padding-inline:calc(var(--spacing) * 3)}.px-3\.5{padding-inline:calc(var(--spacing) * 3.5)}.px-4{padding-inline:calc(var(--spacing) * 4)}.px-6{padding-inline:calc(var(--spacing) * 6)}.py-0\.5{padding-block:calc(var(--spacing) * .5)}.py-1{padding-block:var(--spacing)}.py-1\.5{padding-block:calc(var(--spacing) * 1.5)}.py-2{padding-block:calc(var(--spacing) * 2)}.py-2\.5{padding-block:calc(var(--spacing) * 2.5)}.py-3{padding-block:calc(var(--spacing) * 3)}.py-4{padding-block:calc(var(--spacing) * 4)}.py-8{padding-block:calc(var(--spacing) * 8)}.py-10{padding-block:calc(var(--spacing) * 10)}.pr-4{padding-right:calc(var(--spacing) * 4)}.pb-2{padding-bottom:calc(var(--spacing) * 2)}.text-center{text-align:center}.text-left{text-align:left}.font-mono{font-family:var(--font-mono)}.text-2xl{font-size:var(--text-2xl);line-height:var(--tw-leading,var(--text-2xl--line-height))}.text-lg{font-size:var(--text-lg);line-height:var(--tw-leading,var(--text-lg--line-height))}.text-sm{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height))}.text-xl{font-size:var(--text-xl);line-height:var(--tw-leading,var(--text-xl--line-height))}.text-xs{font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height))}.text-\[10px\]{font-size:10px}.leading-5{--tw-leading:calc(var(--spacing) * 5);line-height:calc(var(--spacing) * 5)}.font-medium{--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium)}.font-normal{--tw-font-weight:var(--font-weight-normal);font-weight:var(--font-weight-normal)}.font-semibold{--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold)}.tracking-tight{--tw-tracking:var(--tracking-tight);letter-spacing:var(--tracking-tight)}.tracking-wide{--tw-tracking:var(--tracking-wide);letter-spacing:var(--tracking-wide)}.break-all{word-break:break-all}.whitespace-pre-wrap{white-space:pre-wrap}.text-accent{color:var(--accent)}.text-critical{color:var(--status-critical)}.text-good-text{color:var(--good-text)}.text-ink{color:var(--text-primary)}.text-ink-muted{color:var(--text-muted)}.text-ink-secondary{color:var(--text-secondary)}.text-warning{color:var(--status-warning)}.text-white{color:var(--color-white)}.capitalize{text-transform:capitalize}.uppercase{text-transform:uppercase}.underline-offset-2{text-underline-offset:2px}.accent-\(--accent\){accent-color:var(--accent)}.shadow-sm{--tw-shadow:0 1px 3px 0 var(--tw-shadow-color,#0000001a), 0 1px 2px -1px var(--tw-shadow-color,#0000001a);box-shadow:var(--tw-inset-shadow),var(--tw-inset-ring-shadow),var(--tw-ring-offset-shadow),var(--tw-ring-shadow),var(--tw-shadow)}.transition{transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to,opacity,box-shadow,transform,translate,scale,rotate,filter,-webkit-backdrop-filter,backdrop-filter,display,content-visibility,overlay,pointer-events;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.outline-none{--tw-outline-style:none;outline-style:none}.select-none{-webkit-user-select:none;user-select:none}.placeholder\:text-ink-muted\/60::placeholder{color:var(--text-muted)}@supports (color:color-mix(in lab,red,red)){.placeholder\:text-ink-muted\/60::placeholder{color:color-mix(in oklab,var(--text-muted) 60%,transparent)}}.last\:border-0:last-child{border-style:var(--tw-border-style);border-width:0}@media(hover:hover){.hover\:border-accent:hover,.hover\:border-accent\/50:hover{border-color:var(--accent)}@supports (color:color-mix(in lab,red,red)){.hover\:border-accent\/50:hover{border-color:color-mix(in oklab,var(--accent) 50%,transparent)}}.hover\:bg-critical\/10:hover{background-color:var(--status-critical)}@supports (color:color-mix(in lab,red,red)){.hover\:bg-critical\/10:hover{background-color:color-mix(in oklab,var(--status-critical) 10%,transparent)}}.hover\:bg-ink\/\[0\.03\]:hover{background-color:var(--text-primary)}@supports (color:color-mix(in lab,red,red)){.hover\:bg-ink\/\[0\.03\]:hover{background-color:color-mix(in oklab,var(--text-primary) 3%,transparent)}}.hover\:text-accent:hover{color:var(--accent)}.hover\:text-critical:hover{color:var(--status-critical)}.hover\:text-ink-secondary:hover{color:var(--text-secondary)}.hover\:underline:hover{text-decoration-line:underline}.hover\:opacity-90:hover{opacity:.9}}.focus\:border-accent:focus{border-color:var(--accent)}.disabled\:cursor-not-allowed:disabled{cursor:not-allowed}.disabled\:opacity-40:disabled{opacity:.4}@media(min-width:40rem){.sm\:col-span-2{grid-column:span 2/span 2}.sm\:grid-cols-2{grid-template-columns:repeat(2,minmax(0,1fr))}}@media(min-width:48rem){.md\:grid-cols-4{grid-template-columns:repeat(4,minmax(0,1fr))}}}:root{--page:#f9f9f7;--surface-1:#fcfcfb;--text-primary:#0b0b0b;--text-secondary:#52514e;--text-muted:#898781;--gridline:#e1e0d9;--baseline:#c3c2b7;--border:#0b0b0b1a;--series-1:#2a78d6;--status-good:#0ca30c;--status-warning:#fab219;--status-critical:#d03b3b;--good-text:#006300;--accent:#2a78d6}@media(prefers-color-scheme:dark){:root{--page:#0d0d0d;--surface-1:#1a1a19;--text-primary:#fff;--text-secondary:#c3c2b7;--text-muted:#898781;--gridline:#2c2c2a;--baseline:#383835;--border:#ffffff1a;--series-1:#3987e5;--status-good:#0ca30c;--status-warning:#fab219;--status-critical:#d03b3b;--good-text:#0ca30c;--accent:#3987e5}}html{background:var(--page);color:var(--text-primary);font-family:system-ui,-apple-system,Segoe UI,sans-serif}.log-scroll::-webkit-scrollbar{width:10px;height:10px}.log-scroll::-webkit-scrollbar-thumb{background:var(--baseline);border-radius:5px}@property --tw-space-y-reverse{syntax:"*";inherits:false;initial-value:0}@property --tw-border-style{syntax:"*";inherits:false;initial-value:solid}@property --tw-leading{syntax:"*";inherits:false}@property --tw-font-weight{syntax:"*";inherits:false}@property --tw-tracking{syntax:"*";inherits:false}@property --tw-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-shadow-color{syntax:"*";inherits:false}@property --tw-shadow-alpha{syntax:"<percentage>";inherits:false;initial-value:100%}@property --tw-inset-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-inset-shadow-color{syntax:"*";inherits:false}@property --tw-inset-shadow-alpha{syntax:"<percentage>";inherits:false;initial-value:100%}@property --tw-ring-color{syntax:"*";inherits:false}@property --tw-ring-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-inset-ring-color{syntax:"*";inherits:false}@property --tw-inset-ring-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-ring-inset{syntax:"*";inherits:false}@property --tw-ring-offset-width{syntax:"<length>";inherits:false;initial-value:0}@property --tw-ring-offset-color{syntax:"*";inherits:false;initial-value:#fff}@property --tw-ring-offset-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@keyframes pulse{50%{opacity:.5}}