@telepat/snoopy 0.1.10 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -10
- package/dist/src/cli/commands/consume.d.ts +7 -0
- package/dist/src/cli/commands/consume.js +60 -0
- package/dist/src/cli/index.js +11 -0
- package/dist/src/services/db/repositories/runsRepo.d.ts +4 -0
- package/dist/src/services/db/repositories/runsRepo.js +27 -10
- package/dist/src/services/db/repositories/scanItemsRepo.d.ts +12 -0
- package/dist/src/services/db/repositories/scanItemsRepo.js +116 -10
- package/dist/src/services/db/sqlite.js +9 -0
- package/dist/src/services/scheduler/jobRunner.d.ts +1 -1
- package/dist/src/services/scheduler/jobRunner.js +77 -39
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -1,10 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
└─┘┘└┘└─┘└─┘┴ ┴
|
|
5
|
-
```
|
|
6
|
-
|
|
7
|
-
# Monitor Reddit Conversations With AI
|
|
1
|
+
<p align="center"><img src="./snoopy-logo.webp" width="128" alt="Snoopy"></p>
|
|
2
|
+
<h1 align="center">Snoopy</h1>
|
|
3
|
+
<p align="center"><em>Sniff out the conversations that matter.</em></p>
|
|
8
4
|
|
|
9
5
|
[](https://github.com/telepat-io/snoopy/actions/workflows/ci.yml)
|
|
10
6
|
[](https://codecov.io/gh/telepat-io/snoopy)
|
|
@@ -126,6 +122,9 @@ snoopy job run --limit 5
|
|
|
126
122
|
snoopy job run <jobRef> --limit 5
|
|
127
123
|
```
|
|
128
124
|
|
|
125
|
+
If another run is already active for the same job, Snoopy marks the new attempt as `skipped` with an `already active` message.
|
|
126
|
+
Duplicate candidates matching an existing scan item are treated as already scanned and do not fail the run.
|
|
127
|
+
|
|
129
128
|
If `<jobRef>` is omitted for `job run`, `job enable`, `job disable`, `job delete`, `start`, `stop`, `errors`, or `results`, Snoopy shows your job list and lets you pick with up/down arrows and Enter.
|
|
130
129
|
|
|
131
130
|
4. View run history:
|
|
@@ -156,7 +155,15 @@ snoopy export
|
|
|
156
155
|
snoopy export <jobRef> --json --last-run
|
|
157
156
|
```
|
|
158
157
|
|
|
159
|
-
8.
|
|
158
|
+
8. Consume unconsumed qualified results (read-once, most recent first):
|
|
159
|
+
|
|
160
|
+
```bash
|
|
161
|
+
snoopy consume
|
|
162
|
+
snoopy consume <jobRef> --limit 10
|
|
163
|
+
snoopy consume <jobRef> --json --dry-run
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
9. Inspect one run's detailed log output:
|
|
160
167
|
|
|
161
168
|
```bash
|
|
162
169
|
snoopy logs
|
|
@@ -166,13 +173,13 @@ snoopy logs <runId> --raw
|
|
|
166
173
|
|
|
167
174
|
When `runId` is omitted for `logs`, Snoopy first prompts for a job, then prompts for a run from that job (up/down arrows + Enter).
|
|
168
175
|
|
|
169
|
-
|
|
176
|
+
10. Show recent errors for one job:
|
|
170
177
|
|
|
171
178
|
```bash
|
|
172
179
|
snoopy errors <jobRef>
|
|
173
180
|
```
|
|
174
181
|
|
|
175
|
-
|
|
182
|
+
11. Enable daemon mode:
|
|
176
183
|
|
|
177
184
|
```bash
|
|
178
185
|
snoopy daemon start
|
|
@@ -188,6 +195,7 @@ snoopy daemon reload
|
|
|
188
195
|
- `analytics [jobRef] --days <N>`
|
|
189
196
|
- `results [jobRef]`
|
|
190
197
|
- `export [jobRef] --csv|--json [--last-run]`
|
|
198
|
+
- `consume [jobRef] [--limit <N>] [--json] [--dry-run]`
|
|
191
199
|
- `logs [runId]`
|
|
192
200
|
- `errors [jobRef] --hours <N>`
|
|
193
201
|
- `start [jobRef]` / `stop [jobRef]`
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { JobsRepository } from '../../services/db/repositories/jobsRepo.js';
|
|
2
|
+
import { ScanItemsRepository } from '../../services/db/repositories/scanItemsRepo.js';
|
|
3
|
+
import { printCommandScreen, printError, printInfo, printKeyValue, printMuted, printSuccess, printWarning } from '../ui/consoleUi.js';
|
|
4
|
+
function truncateContent(content, maxLength) {
|
|
5
|
+
if (content.length <= maxLength) {
|
|
6
|
+
return content;
|
|
7
|
+
}
|
|
8
|
+
return `${content.slice(0, maxLength)}...`;
|
|
9
|
+
}
|
|
10
|
+
function renderResultRow(row, index) {
|
|
11
|
+
printInfo(`#${index + 1}`);
|
|
12
|
+
printKeyValue('Job', row.jobId);
|
|
13
|
+
printKeyValue('Author', row.author);
|
|
14
|
+
if (row.title) {
|
|
15
|
+
printKeyValue('Title', row.title);
|
|
16
|
+
}
|
|
17
|
+
printKeyValue('URL', row.url);
|
|
18
|
+
printKeyValue('Date', row.redditPostedAt);
|
|
19
|
+
if (row.qualificationReason) {
|
|
20
|
+
printKeyValue('Reason', truncateContent(row.qualificationReason, 120));
|
|
21
|
+
}
|
|
22
|
+
printMuted(truncateContent(row.body, 200));
|
|
23
|
+
console.log('');
|
|
24
|
+
}
|
|
25
|
+
export function consumeResults(jobRef, options = {}) {
|
|
26
|
+
printCommandScreen('Consume results', 'List and consume unconsumed qualified results');
|
|
27
|
+
const jobsRepo = new JobsRepository();
|
|
28
|
+
const scanItemsRepo = new ScanItemsRepository();
|
|
29
|
+
let jobId;
|
|
30
|
+
if (jobRef) {
|
|
31
|
+
const job = jobsRepo.getByRef(jobRef);
|
|
32
|
+
if (!job) {
|
|
33
|
+
printError(`Job not found: ${jobRef}`);
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
jobId = job.id;
|
|
37
|
+
}
|
|
38
|
+
const rows = scanItemsRepo.listUnconsumedQualified(jobId, options.limit);
|
|
39
|
+
if (rows.length === 0) {
|
|
40
|
+
if (options.json) {
|
|
41
|
+
console.log(JSON.stringify([], null, 2));
|
|
42
|
+
return;
|
|
43
|
+
}
|
|
44
|
+
printWarning('No unconsumed qualified results found.');
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
if (options.json) {
|
|
48
|
+
console.log(JSON.stringify(rows, null, 2));
|
|
49
|
+
}
|
|
50
|
+
else {
|
|
51
|
+
rows.forEach((row, index) => renderResultRow(row, index));
|
|
52
|
+
}
|
|
53
|
+
if (options.dryRun) {
|
|
54
|
+
printSuccess(`Found ${rows.length} result(s) (dry run — not consumed).`);
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
const consumedCount = scanItemsRepo.markConsumed(rows.map((r) => r.id));
|
|
58
|
+
printSuccess(`Consumed ${consumedCount} result(s).`);
|
|
59
|
+
}
|
|
60
|
+
//# sourceMappingURL=consume.js.map
|
package/dist/src/cli/index.js
CHANGED
|
@@ -7,6 +7,7 @@ import { daemonReload, daemonRun, daemonStart, daemonStatus, daemonStop } from '
|
|
|
7
7
|
import { showRunLogs } from './commands/logs.js';
|
|
8
8
|
import { showJobErrors } from './commands/errors.js';
|
|
9
9
|
import { exportCsv } from './commands/export.js';
|
|
10
|
+
import { consumeResults } from './commands/consume.js';
|
|
10
11
|
import { showAnalytics } from './commands/analytics.js';
|
|
11
12
|
import { showResults } from './commands/results.js';
|
|
12
13
|
import { disableStartupCommand, enableStartupCommand, installStartupCommand, startupStatusCommand, uninstallStartupCommand } from './commands/startup.js';
|
|
@@ -142,6 +143,16 @@ program
|
|
|
142
143
|
.action((jobRef, options) => {
|
|
143
144
|
exportCsv(jobRef, options);
|
|
144
145
|
});
|
|
146
|
+
program
|
|
147
|
+
.command('consume')
|
|
148
|
+
.argument('[jobRef]', 'Optional job ID or slug')
|
|
149
|
+
.description('List and mark unconsumed qualified results')
|
|
150
|
+
.option('--limit <count>', 'Maximum number of results to consume', parsePositiveInteger)
|
|
151
|
+
.option('--json', 'Output raw JSON array to stdout')
|
|
152
|
+
.option('--dry-run', 'Preview results without marking them consumed')
|
|
153
|
+
.action((jobRef, options) => {
|
|
154
|
+
consumeResults(jobRef, options);
|
|
155
|
+
});
|
|
145
156
|
program.parseAsync(process.argv).catch((error) => {
|
|
146
157
|
console.error(`Error: ${String(error)}`);
|
|
147
158
|
process.exit(1);
|
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
import crypto from 'node:crypto';
|
|
2
2
|
import { getDb } from '../sqlite.js';
|
|
3
|
+
export class ActiveRunConflictError extends Error {
|
|
4
|
+
jobId;
|
|
5
|
+
constructor(jobId) {
|
|
6
|
+
super(`A run is already active for job ${jobId}.`);
|
|
7
|
+
this.name = 'ActiveRunConflictError';
|
|
8
|
+
this.jobId = jobId;
|
|
9
|
+
}
|
|
10
|
+
}
|
|
3
11
|
export class RunsRepository {
|
|
4
12
|
db = getDb();
|
|
5
13
|
runSelectWithJob = `SELECT
|
|
@@ -41,16 +49,25 @@ export class RunsRepository {
|
|
|
41
49
|
}
|
|
42
50
|
startRun(jobId, logFilePath) {
|
|
43
51
|
const id = crypto.randomUUID();
|
|
44
|
-
|
|
45
|
-
.
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
52
|
+
try {
|
|
53
|
+
this.db
|
|
54
|
+
.prepare(`INSERT INTO job_runs (
|
|
55
|
+
id,
|
|
56
|
+
job_id,
|
|
57
|
+
status,
|
|
58
|
+
started_at,
|
|
59
|
+
created_at,
|
|
60
|
+
log_file_path
|
|
61
|
+
) VALUES (?, ?, 'running', datetime('now'), datetime('now'), ?)`)
|
|
62
|
+
.run(id, jobId, logFilePath ?? null);
|
|
63
|
+
}
|
|
64
|
+
catch (error) {
|
|
65
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
66
|
+
if (message.includes('idx_job_runs_active_job') || message.includes('UNIQUE constraint failed: job_runs.job_id')) {
|
|
67
|
+
throw new ActiveRunConflictError(jobId);
|
|
68
|
+
}
|
|
69
|
+
throw error;
|
|
70
|
+
}
|
|
54
71
|
return id;
|
|
55
72
|
}
|
|
56
73
|
completeRun(runId, stats) {
|
|
@@ -15,6 +15,7 @@ export interface NewScanItem {
|
|
|
15
15
|
viewed?: boolean;
|
|
16
16
|
validated?: boolean;
|
|
17
17
|
processed?: boolean;
|
|
18
|
+
consumed?: boolean;
|
|
18
19
|
promptTokens?: number;
|
|
19
20
|
completionTokens?: number;
|
|
20
21
|
estimatedCostUsd?: number | null;
|
|
@@ -46,6 +47,7 @@ export interface QualifiedScanItemRow {
|
|
|
46
47
|
viewed: boolean;
|
|
47
48
|
validated: boolean;
|
|
48
49
|
processed: boolean;
|
|
50
|
+
consumed: boolean;
|
|
49
51
|
qualificationReason: string | null;
|
|
50
52
|
createdAt: string;
|
|
51
53
|
}
|
|
@@ -66,6 +68,7 @@ export interface ScanItemRow {
|
|
|
66
68
|
viewed: boolean;
|
|
67
69
|
validated: boolean;
|
|
68
70
|
processed: boolean;
|
|
71
|
+
consumed: boolean;
|
|
69
72
|
qualificationReason: string | null;
|
|
70
73
|
promptTokens: number;
|
|
71
74
|
completionTokens: number;
|
|
@@ -87,12 +90,18 @@ export interface AnalyticsByJobRow extends AnalyticsTotalsRow {
|
|
|
87
90
|
jobName: string;
|
|
88
91
|
jobSlug: string;
|
|
89
92
|
}
|
|
93
|
+
export interface CreateScanItemResult {
|
|
94
|
+
id: string;
|
|
95
|
+
inserted: boolean;
|
|
96
|
+
}
|
|
90
97
|
interface AnalyticsFilter {
|
|
91
98
|
jobId?: string;
|
|
92
99
|
days: number;
|
|
93
100
|
}
|
|
94
101
|
export declare class ScanItemsRepository {
|
|
95
102
|
private readonly db;
|
|
103
|
+
private isDedupConflict;
|
|
104
|
+
private findExistingScanItemId;
|
|
96
105
|
private mapScanItemRows;
|
|
97
106
|
private buildFilterClause;
|
|
98
107
|
private toAnalyticsTotalsRow;
|
|
@@ -107,6 +116,9 @@ export declare class ScanItemsRepository {
|
|
|
107
116
|
listAnalyticsBySubreddit(filter: AnalyticsFilter): AnalyticsBySubredditRow[];
|
|
108
117
|
listAnalyticsByJob(days: number): AnalyticsByJobRow[];
|
|
109
118
|
existsComment(jobId: string, postId: string, commentId: string): boolean;
|
|
119
|
+
listUnconsumedQualified(jobId?: string, limit?: number): QualifiedScanItemRow[];
|
|
120
|
+
markConsumed(ids: string[]): number;
|
|
121
|
+
createWithStatus(item: NewScanItem): CreateScanItemResult;
|
|
110
122
|
create(item: NewScanItem): string;
|
|
111
123
|
listCommentThreadNodes(scanItemId: string): CommentThreadNodeRow[];
|
|
112
124
|
}
|
|
@@ -2,13 +2,37 @@ import crypto from 'node:crypto';
|
|
|
2
2
|
import { getDb } from '../sqlite.js';
|
|
3
3
|
export class ScanItemsRepository {
|
|
4
4
|
db = getDb();
|
|
5
|
+
isDedupConflict(error) {
|
|
6
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
7
|
+
return message.includes('idx_scan_items_dedup');
|
|
8
|
+
}
|
|
9
|
+
findExistingScanItemId(jobId, postId, commentId) {
|
|
10
|
+
const query = commentId === null
|
|
11
|
+
? `SELECT id
|
|
12
|
+
FROM scan_items
|
|
13
|
+
WHERE job_id = ?
|
|
14
|
+
AND reddit_post_id = ?
|
|
15
|
+
AND reddit_comment_id IS NULL
|
|
16
|
+
LIMIT 1`
|
|
17
|
+
: `SELECT id
|
|
18
|
+
FROM scan_items
|
|
19
|
+
WHERE job_id = ?
|
|
20
|
+
AND reddit_post_id = ?
|
|
21
|
+
AND reddit_comment_id = ?
|
|
22
|
+
LIMIT 1`;
|
|
23
|
+
const row = commentId === null
|
|
24
|
+
? this.db.prepare(query).get(jobId, postId)
|
|
25
|
+
: this.db.prepare(query).get(jobId, postId, commentId);
|
|
26
|
+
return row?.id ?? null;
|
|
27
|
+
}
|
|
5
28
|
mapScanItemRows(rows) {
|
|
6
29
|
return rows.map((row) => ({
|
|
7
30
|
...row,
|
|
8
31
|
qualified: row.qualified === 1,
|
|
9
32
|
viewed: row.viewed === 1,
|
|
10
33
|
validated: row.validated === 1,
|
|
11
|
-
processed: row.processed === 1
|
|
34
|
+
processed: row.processed === 1,
|
|
35
|
+
consumed: row.consumed === 1
|
|
12
36
|
}));
|
|
13
37
|
}
|
|
14
38
|
buildFilterClause(alias, filter) {
|
|
@@ -47,6 +71,7 @@ export class ScanItemsRepository {
|
|
|
47
71
|
viewed,
|
|
48
72
|
validated,
|
|
49
73
|
processed,
|
|
74
|
+
consumed,
|
|
50
75
|
qualification_reason as qualificationReason,
|
|
51
76
|
created_at as createdAt
|
|
52
77
|
FROM scan_items
|
|
@@ -59,7 +84,8 @@ export class ScanItemsRepository {
|
|
|
59
84
|
...row,
|
|
60
85
|
viewed: row.viewed === 1,
|
|
61
86
|
validated: row.validated === 1,
|
|
62
|
-
processed: row.processed === 1
|
|
87
|
+
processed: row.processed === 1,
|
|
88
|
+
consumed: row.consumed === 1
|
|
63
89
|
}));
|
|
64
90
|
}
|
|
65
91
|
listQualifiedByJobRun(jobId, runId, limit = 100) {
|
|
@@ -77,6 +103,7 @@ export class ScanItemsRepository {
|
|
|
77
103
|
viewed,
|
|
78
104
|
validated,
|
|
79
105
|
processed,
|
|
106
|
+
consumed,
|
|
80
107
|
qualification_reason as qualificationReason,
|
|
81
108
|
created_at as createdAt
|
|
82
109
|
FROM scan_items
|
|
@@ -90,7 +117,8 @@ export class ScanItemsRepository {
|
|
|
90
117
|
...row,
|
|
91
118
|
viewed: row.viewed === 1,
|
|
92
119
|
validated: row.validated === 1,
|
|
93
|
-
processed: row.processed === 1
|
|
120
|
+
processed: row.processed === 1,
|
|
121
|
+
consumed: row.consumed === 1
|
|
94
122
|
}));
|
|
95
123
|
}
|
|
96
124
|
listByJob(jobId) {
|
|
@@ -112,6 +140,7 @@ export class ScanItemsRepository {
|
|
|
112
140
|
viewed,
|
|
113
141
|
validated,
|
|
114
142
|
processed,
|
|
143
|
+
consumed,
|
|
115
144
|
qualification_reason as qualificationReason,
|
|
116
145
|
prompt_tokens as promptTokens,
|
|
117
146
|
completion_tokens as completionTokens,
|
|
@@ -152,6 +181,7 @@ export class ScanItemsRepository {
|
|
|
152
181
|
viewed,
|
|
153
182
|
validated,
|
|
154
183
|
processed,
|
|
184
|
+
consumed,
|
|
155
185
|
qualification_reason as qualificationReason,
|
|
156
186
|
prompt_tokens as promptTokens,
|
|
157
187
|
completion_tokens as completionTokens,
|
|
@@ -260,11 +290,61 @@ export class ScanItemsRepository {
|
|
|
260
290
|
.get(jobId, postId, commentId);
|
|
261
291
|
return Boolean(row);
|
|
262
292
|
}
|
|
263
|
-
|
|
293
|
+
listUnconsumedQualified(jobId, limit) {
|
|
294
|
+
const hasJobId = Boolean(jobId);
|
|
295
|
+
const hasLimit = limit !== undefined && limit !== null;
|
|
296
|
+
const boundedLimit = hasLimit ? Math.max(1, Math.floor(limit)) : undefined;
|
|
297
|
+
let query = `SELECT
|
|
298
|
+
id,
|
|
299
|
+
job_id as jobId,
|
|
300
|
+
run_id as runId,
|
|
301
|
+
author,
|
|
302
|
+
title,
|
|
303
|
+
body,
|
|
304
|
+
url,
|
|
305
|
+
reddit_posted_at as redditPostedAt,
|
|
306
|
+
viewed,
|
|
307
|
+
validated,
|
|
308
|
+
processed,
|
|
309
|
+
consumed,
|
|
310
|
+
qualification_reason as qualificationReason,
|
|
311
|
+
created_at as createdAt
|
|
312
|
+
FROM scan_items
|
|
313
|
+
WHERE qualified = 1 AND consumed = 0`;
|
|
314
|
+
const params = [];
|
|
315
|
+
if (hasJobId) {
|
|
316
|
+
query += ' AND job_id = ?';
|
|
317
|
+
params.push(jobId);
|
|
318
|
+
}
|
|
319
|
+
query += ' ORDER BY datetime(created_at) DESC, id DESC';
|
|
320
|
+
if (boundedLimit !== undefined) {
|
|
321
|
+
query += ' LIMIT ?';
|
|
322
|
+
params.push(boundedLimit);
|
|
323
|
+
}
|
|
324
|
+
const rows = this.db.prepare(query).all(...params);
|
|
325
|
+
return rows.map((row) => ({
|
|
326
|
+
...row,
|
|
327
|
+
viewed: row.viewed === 1,
|
|
328
|
+
validated: row.validated === 1,
|
|
329
|
+
processed: row.processed === 1,
|
|
330
|
+
consumed: row.consumed === 1
|
|
331
|
+
}));
|
|
332
|
+
}
|
|
333
|
+
markConsumed(ids) {
|
|
334
|
+
if (ids.length === 0) {
|
|
335
|
+
return 0;
|
|
336
|
+
}
|
|
337
|
+
const placeholders = ids.map(() => '?').join(',');
|
|
338
|
+
const result = this.db
|
|
339
|
+
.prepare(`UPDATE scan_items SET consumed = 1 WHERE id IN (${placeholders})`)
|
|
340
|
+
.run(...ids);
|
|
341
|
+
return Number(result.changes);
|
|
342
|
+
}
|
|
343
|
+
createWithStatus(item) {
|
|
264
344
|
const id = crypto.randomUUID();
|
|
265
345
|
const createInTransaction = this.db.transaction((newId, newItem) => {
|
|
266
|
-
this.db
|
|
267
|
-
.prepare(`INSERT INTO scan_items (
|
|
346
|
+
const insertResult = this.db
|
|
347
|
+
.prepare(`INSERT OR IGNORE INTO scan_items (
|
|
268
348
|
id,
|
|
269
349
|
job_id,
|
|
270
350
|
run_id,
|
|
@@ -281,13 +361,21 @@ export class ScanItemsRepository {
|
|
|
281
361
|
viewed,
|
|
282
362
|
validated,
|
|
283
363
|
processed,
|
|
364
|
+
consumed,
|
|
284
365
|
prompt_tokens,
|
|
285
366
|
completion_tokens,
|
|
286
367
|
estimated_cost_usd,
|
|
287
368
|
qualification_reason,
|
|
288
369
|
created_at
|
|
289
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))`)
|
|
290
|
-
.run(newId, newItem.jobId, newItem.runId, newItem.type, newItem.redditPostId, newItem.redditCommentId, newItem.subreddit, newItem.author, newItem.title, newItem.body, newItem.url, newItem.redditPostedAt, newItem.qualified ? 1 : 0, newItem.viewed ? 1 : 0, newItem.validated ? 1 : 0, newItem.processed ? 1 : 0, newItem.promptTokens ?? 0, newItem.completionTokens ?? 0, newItem.estimatedCostUsd ?? null, newItem.qualificationReason);
|
|
370
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))`)
|
|
371
|
+
.run(newId, newItem.jobId, newItem.runId, newItem.type, newItem.redditPostId, newItem.redditCommentId, newItem.subreddit, newItem.author, newItem.title, newItem.body, newItem.url, newItem.redditPostedAt, newItem.qualified ? 1 : 0, newItem.viewed ? 1 : 0, newItem.validated ? 1 : 0, newItem.processed ? 1 : 0, newItem.consumed ? 1 : 0, newItem.promptTokens ?? 0, newItem.completionTokens ?? 0, newItem.estimatedCostUsd ?? null, newItem.qualificationReason);
|
|
372
|
+
if (insertResult.changes === 0) {
|
|
373
|
+
const existingId = this.findExistingScanItemId(newItem.jobId, newItem.redditPostId, newItem.redditCommentId);
|
|
374
|
+
return {
|
|
375
|
+
id: existingId ?? newId,
|
|
376
|
+
inserted: false
|
|
377
|
+
};
|
|
378
|
+
}
|
|
291
379
|
for (const node of newItem.commentThreadNodes ?? []) {
|
|
292
380
|
this.db
|
|
293
381
|
.prepare(`INSERT INTO comment_thread_nodes (
|
|
@@ -303,9 +391,27 @@ export class ScanItemsRepository {
|
|
|
303
391
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))`)
|
|
304
392
|
.run(crypto.randomUUID(), newId, node.redditCommentId, node.parentRedditCommentId, node.author, node.body, node.depth, node.isTarget ? 1 : 0);
|
|
305
393
|
}
|
|
394
|
+
return {
|
|
395
|
+
id: newId,
|
|
396
|
+
inserted: true
|
|
397
|
+
};
|
|
306
398
|
});
|
|
307
|
-
|
|
308
|
-
|
|
399
|
+
try {
|
|
400
|
+
return createInTransaction(id, item);
|
|
401
|
+
}
|
|
402
|
+
catch (error) {
|
|
403
|
+
if (!this.isDedupConflict(error)) {
|
|
404
|
+
throw error;
|
|
405
|
+
}
|
|
406
|
+
const existingId = this.findExistingScanItemId(item.jobId, item.redditPostId, item.redditCommentId);
|
|
407
|
+
return {
|
|
408
|
+
id: existingId ?? id,
|
|
409
|
+
inserted: false
|
|
410
|
+
};
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
create(item) {
|
|
414
|
+
return this.createWithStatus(item).id;
|
|
309
415
|
}
|
|
310
416
|
listCommentThreadNodes(scanItemId) {
|
|
311
417
|
const rows = this.db
|
|
@@ -68,6 +68,7 @@ export function getDb() {
|
|
|
68
68
|
viewed INTEGER NOT NULL DEFAULT 0,
|
|
69
69
|
validated INTEGER NOT NULL DEFAULT 0,
|
|
70
70
|
processed INTEGER NOT NULL DEFAULT 0,
|
|
71
|
+
consumed INTEGER NOT NULL DEFAULT 0,
|
|
71
72
|
prompt_tokens INTEGER NOT NULL DEFAULT 0,
|
|
72
73
|
completion_tokens INTEGER NOT NULL DEFAULT 0,
|
|
73
74
|
estimated_cost_usd REAL,
|
|
@@ -109,6 +110,7 @@ export function getDb() {
|
|
|
109
110
|
// Column already exists.
|
|
110
111
|
}
|
|
111
112
|
db.exec('CREATE UNIQUE INDEX IF NOT EXISTS idx_jobs_slug ON jobs(slug)');
|
|
113
|
+
db.exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_job_runs_active_job ON job_runs(job_id) WHERE status = 'running'");
|
|
112
114
|
db.exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_scan_items_dedup ON scan_items(job_id, reddit_post_id, COALESCE(reddit_comment_id, ''))");
|
|
113
115
|
db.exec('CREATE INDEX IF NOT EXISTS idx_scan_items_job_qualified_posted ON scan_items(job_id, qualified, reddit_posted_at DESC, created_at DESC)');
|
|
114
116
|
db.exec('CREATE INDEX IF NOT EXISTS idx_scan_items_job_created ON scan_items(job_id, created_at DESC)');
|
|
@@ -205,6 +207,13 @@ export function getDb() {
|
|
|
205
207
|
catch {
|
|
206
208
|
// Column already exists.
|
|
207
209
|
}
|
|
210
|
+
try {
|
|
211
|
+
db.exec('ALTER TABLE scan_items ADD COLUMN consumed INTEGER NOT NULL DEFAULT 0');
|
|
212
|
+
}
|
|
213
|
+
catch {
|
|
214
|
+
// Column already exists.
|
|
215
|
+
}
|
|
216
|
+
db.exec('CREATE INDEX IF NOT EXISTS idx_scan_items_consumed ON scan_items(job_id, qualified, consumed, created_at DESC)');
|
|
208
217
|
return db;
|
|
209
218
|
}
|
|
210
219
|
//# sourceMappingURL=sqlite.js.map
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { SettingsRepository } from '../db/repositories/settingsRepo.js';
|
|
2
|
-
import { RunsRepository } from '../db/repositories/runsRepo.js';
|
|
2
|
+
import { ActiveRunConflictError, RunsRepository } from '../db/repositories/runsRepo.js';
|
|
3
3
|
import { ScanItemsRepository } from '../db/repositories/scanItemsRepo.js';
|
|
4
4
|
import { getOpenRouterApiKey } from '../security/secretStore.js';
|
|
5
5
|
import { logger } from '../../utils/logger.js';
|
|
@@ -86,7 +86,20 @@ export class JobRunner {
|
|
|
86
86
|
const appSettings = this.settingsRepo.getAppSettings();
|
|
87
87
|
const model = appSettings.model;
|
|
88
88
|
const modelSettings = appSettings.modelSettings;
|
|
89
|
-
|
|
89
|
+
let runId;
|
|
90
|
+
try {
|
|
91
|
+
runId = this.runsRepo.startRun(job.id);
|
|
92
|
+
}
|
|
93
|
+
catch (error) {
|
|
94
|
+
if (error instanceof ActiveRunConflictError) {
|
|
95
|
+
const message = `Skipped job ${job.name} (${job.id}): another run is already active.`;
|
|
96
|
+
this.runsRepo.addRun(job.id, 'skipped', message);
|
|
97
|
+
this.emit(options, { type: 'run_skipped', reason: 'already_running', message });
|
|
98
|
+
logger.warn(message);
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
throw error;
|
|
102
|
+
}
|
|
90
103
|
const runLogger = createRunLogger(runId);
|
|
91
104
|
this.runsRepo.setLogFilePath(runId, runLogger.getLogFilePath());
|
|
92
105
|
const redditTraceHooks = {
|
|
@@ -174,7 +187,7 @@ export class JobRunner {
|
|
|
174
187
|
postBody: post.body
|
|
175
188
|
});
|
|
176
189
|
runLogger.info(JSON.stringify({ event: 'post_qualify_result', postId: post.id, result }, null, 2));
|
|
177
|
-
this.scanItemsRepo.
|
|
190
|
+
const postInsert = this.scanItemsRepo.createWithStatus({
|
|
178
191
|
jobId: job.id,
|
|
179
192
|
runId,
|
|
180
193
|
type: 'post',
|
|
@@ -192,24 +205,36 @@ export class JobRunner {
|
|
|
192
205
|
estimatedCostUsd: this.estimateCost(result.promptTokens, result.completionTokens),
|
|
193
206
|
qualificationReason: result.reason
|
|
194
207
|
});
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
208
|
+
if (!postInsert.inserted) {
|
|
209
|
+
this.emit(options, {
|
|
210
|
+
type: 'post_scanned',
|
|
211
|
+
postId: post.id,
|
|
212
|
+
subreddit: post.subreddit,
|
|
213
|
+
status: 'existing',
|
|
214
|
+
itemsNew: runStats.itemsNew,
|
|
215
|
+
itemsQualified: runStats.itemsQualified
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
else {
|
|
219
|
+
runStats.itemsNew += 1;
|
|
220
|
+
if (result.qualified) {
|
|
221
|
+
runStats.itemsQualified += 1;
|
|
222
|
+
}
|
|
223
|
+
this.accumulateTokens(runStats, result);
|
|
224
|
+
this.emit(options, {
|
|
225
|
+
type: 'post_scanned',
|
|
226
|
+
postId: post.id,
|
|
227
|
+
subreddit: post.subreddit,
|
|
228
|
+
status: 'new',
|
|
229
|
+
title: post.title,
|
|
230
|
+
bodySnippet: toSnippet(post.body),
|
|
231
|
+
postUrl: post.url,
|
|
232
|
+
qualified: result.qualified,
|
|
233
|
+
qualificationReason: result.reason,
|
|
234
|
+
itemsNew: runStats.itemsNew,
|
|
235
|
+
itemsQualified: runStats.itemsQualified
|
|
236
|
+
});
|
|
198
237
|
}
|
|
199
|
-
this.accumulateTokens(runStats, result);
|
|
200
|
-
this.emit(options, {
|
|
201
|
-
type: 'post_scanned',
|
|
202
|
-
postId: post.id,
|
|
203
|
-
subreddit: post.subreddit,
|
|
204
|
-
status: 'new',
|
|
205
|
-
title: post.title,
|
|
206
|
-
bodySnippet: toSnippet(post.body),
|
|
207
|
-
postUrl: post.url,
|
|
208
|
-
qualified: result.qualified,
|
|
209
|
-
qualificationReason: result.reason,
|
|
210
|
-
itemsNew: runStats.itemsNew,
|
|
211
|
-
itemsQualified: runStats.itemsQualified
|
|
212
|
-
});
|
|
213
238
|
}
|
|
214
239
|
else {
|
|
215
240
|
this.emit(options, {
|
|
@@ -319,7 +344,7 @@ export class JobRunner {
|
|
|
319
344
|
author,
|
|
320
345
|
result
|
|
321
346
|
}, null, 2));
|
|
322
|
-
this.scanItemsRepo.
|
|
347
|
+
const commentInsert = this.scanItemsRepo.createWithStatus({
|
|
323
348
|
jobId: job.id,
|
|
324
349
|
runId,
|
|
325
350
|
type: 'comment',
|
|
@@ -345,25 +370,38 @@ export class JobRunner {
|
|
|
345
370
|
isTarget: index === thread.length - 1
|
|
346
371
|
}))
|
|
347
372
|
});
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
373
|
+
if (!commentInsert.inserted) {
|
|
374
|
+
this.emit(options, {
|
|
375
|
+
type: 'comment_scanned',
|
|
376
|
+
postId: post.id,
|
|
377
|
+
commentId: lastComment.id,
|
|
378
|
+
author,
|
|
379
|
+
status: 'existing',
|
|
380
|
+
itemsNew: runStats.itemsNew,
|
|
381
|
+
itemsQualified: runStats.itemsQualified
|
|
382
|
+
});
|
|
383
|
+
}
|
|
384
|
+
else {
|
|
385
|
+
runStats.itemsNew += 1;
|
|
386
|
+
if (result.qualified) {
|
|
387
|
+
runStats.itemsQualified += 1;
|
|
388
|
+
}
|
|
389
|
+
this.accumulateTokens(runStats, result);
|
|
390
|
+
this.emit(options, {
|
|
391
|
+
type: 'comment_scanned',
|
|
392
|
+
postId: post.id,
|
|
393
|
+
commentId: lastComment.id,
|
|
394
|
+
author,
|
|
395
|
+
status: 'new',
|
|
396
|
+
commentSnippet: toSnippet(lastComment.body),
|
|
397
|
+
postUrl: post.url,
|
|
398
|
+
commentUrl: buildCommentUrl(post.url, lastComment),
|
|
399
|
+
qualified: result.qualified,
|
|
400
|
+
qualificationReason: result.reason,
|
|
401
|
+
itemsNew: runStats.itemsNew,
|
|
402
|
+
itemsQualified: runStats.itemsQualified
|
|
403
|
+
});
|
|
351
404
|
}
|
|
352
|
-
this.accumulateTokens(runStats, result);
|
|
353
|
-
this.emit(options, {
|
|
354
|
-
type: 'comment_scanned',
|
|
355
|
-
postId: post.id,
|
|
356
|
-
commentId: lastComment.id,
|
|
357
|
-
author,
|
|
358
|
-
status: 'new',
|
|
359
|
-
commentSnippet: toSnippet(lastComment.body),
|
|
360
|
-
postUrl: post.url,
|
|
361
|
-
commentUrl: buildCommentUrl(post.url, lastComment),
|
|
362
|
-
qualified: result.qualified,
|
|
363
|
-
qualificationReason: result.reason,
|
|
364
|
-
itemsNew: runStats.itemsNew,
|
|
365
|
-
itemsQualified: runStats.itemsQualified
|
|
366
|
-
});
|
|
367
405
|
}
|
|
368
406
|
}
|
|
369
407
|
}
|