@telepat/snoopy 0.1.4 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -24
- package/dist/src/cli/commands/doctor.js +5 -1
- package/dist/src/cli/commands/export.d.ts +8 -1
- package/dist/src/cli/commands/export.js +33 -7
- package/dist/src/cli/commands/job.d.ts +2 -2
- package/dist/src/cli/commands/job.js +94 -30
- package/dist/src/cli/commands/results.d.ts +1 -0
- package/dist/src/cli/commands/results.js +118 -0
- package/dist/src/cli/commands/startup.js +5 -2
- package/dist/src/cli/index.js +17 -6
- package/dist/src/cli/ui/consoleUi.js +0 -3
- package/dist/src/services/db/repositories/jobsRepo.d.ts +15 -0
- package/dist/src/services/db/repositories/jobsRepo.js +21 -14
- package/dist/src/services/db/repositories/runsRepo.d.ts +8 -0
- package/dist/src/services/db/repositories/runsRepo.js +64 -54
- package/dist/src/services/db/repositories/scanItemsRepo.d.ts +45 -1
- package/dist/src/services/db/repositories/scanItemsRepo.js +184 -28
- package/dist/src/services/db/sqlite.js +18 -0
- package/dist/src/services/export/csvResults.d.ts +1 -1
- package/dist/src/services/export/csvResults.js +3 -2
- package/dist/src/services/export/fileNaming.d.ts +2 -0
- package/dist/src/services/export/fileNaming.js +10 -0
- package/dist/src/services/export/jsonResults.d.ts +10 -0
- package/dist/src/services/export/jsonResults.js +16 -0
- package/dist/src/services/openrouter/client.d.ts +1 -0
- package/dist/src/services/openrouter/client.js +36 -5
- package/dist/src/services/scheduler/jobRunner.js +10 -1
- package/dist/src/services/startup/index.js +16 -28
- package/dist/src/services/startup/windowsTaskScheduler.d.ts +1 -0
- package/dist/src/services/startup/windowsTaskScheduler.js +24 -4
- package/dist/src/ui/components/JobsTable.d.ts +8 -0
- package/dist/src/ui/components/JobsTable.js +78 -0
- package/dist/src/ui/components/ResultsViewer.d.ts +11 -0
- package/dist/src/ui/components/ResultsViewer.js +81 -0
- package/dist/src/ui/components/RunsTable.d.ts +9 -0
- package/dist/src/ui/components/RunsTable.js +99 -0
- package/dist/src/ui/components/jobsTableModel.d.ts +15 -0
- package/dist/src/ui/components/jobsTableModel.js +79 -0
- package/dist/src/ui/components/resultsViewerModel.d.ts +14 -0
- package/dist/src/ui/components/resultsViewerModel.js +122 -0
- package/dist/src/ui/components/runsTableModel.d.ts +28 -0
- package/dist/src/ui/components/runsTableModel.js +109 -0
- package/package.json +5 -2
- package/dist/src/cli/commands/analytics.js.map +0 -1
- package/dist/src/cli/commands/daemon.js.map +0 -1
- package/dist/src/cli/commands/doctor.js.map +0 -1
- package/dist/src/cli/commands/errors.js.map +0 -1
- package/dist/src/cli/commands/export.js.map +0 -1
- package/dist/src/cli/commands/job.js.map +0 -1
- package/dist/src/cli/commands/logs.js.map +0 -1
- package/dist/src/cli/commands/selection.js.map +0 -1
- package/dist/src/cli/commands/settings.js.map +0 -1
- package/dist/src/cli/commands/startup.js.map +0 -1
- package/dist/src/cli/flows/jobAddFlow.js.map +0 -1
- package/dist/src/cli/flows/settingsFlow.js.map +0 -1
- package/dist/src/cli/flows/settingsFlowModel.js.map +0 -1
- package/dist/src/cli/index.js.map +0 -1
- package/dist/src/cli/ui/consoleUi.js.map +0 -1
- package/dist/src/cli/ui/time.js.map +0 -1
- package/dist/src/index.js.map +0 -1
- package/dist/src/scripts/e2eSmoke.d.ts +0 -1
- package/dist/src/scripts/e2eSmoke.js +0 -102
- package/dist/src/scripts/e2eSmoke.js.map +0 -1
- package/dist/src/services/analytics/analyticsService.js.map +0 -1
- package/dist/src/services/daemonControl.js.map +0 -1
- package/dist/src/services/db/repositories/jobsRepo.js.map +0 -1
- package/dist/src/services/db/repositories/runsRepo.js.map +0 -1
- package/dist/src/services/db/repositories/scanItemsRepo.js.map +0 -1
- package/dist/src/services/db/repositories/settingsRepo.js.map +0 -1
- package/dist/src/services/db/sqlite.js.map +0 -1
- package/dist/src/services/export/csvResults.js.map +0 -1
- package/dist/src/services/logging/logReader.js.map +0 -1
- package/dist/src/services/logging/logRotation.js.map +0 -1
- package/dist/src/services/logging/runLogger.js.map +0 -1
- package/dist/src/services/openrouter/client.js.map +0 -1
- package/dist/src/services/openrouter/prompts.js.map +0 -1
- package/dist/src/services/reddit/client.js.map +0 -1
- package/dist/src/services/scheduler/cronScheduler.js.map +0 -1
- package/dist/src/services/scheduler/jobRunner.js.map +0 -1
- package/dist/src/services/scheduler/jobRunnerStub.js.map +0 -1
- package/dist/src/services/security/secretStore.js.map +0 -1
- package/dist/src/services/startup/index.js.map +0 -1
- package/dist/src/services/startup/linuxCronFallback.js.map +0 -1
- package/dist/src/services/startup/linuxSystemd.js.map +0 -1
- package/dist/src/services/startup/macosLaunchd.js.map +0 -1
- package/dist/src/services/startup/windowsRunFallback.d.ts +0 -2
- package/dist/src/services/startup/windowsRunFallback.js +0 -17
- package/dist/src/services/startup/windowsRunFallback.js.map +0 -1
- package/dist/src/services/startup/windowsTaskScheduler.js.map +0 -1
- package/dist/src/types/job.js.map +0 -1
- package/dist/src/types/settings.js.map +0 -1
- package/dist/src/ui/components/AppFrame.js.map +0 -1
- package/dist/src/ui/components/CliHeader.js.map +0 -1
- package/dist/src/ui/components/SubredditMultiSelect.js.map +0 -1
- package/dist/src/ui/components/TextPrompt.js.map +0 -1
- package/dist/src/ui/components/YesNoSelector.js.map +0 -1
- package/dist/src/ui/components/subredditOptions.js.map +0 -1
- package/dist/src/ui/components/yesNoSelectorModel.js.map +0 -1
- package/dist/src/ui/theme.js.map +0 -1
- package/dist/src/utils/logger.js.map +0 -1
- package/dist/src/utils/notify.js.map +0 -1
- package/dist/src/utils/paths.js.map +0 -1
- package/dist/src/utils/scanLogFormatting.js.map +0 -1
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
import crypto from 'node:crypto';
|
|
2
2
|
import fs from 'node:fs';
|
|
3
|
-
import path from 'node:path';
|
|
4
3
|
import { getDb } from '../sqlite.js';
|
|
5
|
-
import { getAppPaths } from '../../../utils/paths.js';
|
|
6
4
|
function toSlug(value) {
|
|
7
5
|
const cleaned = value
|
|
8
6
|
.toLowerCase()
|
|
@@ -31,7 +29,6 @@ function mapRow(row) {
|
|
|
31
29
|
export class JobsRepository {
|
|
32
30
|
db = getDb();
|
|
33
31
|
removeCascadeStmt = this.db.transaction((jobId) => {
|
|
34
|
-
const jobRow = this.db.prepare('SELECT slug FROM jobs WHERE id = ?').get(jobId);
|
|
35
32
|
const runLogs = this.db
|
|
36
33
|
.prepare(`SELECT log_file_path as logFilePath
|
|
37
34
|
FROM job_runs
|
|
@@ -51,17 +48,6 @@ export class JobsRepository {
|
|
|
51
48
|
// Ignore filesystem cleanup failures and continue DB cleanup.
|
|
52
49
|
}
|
|
53
50
|
}
|
|
54
|
-
if (jobRow?.slug) {
|
|
55
|
-
const csvPath = path.join(getAppPaths().resultsDir, `${jobRow.slug}.csv`);
|
|
56
|
-
try {
|
|
57
|
-
if (fs.existsSync(csvPath)) {
|
|
58
|
-
fs.unlinkSync(csvPath);
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
catch {
|
|
62
|
-
// Ignore filesystem cleanup failures and continue DB cleanup.
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
51
|
this.db.prepare('DELETE FROM scan_items WHERE job_id = ?').run(jobId);
|
|
66
52
|
this.db.prepare('DELETE FROM job_runs WHERE job_id = ?').run(jobId);
|
|
67
53
|
this.db.prepare('DELETE FROM jobs WHERE id = ?').run(jobId);
|
|
@@ -160,5 +146,26 @@ export class JobsRepository {
|
|
|
160
146
|
this.remove(job.id);
|
|
161
147
|
return job;
|
|
162
148
|
}
|
|
149
|
+
listWithStats() {
|
|
150
|
+
return this.db
|
|
151
|
+
.prepare(`SELECT
|
|
152
|
+
j.id as jobId,
|
|
153
|
+
j.slug as jobSlug,
|
|
154
|
+
j.name as jobName,
|
|
155
|
+
j.enabled as enabled,
|
|
156
|
+
j.description as description,
|
|
157
|
+
j.subreddits_json as subredditsJson,
|
|
158
|
+
j.schedule_cron as scheduleCron,
|
|
159
|
+
MAX(COALESCE(jr.finished_at, jr.started_at, jr.created_at)) as lastRunAt,
|
|
160
|
+
COALESCE(SUM(jr.items_new), 0) as totalScanned,
|
|
161
|
+
COALESCE(SUM(jr.items_qualified), 0) as totalQualified,
|
|
162
|
+
COALESCE(SUM(jr.estimated_cost_usd), 0) as totalCostUsd,
|
|
163
|
+
COUNT(jr.id) as runCount
|
|
164
|
+
FROM jobs j
|
|
165
|
+
LEFT JOIN job_runs jr ON jr.job_id = j.id
|
|
166
|
+
GROUP BY j.id
|
|
167
|
+
ORDER BY j.name ASC`)
|
|
168
|
+
.all();
|
|
169
|
+
}
|
|
163
170
|
}
|
|
164
171
|
//# sourceMappingURL=jobsRepo.js.map
|
|
@@ -10,6 +10,7 @@ export interface RunRow {
|
|
|
10
10
|
id: string;
|
|
11
11
|
jobId: string;
|
|
12
12
|
jobName: string | null;
|
|
13
|
+
jobSlug: string | null;
|
|
13
14
|
status: string;
|
|
14
15
|
message: string | null;
|
|
15
16
|
startedAt: string | null;
|
|
@@ -34,6 +35,7 @@ interface RunAnalyticsFilter {
|
|
|
34
35
|
}
|
|
35
36
|
export declare class RunsRepository {
|
|
36
37
|
private readonly db;
|
|
38
|
+
private readonly runSelectWithJob;
|
|
37
39
|
private buildRunFilter;
|
|
38
40
|
addRun(jobId: string, status: string, message: string): void;
|
|
39
41
|
startRun(jobId: string, logFilePath?: string): string;
|
|
@@ -49,6 +51,12 @@ export declare class RunsRepository {
|
|
|
49
51
|
}>;
|
|
50
52
|
listByJob(jobId: string, limit?: number): RunRow[];
|
|
51
53
|
latestWithJobNames(limit?: number): RunRow[];
|
|
54
|
+
countByJob(jobId: string): number;
|
|
55
|
+
countAll(): number;
|
|
56
|
+
listByJobPage(jobId: string, limit: number, offset: number): RunRow[];
|
|
57
|
+
latestWithJobNamesPage(limit: number, offset: number): RunRow[];
|
|
58
|
+
getByJobIndex(jobId: string, index: number): RunRow | null;
|
|
59
|
+
getLatestWithJobNamesByIndex(index: number): RunRow | null;
|
|
52
60
|
listAnalyticsRuns(filter: RunAnalyticsFilter): RunAnalyticsRow[];
|
|
53
61
|
countRuns(filter: {
|
|
54
62
|
jobId?: string;
|
|
@@ -2,6 +2,25 @@ import crypto from 'node:crypto';
|
|
|
2
2
|
import { getDb } from '../sqlite.js';
|
|
3
3
|
export class RunsRepository {
|
|
4
4
|
db = getDb();
|
|
5
|
+
runSelectWithJob = `SELECT
|
|
6
|
+
jr.id as id,
|
|
7
|
+
jr.job_id as jobId,
|
|
8
|
+
j.name as jobName,
|
|
9
|
+
j.slug as jobSlug,
|
|
10
|
+
jr.status as status,
|
|
11
|
+
jr.message as message,
|
|
12
|
+
jr.started_at as startedAt,
|
|
13
|
+
jr.finished_at as finishedAt,
|
|
14
|
+
jr.created_at as createdAt,
|
|
15
|
+
jr.items_discovered as itemsDiscovered,
|
|
16
|
+
jr.items_new as itemsNew,
|
|
17
|
+
jr.items_qualified as itemsQualified,
|
|
18
|
+
jr.prompt_tokens as promptTokens,
|
|
19
|
+
jr.completion_tokens as completionTokens,
|
|
20
|
+
jr.estimated_cost_usd as estimatedCostUsd,
|
|
21
|
+
jr.log_file_path as logFilePath
|
|
22
|
+
FROM job_runs jr
|
|
23
|
+
LEFT JOIN jobs j ON j.id = jr.job_id`;
|
|
5
24
|
buildRunFilter(filter) {
|
|
6
25
|
const params = [`-${filter.days} days`];
|
|
7
26
|
const conditions = ["datetime(jr.created_at) >= datetime('now', ?)"];
|
|
@@ -66,24 +85,7 @@ export class RunsRepository {
|
|
|
66
85
|
}
|
|
67
86
|
getById(runId) {
|
|
68
87
|
const row = this.db
|
|
69
|
-
.prepare(
|
|
70
|
-
jr.id as id,
|
|
71
|
-
jr.job_id as jobId,
|
|
72
|
-
j.name as jobName,
|
|
73
|
-
jr.status as status,
|
|
74
|
-
jr.message as message,
|
|
75
|
-
jr.started_at as startedAt,
|
|
76
|
-
jr.finished_at as finishedAt,
|
|
77
|
-
jr.created_at as createdAt,
|
|
78
|
-
jr.items_discovered as itemsDiscovered,
|
|
79
|
-
jr.items_new as itemsNew,
|
|
80
|
-
jr.items_qualified as itemsQualified,
|
|
81
|
-
jr.prompt_tokens as promptTokens,
|
|
82
|
-
jr.completion_tokens as completionTokens,
|
|
83
|
-
jr.estimated_cost_usd as estimatedCostUsd,
|
|
84
|
-
jr.log_file_path as logFilePath
|
|
85
|
-
FROM job_runs jr
|
|
86
|
-
LEFT JOIN jobs j ON j.id = jr.job_id
|
|
88
|
+
.prepare(`${this.runSelectWithJob}
|
|
87
89
|
WHERE jr.id = ?
|
|
88
90
|
LIMIT 1`)
|
|
89
91
|
.get(runId);
|
|
@@ -99,24 +101,7 @@ export class RunsRepository {
|
|
|
99
101
|
}
|
|
100
102
|
listByJob(jobId, limit = 20) {
|
|
101
103
|
return this.db
|
|
102
|
-
.prepare(
|
|
103
|
-
jr.id as id,
|
|
104
|
-
jr.job_id as jobId,
|
|
105
|
-
j.name as jobName,
|
|
106
|
-
jr.status as status,
|
|
107
|
-
jr.message as message,
|
|
108
|
-
jr.started_at as startedAt,
|
|
109
|
-
jr.finished_at as finishedAt,
|
|
110
|
-
jr.created_at as createdAt,
|
|
111
|
-
jr.items_discovered as itemsDiscovered,
|
|
112
|
-
jr.items_new as itemsNew,
|
|
113
|
-
jr.items_qualified as itemsQualified,
|
|
114
|
-
jr.prompt_tokens as promptTokens,
|
|
115
|
-
jr.completion_tokens as completionTokens,
|
|
116
|
-
jr.estimated_cost_usd as estimatedCostUsd,
|
|
117
|
-
jr.log_file_path as logFilePath
|
|
118
|
-
FROM job_runs jr
|
|
119
|
-
LEFT JOIN jobs j ON j.id = jr.job_id
|
|
104
|
+
.prepare(`${this.runSelectWithJob}
|
|
120
105
|
WHERE jr.job_id = ?
|
|
121
106
|
ORDER BY jr.created_at DESC
|
|
122
107
|
LIMIT ?`)
|
|
@@ -124,28 +109,53 @@ export class RunsRepository {
|
|
|
124
109
|
}
|
|
125
110
|
latestWithJobNames(limit = 20) {
|
|
126
111
|
return this.db
|
|
127
|
-
.prepare(
|
|
128
|
-
jr.id as id,
|
|
129
|
-
jr.job_id as jobId,
|
|
130
|
-
j.name as jobName,
|
|
131
|
-
jr.status as status,
|
|
132
|
-
jr.message as message,
|
|
133
|
-
jr.started_at as startedAt,
|
|
134
|
-
jr.finished_at as finishedAt,
|
|
135
|
-
jr.created_at as createdAt,
|
|
136
|
-
jr.items_discovered as itemsDiscovered,
|
|
137
|
-
jr.items_new as itemsNew,
|
|
138
|
-
jr.items_qualified as itemsQualified,
|
|
139
|
-
jr.prompt_tokens as promptTokens,
|
|
140
|
-
jr.completion_tokens as completionTokens,
|
|
141
|
-
jr.estimated_cost_usd as estimatedCostUsd,
|
|
142
|
-
jr.log_file_path as logFilePath
|
|
143
|
-
FROM job_runs jr
|
|
144
|
-
LEFT JOIN jobs j ON j.id = jr.job_id
|
|
112
|
+
.prepare(`${this.runSelectWithJob}
|
|
145
113
|
ORDER BY jr.created_at DESC
|
|
146
114
|
LIMIT ?`)
|
|
147
115
|
.all(limit);
|
|
148
116
|
}
|
|
117
|
+
countByJob(jobId) {
|
|
118
|
+
const row = this.db
|
|
119
|
+
.prepare(`SELECT COUNT(*) as count
|
|
120
|
+
FROM job_runs
|
|
121
|
+
WHERE job_id = ?`)
|
|
122
|
+
.get(jobId);
|
|
123
|
+
return Number(row?.count ?? 0);
|
|
124
|
+
}
|
|
125
|
+
countAll() {
|
|
126
|
+
const row = this.db
|
|
127
|
+
.prepare(`SELECT COUNT(*) as count
|
|
128
|
+
FROM job_runs`)
|
|
129
|
+
.get();
|
|
130
|
+
return Number(row?.count ?? 0);
|
|
131
|
+
}
|
|
132
|
+
listByJobPage(jobId, limit, offset) {
|
|
133
|
+
const boundedLimit = Math.max(1, Math.floor(limit));
|
|
134
|
+
const boundedOffset = Math.max(0, Math.floor(offset));
|
|
135
|
+
return this.db
|
|
136
|
+
.prepare(`${this.runSelectWithJob}
|
|
137
|
+
WHERE jr.job_id = ?
|
|
138
|
+
ORDER BY jr.created_at DESC
|
|
139
|
+
LIMIT ? OFFSET ?`)
|
|
140
|
+
.all(jobId, boundedLimit, boundedOffset);
|
|
141
|
+
}
|
|
142
|
+
latestWithJobNamesPage(limit, offset) {
|
|
143
|
+
const boundedLimit = Math.max(1, Math.floor(limit));
|
|
144
|
+
const boundedOffset = Math.max(0, Math.floor(offset));
|
|
145
|
+
return this.db
|
|
146
|
+
.prepare(`${this.runSelectWithJob}
|
|
147
|
+
ORDER BY jr.created_at DESC
|
|
148
|
+
LIMIT ? OFFSET ?`)
|
|
149
|
+
.all(boundedLimit, boundedOffset);
|
|
150
|
+
}
|
|
151
|
+
getByJobIndex(jobId, index) {
|
|
152
|
+
const rows = this.listByJobPage(jobId, 1, index);
|
|
153
|
+
return rows[0] ?? null;
|
|
154
|
+
}
|
|
155
|
+
getLatestWithJobNamesByIndex(index) {
|
|
156
|
+
const rows = this.latestWithJobNamesPage(1, index);
|
|
157
|
+
return rows[0] ?? null;
|
|
158
|
+
}
|
|
149
159
|
listAnalyticsRuns(filter) {
|
|
150
160
|
const { clause, params } = this.buildRunFilter(filter);
|
|
151
161
|
const limit = filter.limit ?? 20;
|
|
@@ -19,6 +19,20 @@ export interface NewScanItem {
|
|
|
19
19
|
completionTokens?: number;
|
|
20
20
|
estimatedCostUsd?: number | null;
|
|
21
21
|
qualificationReason: string | null;
|
|
22
|
+
commentThreadNodes?: NewCommentThreadNode[];
|
|
23
|
+
}
|
|
24
|
+
export interface NewCommentThreadNode {
|
|
25
|
+
redditCommentId: string;
|
|
26
|
+
parentRedditCommentId: string | null;
|
|
27
|
+
author: string;
|
|
28
|
+
body: string;
|
|
29
|
+
depth: number;
|
|
30
|
+
isTarget: boolean;
|
|
31
|
+
}
|
|
32
|
+
export interface CommentThreadNodeRow extends NewCommentThreadNode {
|
|
33
|
+
id: string;
|
|
34
|
+
scanItemId: string;
|
|
35
|
+
createdAt: string;
|
|
22
36
|
}
|
|
23
37
|
export interface QualifiedScanItemRow {
|
|
24
38
|
id: string;
|
|
@@ -35,6 +49,29 @@ export interface QualifiedScanItemRow {
|
|
|
35
49
|
qualificationReason: string | null;
|
|
36
50
|
createdAt: string;
|
|
37
51
|
}
|
|
52
|
+
export interface ScanItemRow {
|
|
53
|
+
id: string;
|
|
54
|
+
jobId: string;
|
|
55
|
+
runId: string;
|
|
56
|
+
type: ScanItemType;
|
|
57
|
+
redditPostId: string;
|
|
58
|
+
redditCommentId: string | null;
|
|
59
|
+
subreddit: string;
|
|
60
|
+
author: string;
|
|
61
|
+
title: string | null;
|
|
62
|
+
body: string;
|
|
63
|
+
url: string;
|
|
64
|
+
redditPostedAt: string;
|
|
65
|
+
qualified: boolean;
|
|
66
|
+
viewed: boolean;
|
|
67
|
+
validated: boolean;
|
|
68
|
+
processed: boolean;
|
|
69
|
+
qualificationReason: string | null;
|
|
70
|
+
promptTokens: number;
|
|
71
|
+
completionTokens: number;
|
|
72
|
+
estimatedCostUsd: number | null;
|
|
73
|
+
createdAt: string;
|
|
74
|
+
}
|
|
38
75
|
export interface AnalyticsTotalsRow {
|
|
39
76
|
newPosts: number;
|
|
40
77
|
newComments: number;
|
|
@@ -56,14 +93,21 @@ interface AnalyticsFilter {
|
|
|
56
93
|
}
|
|
57
94
|
export declare class ScanItemsRepository {
|
|
58
95
|
private readonly db;
|
|
96
|
+
private mapScanItemRows;
|
|
59
97
|
private buildFilterClause;
|
|
60
98
|
private toAnalyticsTotalsRow;
|
|
61
|
-
listQualifiedByJob(jobId: string): QualifiedScanItemRow[];
|
|
99
|
+
listQualifiedByJob(jobId: string, limit?: number): QualifiedScanItemRow[];
|
|
100
|
+
listQualifiedByJobRun(jobId: string, runId: string, limit?: number): QualifiedScanItemRow[];
|
|
101
|
+
listByJob(jobId: string): ScanItemRow[];
|
|
102
|
+
countByJob(jobId: string): number;
|
|
103
|
+
listByJobPage(jobId: string, limit: number, offset: number): ScanItemRow[];
|
|
104
|
+
getByJobIndex(jobId: string, index: number): ScanItemRow | null;
|
|
62
105
|
existsPost(jobId: string, postId: string): boolean;
|
|
63
106
|
getAnalyticsTotals(filter: AnalyticsFilter): AnalyticsTotalsRow;
|
|
64
107
|
listAnalyticsBySubreddit(filter: AnalyticsFilter): AnalyticsBySubredditRow[];
|
|
65
108
|
listAnalyticsByJob(days: number): AnalyticsByJobRow[];
|
|
66
109
|
existsComment(jobId: string, postId: string, commentId: string): boolean;
|
|
67
110
|
create(item: NewScanItem): string;
|
|
111
|
+
listCommentThreadNodes(scanItemId: string): CommentThreadNodeRow[];
|
|
68
112
|
}
|
|
69
113
|
export {};
|
|
@@ -2,6 +2,15 @@ import crypto from 'node:crypto';
|
|
|
2
2
|
import { getDb } from '../sqlite.js';
|
|
3
3
|
export class ScanItemsRepository {
|
|
4
4
|
db = getDb();
|
|
5
|
+
mapScanItemRows(rows) {
|
|
6
|
+
return rows.map((row) => ({
|
|
7
|
+
...row,
|
|
8
|
+
qualified: row.qualified === 1,
|
|
9
|
+
viewed: row.viewed === 1,
|
|
10
|
+
validated: row.validated === 1,
|
|
11
|
+
processed: row.processed === 1
|
|
12
|
+
}));
|
|
13
|
+
}
|
|
5
14
|
buildFilterClause(alias, filter) {
|
|
6
15
|
const params = [`-${filter.days} days`];
|
|
7
16
|
const conditions = [`datetime(${alias}.created_at) >= datetime('now', ?)`];
|
|
@@ -23,7 +32,8 @@ export class ScanItemsRepository {
|
|
|
23
32
|
estimatedCostUsd: Number(row.estimatedCostUsd ?? 0)
|
|
24
33
|
};
|
|
25
34
|
}
|
|
26
|
-
listQualifiedByJob(jobId) {
|
|
35
|
+
listQualifiedByJob(jobId, limit = 100) {
|
|
36
|
+
const boundedLimit = Math.max(1, Math.floor(limit));
|
|
27
37
|
const rows = this.db
|
|
28
38
|
.prepare(`SELECT
|
|
29
39
|
id,
|
|
@@ -42,8 +52,9 @@ export class ScanItemsRepository {
|
|
|
42
52
|
FROM scan_items
|
|
43
53
|
WHERE job_id = ?
|
|
44
54
|
AND qualified = 1
|
|
45
|
-
ORDER BY datetime(reddit_posted_at) DESC, datetime(created_at) DESC, id DESC
|
|
46
|
-
|
|
55
|
+
ORDER BY datetime(reddit_posted_at) DESC, datetime(created_at) DESC, id DESC
|
|
56
|
+
LIMIT ?`)
|
|
57
|
+
.all(jobId, boundedLimit);
|
|
47
58
|
return rows.map((row) => ({
|
|
48
59
|
...row,
|
|
49
60
|
viewed: row.viewed === 1,
|
|
@@ -51,6 +62,112 @@ export class ScanItemsRepository {
|
|
|
51
62
|
processed: row.processed === 1
|
|
52
63
|
}));
|
|
53
64
|
}
|
|
65
|
+
listQualifiedByJobRun(jobId, runId, limit = 100) {
|
|
66
|
+
const boundedLimit = Math.max(1, Math.floor(limit));
|
|
67
|
+
const rows = this.db
|
|
68
|
+
.prepare(`SELECT
|
|
69
|
+
id,
|
|
70
|
+
job_id as jobId,
|
|
71
|
+
run_id as runId,
|
|
72
|
+
author,
|
|
73
|
+
title,
|
|
74
|
+
body,
|
|
75
|
+
url,
|
|
76
|
+
reddit_posted_at as redditPostedAt,
|
|
77
|
+
viewed,
|
|
78
|
+
validated,
|
|
79
|
+
processed,
|
|
80
|
+
qualification_reason as qualificationReason,
|
|
81
|
+
created_at as createdAt
|
|
82
|
+
FROM scan_items
|
|
83
|
+
WHERE job_id = ?
|
|
84
|
+
AND run_id = ?
|
|
85
|
+
AND qualified = 1
|
|
86
|
+
ORDER BY datetime(reddit_posted_at) DESC, datetime(created_at) DESC, id DESC
|
|
87
|
+
LIMIT ?`)
|
|
88
|
+
.all(jobId, runId, boundedLimit);
|
|
89
|
+
return rows.map((row) => ({
|
|
90
|
+
...row,
|
|
91
|
+
viewed: row.viewed === 1,
|
|
92
|
+
validated: row.validated === 1,
|
|
93
|
+
processed: row.processed === 1
|
|
94
|
+
}));
|
|
95
|
+
}
|
|
96
|
+
listByJob(jobId) {
|
|
97
|
+
const rows = this.db
|
|
98
|
+
.prepare(`SELECT
|
|
99
|
+
id,
|
|
100
|
+
job_id as jobId,
|
|
101
|
+
run_id as runId,
|
|
102
|
+
type,
|
|
103
|
+
reddit_post_id as redditPostId,
|
|
104
|
+
reddit_comment_id as redditCommentId,
|
|
105
|
+
subreddit,
|
|
106
|
+
author,
|
|
107
|
+
title,
|
|
108
|
+
body,
|
|
109
|
+
url,
|
|
110
|
+
reddit_posted_at as redditPostedAt,
|
|
111
|
+
qualified,
|
|
112
|
+
viewed,
|
|
113
|
+
validated,
|
|
114
|
+
processed,
|
|
115
|
+
qualification_reason as qualificationReason,
|
|
116
|
+
prompt_tokens as promptTokens,
|
|
117
|
+
completion_tokens as completionTokens,
|
|
118
|
+
estimated_cost_usd as estimatedCostUsd,
|
|
119
|
+
created_at as createdAt
|
|
120
|
+
FROM scan_items
|
|
121
|
+
WHERE job_id = ?
|
|
122
|
+
ORDER BY datetime(reddit_posted_at) DESC, datetime(created_at) DESC, id DESC`)
|
|
123
|
+
.all(jobId);
|
|
124
|
+
return this.mapScanItemRows(rows);
|
|
125
|
+
}
|
|
126
|
+
countByJob(jobId) {
|
|
127
|
+
const row = this.db
|
|
128
|
+
.prepare(`SELECT COUNT(*) as count
|
|
129
|
+
FROM scan_items
|
|
130
|
+
WHERE job_id = ?`)
|
|
131
|
+
.get(jobId);
|
|
132
|
+
return Number(row?.count ?? 0);
|
|
133
|
+
}
|
|
134
|
+
listByJobPage(jobId, limit, offset) {
|
|
135
|
+
const boundedLimit = Math.max(1, Math.floor(limit));
|
|
136
|
+
const boundedOffset = Math.max(0, Math.floor(offset));
|
|
137
|
+
const rows = this.db
|
|
138
|
+
.prepare(`SELECT
|
|
139
|
+
id,
|
|
140
|
+
job_id as jobId,
|
|
141
|
+
run_id as runId,
|
|
142
|
+
type,
|
|
143
|
+
reddit_post_id as redditPostId,
|
|
144
|
+
reddit_comment_id as redditCommentId,
|
|
145
|
+
subreddit,
|
|
146
|
+
author,
|
|
147
|
+
title,
|
|
148
|
+
body,
|
|
149
|
+
url,
|
|
150
|
+
reddit_posted_at as redditPostedAt,
|
|
151
|
+
qualified,
|
|
152
|
+
viewed,
|
|
153
|
+
validated,
|
|
154
|
+
processed,
|
|
155
|
+
qualification_reason as qualificationReason,
|
|
156
|
+
prompt_tokens as promptTokens,
|
|
157
|
+
completion_tokens as completionTokens,
|
|
158
|
+
estimated_cost_usd as estimatedCostUsd,
|
|
159
|
+
created_at as createdAt
|
|
160
|
+
FROM scan_items
|
|
161
|
+
WHERE job_id = ?
|
|
162
|
+
ORDER BY datetime(reddit_posted_at) DESC, datetime(created_at) DESC, id DESC
|
|
163
|
+
LIMIT ? OFFSET ?`)
|
|
164
|
+
.all(jobId, boundedLimit, boundedOffset);
|
|
165
|
+
return this.mapScanItemRows(rows);
|
|
166
|
+
}
|
|
167
|
+
getByJobIndex(jobId, index) {
|
|
168
|
+
const rows = this.listByJobPage(jobId, 1, index);
|
|
169
|
+
return rows[0] ?? null;
|
|
170
|
+
}
|
|
54
171
|
existsPost(jobId, postId) {
|
|
55
172
|
const row = this.db
|
|
56
173
|
.prepare(`SELECT 1
|
|
@@ -145,32 +262,71 @@ export class ScanItemsRepository {
|
|
|
145
262
|
}
|
|
146
263
|
create(item) {
|
|
147
264
|
const id = crypto.randomUUID();
|
|
148
|
-
this.db
|
|
149
|
-
.
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
265
|
+
const createInTransaction = this.db.transaction((newId, newItem) => {
|
|
266
|
+
this.db
|
|
267
|
+
.prepare(`INSERT INTO scan_items (
|
|
268
|
+
id,
|
|
269
|
+
job_id,
|
|
270
|
+
run_id,
|
|
271
|
+
type,
|
|
272
|
+
reddit_post_id,
|
|
273
|
+
reddit_comment_id,
|
|
274
|
+
subreddit,
|
|
275
|
+
author,
|
|
276
|
+
title,
|
|
277
|
+
body,
|
|
278
|
+
url,
|
|
279
|
+
reddit_posted_at,
|
|
280
|
+
qualified,
|
|
281
|
+
viewed,
|
|
282
|
+
validated,
|
|
283
|
+
processed,
|
|
284
|
+
prompt_tokens,
|
|
285
|
+
completion_tokens,
|
|
286
|
+
estimated_cost_usd,
|
|
287
|
+
qualification_reason,
|
|
288
|
+
created_at
|
|
289
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))`)
|
|
290
|
+
.run(newId, newItem.jobId, newItem.runId, newItem.type, newItem.redditPostId, newItem.redditCommentId, newItem.subreddit, newItem.author, newItem.title, newItem.body, newItem.url, newItem.redditPostedAt, newItem.qualified ? 1 : 0, newItem.viewed ? 1 : 0, newItem.validated ? 1 : 0, newItem.processed ? 1 : 0, newItem.promptTokens ?? 0, newItem.completionTokens ?? 0, newItem.estimatedCostUsd ?? null, newItem.qualificationReason);
|
|
291
|
+
for (const node of newItem.commentThreadNodes ?? []) {
|
|
292
|
+
this.db
|
|
293
|
+
.prepare(`INSERT INTO comment_thread_nodes (
|
|
294
|
+
id,
|
|
295
|
+
scan_item_id,
|
|
296
|
+
reddit_comment_id,
|
|
297
|
+
parent_reddit_comment_id,
|
|
298
|
+
author,
|
|
299
|
+
body,
|
|
300
|
+
depth,
|
|
301
|
+
is_target,
|
|
302
|
+
created_at
|
|
303
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))`)
|
|
304
|
+
.run(crypto.randomUUID(), newId, node.redditCommentId, node.parentRedditCommentId, node.author, node.body, node.depth, node.isTarget ? 1 : 0);
|
|
305
|
+
}
|
|
306
|
+
});
|
|
307
|
+
createInTransaction(id, item);
|
|
173
308
|
return id;
|
|
174
309
|
}
|
|
310
|
+
listCommentThreadNodes(scanItemId) {
|
|
311
|
+
const rows = this.db
|
|
312
|
+
.prepare(`SELECT
|
|
313
|
+
id,
|
|
314
|
+
scan_item_id as scanItemId,
|
|
315
|
+
reddit_comment_id as redditCommentId,
|
|
316
|
+
parent_reddit_comment_id as parentRedditCommentId,
|
|
317
|
+
author,
|
|
318
|
+
body,
|
|
319
|
+
depth,
|
|
320
|
+
is_target as isTarget,
|
|
321
|
+
created_at as createdAt
|
|
322
|
+
FROM comment_thread_nodes
|
|
323
|
+
WHERE scan_item_id = ?
|
|
324
|
+
ORDER BY depth ASC, created_at ASC, id ASC`)
|
|
325
|
+
.all(scanItemId);
|
|
326
|
+
return rows.map((row) => ({
|
|
327
|
+
...row,
|
|
328
|
+
isTarget: row.isTarget === 1
|
|
329
|
+
}));
|
|
330
|
+
}
|
|
175
331
|
}
|
|
176
332
|
//# sourceMappingURL=scanItemsRepo.js.map
|
|
@@ -77,6 +77,19 @@ export function getDb() {
|
|
|
77
77
|
FOREIGN KEY (run_id) REFERENCES job_runs(id)
|
|
78
78
|
);
|
|
79
79
|
|
|
80
|
+
CREATE TABLE IF NOT EXISTS comment_thread_nodes (
|
|
81
|
+
id TEXT PRIMARY KEY,
|
|
82
|
+
scan_item_id TEXT NOT NULL,
|
|
83
|
+
reddit_comment_id TEXT NOT NULL,
|
|
84
|
+
parent_reddit_comment_id TEXT,
|
|
85
|
+
author TEXT NOT NULL,
|
|
86
|
+
body TEXT NOT NULL,
|
|
87
|
+
depth INTEGER NOT NULL,
|
|
88
|
+
is_target INTEGER NOT NULL DEFAULT 0,
|
|
89
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
90
|
+
FOREIGN KEY (scan_item_id) REFERENCES scan_items(id)
|
|
91
|
+
);
|
|
92
|
+
|
|
80
93
|
CREATE TABLE IF NOT EXISTS daemon_state (
|
|
81
94
|
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
82
95
|
is_running INTEGER NOT NULL,
|
|
@@ -97,6 +110,11 @@ export function getDb() {
|
|
|
97
110
|
}
|
|
98
111
|
db.exec('CREATE UNIQUE INDEX IF NOT EXISTS idx_jobs_slug ON jobs(slug)');
|
|
99
112
|
db.exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_scan_items_dedup ON scan_items(job_id, reddit_post_id, COALESCE(reddit_comment_id, ''))");
|
|
113
|
+
db.exec('CREATE INDEX IF NOT EXISTS idx_scan_items_job_qualified_posted ON scan_items(job_id, qualified, reddit_posted_at DESC, created_at DESC)');
|
|
114
|
+
db.exec('CREATE INDEX IF NOT EXISTS idx_scan_items_job_created ON scan_items(job_id, created_at DESC)');
|
|
115
|
+
db.exec('CREATE INDEX IF NOT EXISTS idx_scan_items_created ON scan_items(created_at DESC)');
|
|
116
|
+
db.exec('CREATE INDEX IF NOT EXISTS idx_comment_thread_nodes_scan_item_depth ON comment_thread_nodes(scan_item_id, depth ASC)');
|
|
117
|
+
db.exec('CREATE INDEX IF NOT EXISTS idx_comment_thread_nodes_parent ON comment_thread_nodes(parent_reddit_comment_id)');
|
|
100
118
|
try {
|
|
101
119
|
db.exec('ALTER TABLE job_runs ADD COLUMN started_at TEXT');
|
|
102
120
|
}
|
|
@@ -6,5 +6,5 @@ export interface CsvExportSummary {
|
|
|
6
6
|
}
|
|
7
7
|
export declare class CsvResultsExporter {
|
|
8
8
|
private readonly resultsDir;
|
|
9
|
-
exportJobResults(job: Job, qualifiedRows: QualifiedScanItemRow[]): CsvExportSummary;
|
|
9
|
+
exportJobResults(job: Job, qualifiedRows: QualifiedScanItemRow[], exportedAt?: Date): CsvExportSummary;
|
|
10
10
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import fs from 'node:fs';
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { getAppPaths } from '../../utils/paths.js';
|
|
4
|
+
import { createExportFileName } from './fileNaming.js';
|
|
4
5
|
const CSV_HEADERS = ['URL', 'Title', 'Truncated Content', 'Author', 'Justification', 'Date'];
|
|
5
6
|
function truncateContent(value, maxLength) {
|
|
6
7
|
if (value.length <= maxLength) {
|
|
@@ -19,8 +20,8 @@ function toCsvLine(values) {
|
|
|
19
20
|
}
|
|
20
21
|
export class CsvResultsExporter {
|
|
21
22
|
resultsDir = getAppPaths().resultsDir;
|
|
22
|
-
exportJobResults(job, qualifiedRows) {
|
|
23
|
-
const outputPath = path.join(this.resultsDir,
|
|
23
|
+
exportJobResults(job, qualifiedRows, exportedAt = new Date()) {
|
|
24
|
+
const outputPath = path.join(this.resultsDir, createExportFileName(job.slug, 'csv', exportedAt));
|
|
24
25
|
const lines = [toCsvLine([...CSV_HEADERS])];
|
|
25
26
|
for (const row of qualifiedRows) {
|
|
26
27
|
lines.push(toCsvLine([
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
function formatDatePart(value) {
|
|
2
|
+
return String(value).padStart(2, '0');
|
|
3
|
+
}
|
|
4
|
+
export function formatUtcTimestampCompact(date) {
|
|
5
|
+
return `${date.getUTCFullYear()}${formatDatePart(date.getUTCMonth() + 1)}${formatDatePart(date.getUTCDate())}-${formatDatePart(date.getUTCHours())}${formatDatePart(date.getUTCMinutes())}${formatDatePart(date.getUTCSeconds())}`;
|
|
6
|
+
}
|
|
7
|
+
export function createExportFileName(jobSlug, extension, exportedAt = new Date()) {
|
|
8
|
+
return `${formatUtcTimestampCompact(exportedAt)}_${jobSlug}.${extension}`;
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=fileNaming.js.map
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { Job } from '../../types/job.js';
|
|
2
|
+
import type { QualifiedScanItemRow } from '../db/repositories/scanItemsRepo.js';
|
|
3
|
+
export interface JsonExportSummary {
|
|
4
|
+
outputPath: string;
|
|
5
|
+
rowCount: number;
|
|
6
|
+
}
|
|
7
|
+
export declare class JsonResultsExporter {
|
|
8
|
+
private readonly resultsDir;
|
|
9
|
+
exportJobResults(job: Job, qualifiedRows: QualifiedScanItemRow[], exportedAt?: Date): JsonExportSummary;
|
|
10
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { getAppPaths } from '../../utils/paths.js';
|
|
4
|
+
import { createExportFileName } from './fileNaming.js';
|
|
5
|
+
export class JsonResultsExporter {
|
|
6
|
+
resultsDir = getAppPaths().resultsDir;
|
|
7
|
+
exportJobResults(job, qualifiedRows, exportedAt = new Date()) {
|
|
8
|
+
const outputPath = path.join(this.resultsDir, createExportFileName(job.slug, 'json', exportedAt));
|
|
9
|
+
fs.writeFileSync(outputPath, `${JSON.stringify(qualifiedRows, null, 2)}\n`, 'utf8');
|
|
10
|
+
return {
|
|
11
|
+
outputPath,
|
|
12
|
+
rowCount: qualifiedRows.length
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
//# sourceMappingURL=jsonResults.js.map
|