mcp-scraper 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/dist/bin/api-server.cjs +15730 -7780
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-stdio-server.cjs +300 -110
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +1 -1
- package/dist/bin/paa-harvest.cjs +1537 -165
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +1 -1
- package/dist/{chunk-ZBP4RHNW.js → chunk-4743MZHT.js} +298 -106
- package/dist/chunk-4743MZHT.js.map +1 -0
- package/dist/{chunk-LXZDJJXR.js → chunk-D4CJBZBY.js} +426 -29
- package/dist/chunk-D4CJBZBY.js.map +1 -0
- package/dist/chunk-HERFK7W6.js +2781 -0
- package/dist/chunk-HERFK7W6.js.map +1 -0
- package/dist/chunk-Y74EXABN.js +295 -0
- package/dist/chunk-Y74EXABN.js.map +1 -0
- package/dist/{db-IOYMX64U.js → db-YWCNHBLH.js} +36 -4
- package/dist/index.cjs +1660 -237
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +169 -2
- package/dist/index.d.ts +169 -2
- package/dist/index.js +120 -69
- package/dist/index.js.map +1 -1
- package/dist/server-N7Q6H4OR.js +11612 -0
- package/dist/server-N7Q6H4OR.js.map +1 -0
- package/dist/{worker-3ECJHPRE.js → worker-D4D2YQTA.js} +44 -9
- package/dist/worker-D4D2YQTA.js.map +1 -0
- package/package.json +17 -5
- package/dist/chunk-4API3ZCT.js +0 -1387
- package/dist/chunk-4API3ZCT.js.map +0 -1
- package/dist/chunk-LXZDJJXR.js.map +0 -1
- package/dist/chunk-ZBP4RHNW.js.map +0 -1
- package/dist/server-63DR2HE5.js +0 -6062
- package/dist/server-63DR2HE5.js.map +0 -1
- package/dist/worker-3ECJHPRE.js.map +0 -1
- /package/dist/{db-IOYMX64U.js.map → db-YWCNHBLH.js.map} +0 -0
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
// src/api/db.ts
|
|
2
2
|
import { createClient } from "@libsql/client/http";
|
|
3
|
-
import { randomBytes, randomUUID, scryptSync, timingSafeEqual } from "crypto";
|
|
3
|
+
import { createHash, randomBytes, randomUUID, scryptSync, timingSafeEqual } from "crypto";
|
|
4
4
|
import { z } from "zod";
|
|
5
5
|
var DB_URL = process.env.TURSO_DATABASE_URL ?? "file:./paa-api.db";
|
|
6
6
|
var DB_TOKEN = process.env.TURSO_AUTH_TOKEN;
|
|
7
7
|
var _db = null;
|
|
8
|
+
var _rateLimitSchemaReady = false;
|
|
8
9
|
function getDb() {
|
|
9
10
|
if (!_db) _db = createClient({ url: DB_URL, authToken: DB_TOKEN });
|
|
10
11
|
return _db;
|
|
@@ -39,6 +40,40 @@ async function migrate() {
|
|
|
39
40
|
`);
|
|
40
41
|
await db.execute(`CREATE INDEX IF NOT EXISTS jobs_user_id ON jobs(user_id)`);
|
|
41
42
|
await db.execute(`CREATE INDEX IF NOT EXISTS jobs_status ON jobs(status)`);
|
|
43
|
+
await db.execute(`
|
|
44
|
+
CREATE TABLE IF NOT EXISTS harvest_attempts (
|
|
45
|
+
id TEXT PRIMARY KEY,
|
|
46
|
+
job_id TEXT NOT NULL REFERENCES jobs(id),
|
|
47
|
+
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
48
|
+
attempt_number INTEGER NOT NULL,
|
|
49
|
+
max_attempts INTEGER NOT NULL,
|
|
50
|
+
query TEXT NOT NULL,
|
|
51
|
+
location TEXT,
|
|
52
|
+
max_questions INTEGER NOT NULL,
|
|
53
|
+
status TEXT NOT NULL,
|
|
54
|
+
outcome TEXT,
|
|
55
|
+
kernel_session_id TEXT,
|
|
56
|
+
question_count INTEGER,
|
|
57
|
+
duration_ms INTEGER,
|
|
58
|
+
error TEXT,
|
|
59
|
+
will_retry INTEGER,
|
|
60
|
+
kernel_delete_started INTEGER NOT NULL DEFAULT 0,
|
|
61
|
+
kernel_delete_succeeded INTEGER,
|
|
62
|
+
kernel_delete_error TEXT,
|
|
63
|
+
browser_close_succeeded INTEGER,
|
|
64
|
+
browser_close_error TEXT,
|
|
65
|
+
debug_json TEXT,
|
|
66
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
67
|
+
completed_at TEXT,
|
|
68
|
+
UNIQUE(job_id, attempt_number)
|
|
69
|
+
)
|
|
70
|
+
`);
|
|
71
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS harvest_attempts_job_id ON harvest_attempts(job_id)`);
|
|
72
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS harvest_attempts_user_created_at ON harvest_attempts(user_id, created_at DESC)`);
|
|
73
|
+
try {
|
|
74
|
+
await db.execute(`ALTER TABLE harvest_attempts ADD COLUMN debug_json TEXT`);
|
|
75
|
+
} catch {
|
|
76
|
+
}
|
|
42
77
|
try {
|
|
43
78
|
await db.execute(`ALTER TABLE users ADD COLUMN password_hash TEXT`);
|
|
44
79
|
} catch {
|
|
@@ -82,6 +117,14 @@ async function migrate() {
|
|
|
82
117
|
await db.execute(`ALTER TABLE users ADD COLUMN balance_mc INTEGER NOT NULL DEFAULT 0`);
|
|
83
118
|
} catch {
|
|
84
119
|
}
|
|
120
|
+
try {
|
|
121
|
+
await db.execute(`ALTER TABLE users ADD COLUMN extra_concurrency_slots INTEGER NOT NULL DEFAULT 0`);
|
|
122
|
+
} catch {
|
|
123
|
+
}
|
|
124
|
+
try {
|
|
125
|
+
await db.execute(`ALTER TABLE users ADD COLUMN concurrency_stripe_sub_id TEXT`);
|
|
126
|
+
} catch {
|
|
127
|
+
}
|
|
85
128
|
await db.execute(`
|
|
86
129
|
CREATE TABLE IF NOT EXISTS ledger (
|
|
87
130
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
@@ -94,6 +137,14 @@ async function migrate() {
|
|
|
94
137
|
)
|
|
95
138
|
`);
|
|
96
139
|
await db.execute(`CREATE INDEX IF NOT EXISTS ledger_user_id ON ledger(user_id)`);
|
|
140
|
+
await db.execute(`
|
|
141
|
+
CREATE TABLE IF NOT EXISTS free_credit_refreshes (
|
|
142
|
+
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
143
|
+
month TEXT NOT NULL,
|
|
144
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
145
|
+
PRIMARY KEY (user_id, month)
|
|
146
|
+
)
|
|
147
|
+
`);
|
|
97
148
|
await db.execute(`
|
|
98
149
|
CREATE TABLE IF NOT EXISTS site_audit_jobs (
|
|
99
150
|
id TEXT PRIMARY KEY,
|
|
@@ -120,6 +171,61 @@ async function migrate() {
|
|
|
120
171
|
await db.execute(`CREATE INDEX IF NOT EXISTS site_audit_jobs_user_id ON site_audit_jobs(user_id)`);
|
|
121
172
|
await db.execute(`CREATE INDEX IF NOT EXISTS site_audit_jobs_status ON site_audit_jobs(status)`);
|
|
122
173
|
await db.execute(`CREATE INDEX IF NOT EXISTS site_audit_phase_log_job_id ON site_audit_phase_log(job_id)`);
|
|
174
|
+
await db.execute(`
|
|
175
|
+
CREATE TABLE IF NOT EXISTS password_reset_tokens (
|
|
176
|
+
token TEXT PRIMARY KEY,
|
|
177
|
+
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
178
|
+
expires_at TEXT NOT NULL,
|
|
179
|
+
used INTEGER NOT NULL DEFAULT 0,
|
|
180
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
181
|
+
)
|
|
182
|
+
`);
|
|
183
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS prt_user_id ON password_reset_tokens(user_id)`);
|
|
184
|
+
await db.execute(`
|
|
185
|
+
CREATE TABLE IF NOT EXISTS rate_limits (
|
|
186
|
+
scope TEXT NOT NULL,
|
|
187
|
+
key_hash TEXT NOT NULL,
|
|
188
|
+
window_start INTEGER NOT NULL,
|
|
189
|
+
count INTEGER NOT NULL DEFAULT 0,
|
|
190
|
+
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
191
|
+
PRIMARY KEY (scope, key_hash, window_start)
|
|
192
|
+
)
|
|
193
|
+
`);
|
|
194
|
+
await db.execute(`
|
|
195
|
+
CREATE TABLE IF NOT EXISTS request_events (
|
|
196
|
+
id TEXT PRIMARY KEY,
|
|
197
|
+
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
198
|
+
source TEXT NOT NULL,
|
|
199
|
+
status TEXT NOT NULL,
|
|
200
|
+
query TEXT NOT NULL,
|
|
201
|
+
location TEXT,
|
|
202
|
+
result_count INTEGER,
|
|
203
|
+
result TEXT,
|
|
204
|
+
error TEXT,
|
|
205
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
206
|
+
)
|
|
207
|
+
`);
|
|
208
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS request_events_user_created_at ON request_events(user_id, created_at DESC)`);
|
|
209
|
+
await db.execute(`
|
|
210
|
+
CREATE TABLE IF NOT EXISTS stripe_events (
|
|
211
|
+
event_id TEXT PRIMARY KEY,
|
|
212
|
+
processed_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
213
|
+
)
|
|
214
|
+
`);
|
|
215
|
+
}
|
|
216
|
+
async function recordStripeEvent(eventId) {
|
|
217
|
+
const res = await getDb().execute({
|
|
218
|
+
sql: "INSERT OR IGNORE INTO stripe_events (event_id) VALUES (?)",
|
|
219
|
+
args: [eventId]
|
|
220
|
+
});
|
|
221
|
+
return res.rowsAffected > 0;
|
|
222
|
+
}
|
|
223
|
+
async function stripeEventAlreadyProcessed(eventId) {
|
|
224
|
+
const res = await getDb().execute({
|
|
225
|
+
sql: "SELECT 1 FROM stripe_events WHERE event_id = ? LIMIT 1",
|
|
226
|
+
args: [eventId]
|
|
227
|
+
});
|
|
228
|
+
return res.rows.length > 0;
|
|
123
229
|
}
|
|
124
230
|
function generateApiKey() {
|
|
125
231
|
return "sk_" + randomBytes(24).toString("hex");
|
|
@@ -146,7 +252,9 @@ function rowToUser(row) {
|
|
|
146
252
|
created_at: String(row.created_at),
|
|
147
253
|
active: Number(row.active),
|
|
148
254
|
stripe_customer_id: row.stripe_customer_id != null ? String(row.stripe_customer_id) : null,
|
|
149
|
-
balance_mc: Number(row.balance_mc ?? 0)
|
|
255
|
+
balance_mc: Number(row.balance_mc ?? 0),
|
|
256
|
+
extra_concurrency_slots: Number(row.extra_concurrency_slots ?? 0),
|
|
257
|
+
concurrency_stripe_sub_id: row.concurrency_stripe_sub_id != null ? String(row.concurrency_stripe_sub_id) : null
|
|
150
258
|
};
|
|
151
259
|
}
|
|
152
260
|
async function getUserByEmail(email) {
|
|
@@ -181,17 +289,80 @@ async function getUserStats(userId) {
|
|
|
181
289
|
lastUsed: luR.rows[0]?.t != null ? String(luR.rows[0].t) : null
|
|
182
290
|
};
|
|
183
291
|
}
|
|
184
|
-
async function createUser(email, name, password) {
|
|
292
|
+
async function createUser(email, name, password, stripeCustomerId) {
|
|
185
293
|
const db = getDb();
|
|
186
294
|
const api_key = generateApiKey();
|
|
187
295
|
const plainPassword = password ?? randomBytes(6).toString("hex");
|
|
188
296
|
const password_hash = hashPassword(plainPassword);
|
|
189
297
|
const result = await db.execute({
|
|
190
|
-
sql: "INSERT INTO users (email, name, api_key, password_hash) VALUES (?, ?, ?, ?)",
|
|
191
|
-
args: [email, name ?? null, api_key, password_hash]
|
|
298
|
+
sql: "INSERT INTO users (email, name, api_key, password_hash, stripe_customer_id) VALUES (?, ?, ?, ?, ?)",
|
|
299
|
+
args: [email, name ?? null, api_key, password_hash, stripeCustomerId ?? null]
|
|
192
300
|
});
|
|
193
301
|
return { id: result.lastInsertRowid, email, name, api_key, password: plainPassword };
|
|
194
302
|
}
|
|
303
|
+
async function checkRateLimit(scope, key, limit, windowSeconds) {
|
|
304
|
+
const nowSeconds = Math.floor(Date.now() / 1e3);
|
|
305
|
+
const windowStart = Math.floor(nowSeconds / windowSeconds) * windowSeconds;
|
|
306
|
+
const keyHash = createHash("sha256").update(key).digest("hex");
|
|
307
|
+
const db = getDb();
|
|
308
|
+
if (!_rateLimitSchemaReady) {
|
|
309
|
+
await db.execute(`
|
|
310
|
+
CREATE TABLE IF NOT EXISTS rate_limits (
|
|
311
|
+
scope TEXT NOT NULL,
|
|
312
|
+
key_hash TEXT NOT NULL,
|
|
313
|
+
window_start INTEGER NOT NULL,
|
|
314
|
+
count INTEGER NOT NULL DEFAULT 0,
|
|
315
|
+
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
316
|
+
PRIMARY KEY (scope, key_hash, window_start)
|
|
317
|
+
)
|
|
318
|
+
`);
|
|
319
|
+
_rateLimitSchemaReady = true;
|
|
320
|
+
}
|
|
321
|
+
await db.execute({
|
|
322
|
+
sql: `
|
|
323
|
+
INSERT INTO rate_limits (scope, key_hash, window_start, count)
|
|
324
|
+
VALUES (?, ?, ?, 1)
|
|
325
|
+
ON CONFLICT(scope, key_hash, window_start)
|
|
326
|
+
DO UPDATE SET count = count + 1, updated_at = datetime('now')
|
|
327
|
+
`,
|
|
328
|
+
args: [scope, keyHash, windowStart]
|
|
329
|
+
});
|
|
330
|
+
const res = await db.execute({
|
|
331
|
+
sql: "SELECT count FROM rate_limits WHERE scope = ? AND key_hash = ? AND window_start = ?",
|
|
332
|
+
args: [scope, keyHash, windowStart]
|
|
333
|
+
});
|
|
334
|
+
const count = Number(res.rows[0]?.count ?? 1);
|
|
335
|
+
return {
|
|
336
|
+
allowed: count <= limit,
|
|
337
|
+
count,
|
|
338
|
+
remaining: Math.max(0, limit - count),
|
|
339
|
+
resetSeconds: Math.max(1, windowStart + windowSeconds - nowSeconds)
|
|
340
|
+
};
|
|
341
|
+
}
|
|
342
|
+
async function createPasswordResetToken(userId) {
|
|
343
|
+
const token = randomBytes(32).toString("hex");
|
|
344
|
+
const tokenHash = createHash("sha256").update(token).digest("hex");
|
|
345
|
+
const expiresAt = new Date(Date.now() + 60 * 60 * 1e3).toISOString();
|
|
346
|
+
await getDb().execute({
|
|
347
|
+
sql: "INSERT INTO password_reset_tokens (token, user_id, expires_at) VALUES (?, ?, ?)",
|
|
348
|
+
args: [tokenHash, userId, expiresAt]
|
|
349
|
+
});
|
|
350
|
+
return token;
|
|
351
|
+
}
|
|
352
|
+
async function consumePasswordResetToken(token) {
|
|
353
|
+
const db = getDb();
|
|
354
|
+
const tokenHash = createHash("sha256").update(token).digest("hex");
|
|
355
|
+
const res = await db.execute({
|
|
356
|
+
sql: "SELECT user_id, expires_at, used FROM password_reset_tokens WHERE token = ?",
|
|
357
|
+
args: [tokenHash]
|
|
358
|
+
});
|
|
359
|
+
const row = res.rows[0];
|
|
360
|
+
if (!row) return null;
|
|
361
|
+
if (Number(row.used)) return null;
|
|
362
|
+
if (new Date(String(row.expires_at)) < /* @__PURE__ */ new Date()) return null;
|
|
363
|
+
await db.execute({ sql: "UPDATE password_reset_tokens SET used = 1 WHERE token = ?", args: [tokenHash] });
|
|
364
|
+
return Number(row.user_id);
|
|
365
|
+
}
|
|
195
366
|
async function rotateApiKey(userId) {
|
|
196
367
|
const newKey = generateApiKey();
|
|
197
368
|
await getDb().execute({ sql: "UPDATE users SET api_key = ?, key_active = 1 WHERE id = ?", args: [newKey, userId] });
|
|
@@ -222,6 +393,44 @@ function rowToRawJob(row) {
|
|
|
222
393
|
completed_at: row.completed_at != null ? String(row.completed_at) : null
|
|
223
394
|
};
|
|
224
395
|
}
|
|
396
|
+
function nullableBoolean(value) {
|
|
397
|
+
return value == null ? null : Number(value) === 1;
|
|
398
|
+
}
|
|
399
|
+
function parseNullableJson(value) {
|
|
400
|
+
if (value == null) return null;
|
|
401
|
+
try {
|
|
402
|
+
return JSON.parse(String(value));
|
|
403
|
+
} catch {
|
|
404
|
+
return null;
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
function rowToHarvestAttemptLog(row) {
|
|
408
|
+
return {
|
|
409
|
+
id: String(row.id),
|
|
410
|
+
job_id: String(row.job_id),
|
|
411
|
+
user_id: Number(row.user_id),
|
|
412
|
+
attempt_number: Number(row.attempt_number),
|
|
413
|
+
max_attempts: Number(row.max_attempts),
|
|
414
|
+
query: String(row.query),
|
|
415
|
+
location: row.location != null ? String(row.location) : null,
|
|
416
|
+
max_questions: Number(row.max_questions),
|
|
417
|
+
status: String(row.status),
|
|
418
|
+
outcome: row.outcome != null ? String(row.outcome) : null,
|
|
419
|
+
kernel_session_id: row.kernel_session_id != null ? String(row.kernel_session_id) : null,
|
|
420
|
+
question_count: row.question_count != null ? Number(row.question_count) : null,
|
|
421
|
+
duration_ms: row.duration_ms != null ? Number(row.duration_ms) : null,
|
|
422
|
+
error: row.error != null ? String(row.error) : null,
|
|
423
|
+
will_retry: nullableBoolean(row.will_retry),
|
|
424
|
+
kernel_delete_started: Number(row.kernel_delete_started ?? 0) === 1,
|
|
425
|
+
kernel_delete_succeeded: nullableBoolean(row.kernel_delete_succeeded),
|
|
426
|
+
kernel_delete_error: row.kernel_delete_error != null ? String(row.kernel_delete_error) : null,
|
|
427
|
+
browser_close_succeeded: nullableBoolean(row.browser_close_succeeded),
|
|
428
|
+
browser_close_error: row.browser_close_error != null ? String(row.browser_close_error) : null,
|
|
429
|
+
debug: parseNullableJson(row.debug_json),
|
|
430
|
+
created_at: String(row.created_at),
|
|
431
|
+
completed_at: row.completed_at != null ? String(row.completed_at) : null
|
|
432
|
+
};
|
|
433
|
+
}
|
|
225
434
|
function deserialize(raw) {
|
|
226
435
|
return {
|
|
227
436
|
...raw,
|
|
@@ -245,6 +454,89 @@ async function createRunningJob(userId, query, options) {
|
|
|
245
454
|
});
|
|
246
455
|
return id;
|
|
247
456
|
}
|
|
457
|
+
async function startHarvestAttempt(input) {
|
|
458
|
+
await getDb().execute({
|
|
459
|
+
sql: `
|
|
460
|
+
INSERT INTO harvest_attempts (
|
|
461
|
+
id, job_id, user_id, attempt_number, max_attempts, query, location, max_questions, status, created_at
|
|
462
|
+
)
|
|
463
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'running', ?)
|
|
464
|
+
ON CONFLICT(job_id, attempt_number) DO UPDATE SET
|
|
465
|
+
status = 'running',
|
|
466
|
+
created_at = excluded.created_at,
|
|
467
|
+
completed_at = NULL,
|
|
468
|
+
outcome = NULL,
|
|
469
|
+
error = NULL,
|
|
470
|
+
debug_json = NULL
|
|
471
|
+
`,
|
|
472
|
+
args: [
|
|
473
|
+
randomUUID(),
|
|
474
|
+
input.jobId,
|
|
475
|
+
input.userId,
|
|
476
|
+
input.attemptNumber,
|
|
477
|
+
input.maxAttempts,
|
|
478
|
+
input.query,
|
|
479
|
+
input.location,
|
|
480
|
+
input.maxQuestions,
|
|
481
|
+
input.startedAt
|
|
482
|
+
]
|
|
483
|
+
});
|
|
484
|
+
}
|
|
485
|
+
async function finishHarvestAttempt(input) {
|
|
486
|
+
await getDb().execute({
|
|
487
|
+
sql: `
|
|
488
|
+
UPDATE harvest_attempts SET
|
|
489
|
+
status = 'finished',
|
|
490
|
+
outcome = ?,
|
|
491
|
+
kernel_session_id = ?,
|
|
492
|
+
question_count = ?,
|
|
493
|
+
duration_ms = ?,
|
|
494
|
+
error = ?,
|
|
495
|
+
will_retry = ?,
|
|
496
|
+
kernel_delete_started = ?,
|
|
497
|
+
kernel_delete_succeeded = ?,
|
|
498
|
+
kernel_delete_error = ?,
|
|
499
|
+
browser_close_succeeded = ?,
|
|
500
|
+
browser_close_error = ?,
|
|
501
|
+
debug_json = ?,
|
|
502
|
+
completed_at = ?
|
|
503
|
+
WHERE job_id = ? AND attempt_number = ?
|
|
504
|
+
`,
|
|
505
|
+
args: [
|
|
506
|
+
input.outcome,
|
|
507
|
+
input.kernelSessionId,
|
|
508
|
+
input.questionCount,
|
|
509
|
+
input.durationMs,
|
|
510
|
+
input.error,
|
|
511
|
+
input.willRetry ? 1 : 0,
|
|
512
|
+
input.kernelDeleteStarted ? 1 : 0,
|
|
513
|
+
input.kernelDeleteSucceeded == null ? null : input.kernelDeleteSucceeded ? 1 : 0,
|
|
514
|
+
input.kernelDeleteError,
|
|
515
|
+
input.browserCloseSucceeded == null ? null : input.browserCloseSucceeded ? 1 : 0,
|
|
516
|
+
input.browserCloseError,
|
|
517
|
+
input.debug == null ? null : JSON.stringify(input.debug),
|
|
518
|
+
input.completedAt,
|
|
519
|
+
input.jobId,
|
|
520
|
+
input.attemptNumber
|
|
521
|
+
]
|
|
522
|
+
});
|
|
523
|
+
}
|
|
524
|
+
async function listHarvestAttempts(jobId, userId) {
|
|
525
|
+
const res = userId == null ? await getDb().execute({
|
|
526
|
+
sql: `SELECT * FROM harvest_attempts WHERE job_id = ? ORDER BY attempt_number ASC`,
|
|
527
|
+
args: [jobId]
|
|
528
|
+
}) : await getDb().execute({
|
|
529
|
+
sql: `
|
|
530
|
+
SELECT ha.*
|
|
531
|
+
FROM harvest_attempts ha
|
|
532
|
+
INNER JOIN jobs j ON j.id = ha.job_id
|
|
533
|
+
WHERE ha.job_id = ? AND j.user_id = ?
|
|
534
|
+
ORDER BY ha.attempt_number ASC
|
|
535
|
+
`,
|
|
536
|
+
args: [jobId, userId]
|
|
537
|
+
});
|
|
538
|
+
return res.rows.map((r) => rowToHarvestAttemptLog(r));
|
|
539
|
+
}
|
|
248
540
|
async function getJob(id, userId) {
|
|
249
541
|
const db = getDb();
|
|
250
542
|
const res = userId ? await db.execute({ sql: "SELECT * FROM jobs WHERE id = ? AND user_id = ?", args: [id, userId] }) : await db.execute({ sql: "SELECT * FROM jobs WHERE id = ?", args: [id] });
|
|
@@ -254,6 +546,55 @@ async function listJobs(userId) {
|
|
|
254
546
|
const res = await getDb().execute({ sql: "SELECT * FROM jobs WHERE user_id = ? ORDER BY created_at DESC LIMIT 50", args: [userId] });
|
|
255
547
|
return res.rows.map((r) => deserialize(rowToRawJob(r)));
|
|
256
548
|
}
|
|
549
|
+
function rowToRequestEvent(row) {
|
|
550
|
+
const rawResult = row.result != null ? String(row.result) : null;
|
|
551
|
+
return {
|
|
552
|
+
id: String(row.id),
|
|
553
|
+
user_id: Number(row.user_id),
|
|
554
|
+
source: String(row.source),
|
|
555
|
+
status: String(row.status),
|
|
556
|
+
query: String(row.query),
|
|
557
|
+
location: row.location != null ? String(row.location) : null,
|
|
558
|
+
result_count: row.result_count != null ? Number(row.result_count) : null,
|
|
559
|
+
result: rawResult ? JSON.parse(rawResult) : null,
|
|
560
|
+
error: row.error != null ? String(row.error) : null,
|
|
561
|
+
created_at: String(row.created_at)
|
|
562
|
+
};
|
|
563
|
+
}
|
|
564
|
+
async function logRequestEvent(input) {
|
|
565
|
+
const id = randomUUID();
|
|
566
|
+
await getDb().execute({
|
|
567
|
+
sql: `
|
|
568
|
+
INSERT INTO request_events (id, user_id, source, status, query, location, result_count, result, error)
|
|
569
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
570
|
+
`,
|
|
571
|
+
args: [
|
|
572
|
+
id,
|
|
573
|
+
input.userId,
|
|
574
|
+
input.source,
|
|
575
|
+
input.status,
|
|
576
|
+
input.query,
|
|
577
|
+
input.location ?? null,
|
|
578
|
+
input.resultCount ?? null,
|
|
579
|
+
input.result != null ? JSON.stringify(input.result) : null,
|
|
580
|
+
input.error ?? null
|
|
581
|
+
]
|
|
582
|
+
});
|
|
583
|
+
return id;
|
|
584
|
+
}
|
|
585
|
+
async function listRequestEvents(userId, limit = 100) {
|
|
586
|
+
const res = await getDb().execute({
|
|
587
|
+
sql: `
|
|
588
|
+
SELECT id, user_id, source, status, query, location, result_count, result, error, created_at
|
|
589
|
+
FROM request_events
|
|
590
|
+
WHERE user_id = ?
|
|
591
|
+
ORDER BY created_at DESC
|
|
592
|
+
LIMIT ?
|
|
593
|
+
`,
|
|
594
|
+
args: [userId, limit]
|
|
595
|
+
});
|
|
596
|
+
return res.rows.map((r) => rowToRequestEvent(r));
|
|
597
|
+
}
|
|
257
598
|
async function countActiveUsers() {
|
|
258
599
|
const res = await getDb().execute(`SELECT COUNT(*) as n FROM users WHERE active = 1`);
|
|
259
600
|
return Number(res.rows[0]?.n ?? 0);
|
|
@@ -265,13 +606,6 @@ async function countActiveJobsForUser(userId) {
|
|
|
265
606
|
});
|
|
266
607
|
return Number(res.rows[0]?.n ?? 0);
|
|
267
608
|
}
|
|
268
|
-
async function countJobsLast7Days(userId) {
|
|
269
|
-
const res = await getDb().execute({
|
|
270
|
-
sql: `SELECT COUNT(*) as n FROM jobs WHERE user_id = ? AND created_at > datetime('now', '-7 days')`,
|
|
271
|
-
args: [userId]
|
|
272
|
-
});
|
|
273
|
-
return Number(res.rows[0]?.n ?? 0);
|
|
274
|
-
}
|
|
275
609
|
async function claimPendingJob() {
|
|
276
610
|
const db = getDb();
|
|
277
611
|
const res = await db.execute(`SELECT * FROM jobs WHERE status = 'pending' ORDER BY created_at LIMIT 1`);
|
|
@@ -289,6 +623,9 @@ async function completeJob(id, result) {
|
|
|
289
623
|
async function failJob(id, error) {
|
|
290
624
|
await getDb().execute({ sql: `UPDATE jobs SET status = 'failed', error = ?, completed_at = datetime('now') WHERE id = ?`, args: [error, id] });
|
|
291
625
|
}
|
|
626
|
+
async function cancelJob(id, reason) {
|
|
627
|
+
await getDb().execute({ sql: `UPDATE jobs SET status = 'cancelled', error = ?, completed_at = datetime('now') WHERE id = ?`, args: [reason, id] });
|
|
628
|
+
}
|
|
292
629
|
function rowToKpoJob(row) {
|
|
293
630
|
return {
|
|
294
631
|
id: String(row.id),
|
|
@@ -367,33 +704,57 @@ async function setStripeCustomerId(userId, customerId) {
|
|
|
367
704
|
args: [customerId, userId]
|
|
368
705
|
});
|
|
369
706
|
}
|
|
370
|
-
async function
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
args: [mc, userId]
|
|
707
|
+
async function setExtraConcurrencySlots(userId, slots) {
|
|
708
|
+
await getDb().execute({
|
|
709
|
+
sql: "UPDATE users SET extra_concurrency_slots = ? WHERE id = ?",
|
|
710
|
+
args: [Math.max(0, slots), userId]
|
|
375
711
|
});
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
712
|
+
}
|
|
713
|
+
async function setConcurrencySubId(userId, subId) {
|
|
714
|
+
await getDb().execute({
|
|
715
|
+
sql: "UPDATE users SET concurrency_stripe_sub_id = ? WHERE id = ?",
|
|
716
|
+
args: [subId, userId]
|
|
379
717
|
});
|
|
718
|
+
}
|
|
719
|
+
async function creditMc(userId, mc, operation, description, stripePaymentIntent) {
|
|
720
|
+
const db = getDb();
|
|
721
|
+
await db.batch([
|
|
722
|
+
{ sql: "UPDATE users SET balance_mc = balance_mc + ? WHERE id = ?", args: [mc, userId] },
|
|
723
|
+
{ sql: "INSERT INTO ledger (user_id, amount_mc, operation, description, stripe_pi) VALUES (?, ?, ?, ?, ?)", args: [userId, mc, operation, description ?? null, stripePaymentIntent ?? null] }
|
|
724
|
+
], "write");
|
|
380
725
|
const res = await db.execute({ sql: "SELECT balance_mc FROM users WHERE id = ?", args: [userId] });
|
|
381
726
|
return Number(res.rows[0]?.balance_mc ?? 0);
|
|
382
727
|
}
|
|
728
|
+
async function ledgerExistsForOperation(userId, operation) {
|
|
729
|
+
const res = await getDb().execute({
|
|
730
|
+
sql: "SELECT 1 FROM ledger WHERE user_id = ? AND operation = ? LIMIT 1",
|
|
731
|
+
args: [userId, operation]
|
|
732
|
+
});
|
|
733
|
+
return res.rows.length > 0;
|
|
734
|
+
}
|
|
735
|
+
async function claimMonthlyFreeRefresh(userId, month) {
|
|
736
|
+
const res = await getDb().execute({
|
|
737
|
+
sql: "INSERT OR IGNORE INTO free_credit_refreshes (user_id, month) VALUES (?, ?)",
|
|
738
|
+
args: [userId, month]
|
|
739
|
+
});
|
|
740
|
+
return res.rowsAffected > 0;
|
|
741
|
+
}
|
|
383
742
|
async function debitMc(userId, mc, operation, description) {
|
|
384
743
|
const db = getDb();
|
|
385
|
-
const
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
await db.execute({
|
|
389
|
-
sql: "UPDATE users SET balance_mc = balance_mc - ? WHERE id = ?",
|
|
390
|
-
args: [mc, userId]
|
|
744
|
+
const upd = await db.execute({
|
|
745
|
+
sql: "UPDATE users SET balance_mc = balance_mc - ? WHERE id = ? AND balance_mc >= ?",
|
|
746
|
+
args: [mc, userId, mc]
|
|
391
747
|
});
|
|
748
|
+
if (upd.rowsAffected === 0) {
|
|
749
|
+
const res2 = await db.execute({ sql: "SELECT balance_mc FROM users WHERE id = ?", args: [userId] });
|
|
750
|
+
return { ok: false, balance_mc: Number(res2.rows[0]?.balance_mc ?? 0) };
|
|
751
|
+
}
|
|
392
752
|
await db.execute({
|
|
393
753
|
sql: "INSERT INTO ledger (user_id, amount_mc, operation, description) VALUES (?, ?, ?, ?)",
|
|
394
754
|
args: [userId, -mc, operation, description ?? null]
|
|
395
755
|
});
|
|
396
|
-
|
|
756
|
+
const res = await db.execute({ sql: "SELECT balance_mc FROM users WHERE id = ?", args: [userId] });
|
|
757
|
+
return { ok: true, balance_mc: Number(res.rows[0]?.balance_mc ?? 0) };
|
|
397
758
|
}
|
|
398
759
|
async function getLedger(userId, limit = 50) {
|
|
399
760
|
const res = await getDb().execute({
|
|
@@ -410,6 +771,26 @@ async function getLedger(userId, limit = 50) {
|
|
|
410
771
|
created_at: String(r.created_at)
|
|
411
772
|
}));
|
|
412
773
|
}
|
|
774
|
+
async function ledgerExistsForStripePI(stripePI) {
|
|
775
|
+
const res = await getDb().execute({
|
|
776
|
+
sql: "SELECT 1 FROM ledger WHERE stripe_pi = ? LIMIT 1",
|
|
777
|
+
args: [stripePI]
|
|
778
|
+
});
|
|
779
|
+
return res.rows.length > 0;
|
|
780
|
+
}
|
|
781
|
+
async function reconcileBalanceMc(userId) {
|
|
782
|
+
const db = getDb();
|
|
783
|
+
const res = await db.execute({
|
|
784
|
+
sql: "SELECT COALESCE(SUM(amount_mc), 0) AS balance_mc FROM ledger WHERE user_id = ?",
|
|
785
|
+
args: [userId]
|
|
786
|
+
});
|
|
787
|
+
const balanceMc = Number(res.rows[0]?.balance_mc ?? 0);
|
|
788
|
+
await db.execute({
|
|
789
|
+
sql: "UPDATE users SET balance_mc = ? WHERE id = ? AND balance_mc != ?",
|
|
790
|
+
args: [balanceMc, userId, balanceMc]
|
|
791
|
+
});
|
|
792
|
+
return balanceMc;
|
|
793
|
+
}
|
|
413
794
|
var SiteAuditJobRowSchema = z.object({
|
|
414
795
|
id: z.string(),
|
|
415
796
|
user_id: z.number(),
|
|
@@ -433,6 +814,8 @@ var SiteAuditPhaseLogRowSchema = z.object({
|
|
|
433
814
|
export {
|
|
434
815
|
getDb,
|
|
435
816
|
migrate,
|
|
817
|
+
recordStripeEvent,
|
|
818
|
+
stripeEventAlreadyProcessed,
|
|
436
819
|
generateApiKey,
|
|
437
820
|
hashPassword,
|
|
438
821
|
verifyPassword,
|
|
@@ -442,20 +825,28 @@ export {
|
|
|
442
825
|
setPassword,
|
|
443
826
|
getUserStats,
|
|
444
827
|
createUser,
|
|
828
|
+
checkRateLimit,
|
|
829
|
+
createPasswordResetToken,
|
|
830
|
+
consumePasswordResetToken,
|
|
445
831
|
rotateApiKey,
|
|
446
832
|
revokeApiKey,
|
|
447
833
|
listUsers,
|
|
448
834
|
deactivateUser,
|
|
449
835
|
createJob,
|
|
450
836
|
createRunningJob,
|
|
837
|
+
startHarvestAttempt,
|
|
838
|
+
finishHarvestAttempt,
|
|
839
|
+
listHarvestAttempts,
|
|
451
840
|
getJob,
|
|
452
841
|
listJobs,
|
|
842
|
+
logRequestEvent,
|
|
843
|
+
listRequestEvents,
|
|
453
844
|
countActiveUsers,
|
|
454
845
|
countActiveJobsForUser,
|
|
455
|
-
countJobsLast7Days,
|
|
456
846
|
claimPendingJob,
|
|
457
847
|
completeJob,
|
|
458
848
|
failJob,
|
|
849
|
+
cancelJob,
|
|
459
850
|
createKpoJob,
|
|
460
851
|
claimPendingKpoJob,
|
|
461
852
|
getKpoJob,
|
|
@@ -467,10 +858,16 @@ export {
|
|
|
467
858
|
listKpoJobs,
|
|
468
859
|
getUserByStripeCustomerId,
|
|
469
860
|
setStripeCustomerId,
|
|
861
|
+
setExtraConcurrencySlots,
|
|
862
|
+
setConcurrencySubId,
|
|
470
863
|
creditMc,
|
|
864
|
+
ledgerExistsForOperation,
|
|
865
|
+
claimMonthlyFreeRefresh,
|
|
471
866
|
debitMc,
|
|
472
867
|
getLedger,
|
|
868
|
+
ledgerExistsForStripePI,
|
|
869
|
+
reconcileBalanceMc,
|
|
473
870
|
SiteAuditJobRowSchema,
|
|
474
871
|
SiteAuditPhaseLogRowSchema
|
|
475
872
|
};
|
|
476
|
-
//# sourceMappingURL=chunk-
|
|
873
|
+
//# sourceMappingURL=chunk-D4CJBZBY.js.map
|