mcp-scraper 0.36.0 → 0.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/bin/api-server.cjs +766 -168
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +52 -8
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +5 -5
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +3 -3
- package/dist/chunk-3LWYPAU5.js +7 -0
- package/dist/chunk-3LWYPAU5.js.map +1 -0
- package/dist/{chunk-IFYER7O4.js → chunk-CSCD2HNS.js} +132 -2
- package/dist/chunk-CSCD2HNS.js.map +1 -0
- package/dist/{chunk-D7LM5QZN.js → chunk-FSAXLDB3.js} +2 -2
- package/dist/{chunk-5RULXBJ7.js → chunk-JK2FRDAP.js} +55 -11
- package/dist/{chunk-5RULXBJ7.js.map → chunk-JK2FRDAP.js.map} +1 -1
- package/dist/{chunk-ANCGXUQJ.js → chunk-LOPKN3YL.js} +2 -2
- package/dist/{chunk-E5UEELA7.js → chunk-MA5JBAUZ.js} +2 -2
- package/dist/{chunk-AN3VQARU.js → chunk-PUJFYJXB.js} +2 -2
- package/dist/{chunk-4ZIJ3BKZ.js → chunk-PWPUKR5U.js} +4 -2
- package/dist/{chunk-4ZIJ3BKZ.js.map → chunk-PWPUKR5U.js.map} +1 -1
- package/dist/{chunk-O2MCWFXQ.js → chunk-Q35WZJJK.js} +3 -3
- package/dist/{db-N6MPVMEF.js → db-N3YECFWR.js} +6 -2
- package/dist/{extract-bundle-M4SDJG3V.js → extract-bundle-346R6MXD.js} +4 -4
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +3 -3
- package/dist/{location-data-repository-Z4NQOU5Y.js → location-data-repository-O2VII3ON.js} +3 -3
- package/dist/{server-ZIGAFKOL.js → server-QKDDEBVQ.js} +500 -99
- package/dist/server-QKDDEBVQ.js.map +1 -0
- package/dist/{site-extract-repository-PGQZNW6V.js → site-extract-repository-I3VM6WXN.js} +4 -4
- package/dist/{worker-EBB6CTGW.js → worker-TTFXPPDK.js} +6 -6
- package/docs/mcp-tool-manifest.generated.json +182 -6
- package/package.json +2 -2
- package/dist/chunk-4HO66323.js +0 -7
- package/dist/chunk-4HO66323.js.map +0 -1
- package/dist/chunk-IFYER7O4.js.map +0 -1
- package/dist/server-ZIGAFKOL.js.map +0 -1
- /package/dist/{chunk-D7LM5QZN.js.map → chunk-FSAXLDB3.js.map} +0 -0
- /package/dist/{chunk-ANCGXUQJ.js.map → chunk-LOPKN3YL.js.map} +0 -0
- /package/dist/{chunk-E5UEELA7.js.map → chunk-MA5JBAUZ.js.map} +0 -0
- /package/dist/{chunk-AN3VQARU.js.map → chunk-PUJFYJXB.js.map} +0 -0
- /package/dist/{chunk-O2MCWFXQ.js.map → chunk-Q35WZJJK.js.map} +0 -0
- /package/dist/{db-N6MPVMEF.js.map → db-N3YECFWR.js.map} +0 -0
- /package/dist/{extract-bundle-M4SDJG3V.js.map → extract-bundle-346R6MXD.js.map} +0 -0
- /package/dist/{location-data-repository-Z4NQOU5Y.js.map → location-data-repository-O2VII3ON.js.map} +0 -0
- /package/dist/{site-extract-repository-PGQZNW6V.js.map → site-extract-repository-I3VM6WXN.js.map} +0 -0
- /package/dist/{worker-EBB6CTGW.js.map → worker-TTFXPPDK.js.map} +0 -0
package/dist/bin/paa-harvest.js
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
harvest
|
|
4
|
-
} from "../chunk-
|
|
4
|
+
} from "../chunk-LOPKN3YL.js";
|
|
5
5
|
import {
|
|
6
6
|
browserServiceApiKey
|
|
7
|
-
} from "../chunk-
|
|
7
|
+
} from "../chunk-MA5JBAUZ.js";
|
|
8
8
|
import "../chunk-K443GQY5.js";
|
|
9
9
|
import "../chunk-CB5C3BPB.js";
|
|
10
10
|
import "../chunk-44HZLHDV.js";
|
|
11
|
-
import "../chunk-
|
|
11
|
+
import "../chunk-CSCD2HNS.js";
|
|
12
12
|
|
|
13
13
|
// src/cli.ts
|
|
14
14
|
import { Command } from "commander";
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/version.ts"],"sourcesContent":["export const PACKAGE_VERSION = '0.37.0'\n"],"mappings":";AAAO,IAAM,kBAAkB;","names":[]}
|
|
@@ -6,9 +6,12 @@ var DB_URL = process.env.TURSO_DATABASE_URL ?? "file:./paa-api.db";
|
|
|
6
6
|
var DB_TOKEN = process.env.TURSO_AUTH_TOKEN;
|
|
7
7
|
var _db = null;
|
|
8
8
|
var _rateLimitSchemaReady = false;
|
|
9
|
-
var CORE_SCHEMA_VERSION = "2026-07-
|
|
9
|
+
var CORE_SCHEMA_VERSION = "2026-07-28.1";
|
|
10
10
|
var CORE_SCHEMA_TABLES = [
|
|
11
|
+
"admin_credit_adjustment_events",
|
|
12
|
+
"admin_credit_adjustments",
|
|
11
13
|
"billing_authorizations",
|
|
14
|
+
"billing_credits",
|
|
12
15
|
"billing_debits",
|
|
13
16
|
"billing_debit_allocations",
|
|
14
17
|
"billing_events",
|
|
@@ -42,7 +45,10 @@ var CORE_SCHEMA_TABLES = [
|
|
|
42
45
|
"workflow_webhook_deliveries"
|
|
43
46
|
];
|
|
44
47
|
var CORE_SCHEMA_INDEXES = [
|
|
48
|
+
"admin_credit_adjustment_events_reference",
|
|
49
|
+
"admin_credit_adjustments_user_created",
|
|
45
50
|
"billing_authorizations_user_status",
|
|
51
|
+
"billing_credits_user_status",
|
|
46
52
|
"billing_debits_user_status",
|
|
47
53
|
"billing_events_authorization",
|
|
48
54
|
"billing_events_user_created_at",
|
|
@@ -481,6 +487,50 @@ async function migrate() {
|
|
|
481
487
|
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
482
488
|
)
|
|
483
489
|
`);
|
|
490
|
+
await db.execute(`
|
|
491
|
+
CREATE TABLE IF NOT EXISTS billing_credits (
|
|
492
|
+
idempotency_key TEXT PRIMARY KEY,
|
|
493
|
+
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
494
|
+
amount_mc INTEGER NOT NULL,
|
|
495
|
+
operation TEXT NOT NULL,
|
|
496
|
+
description TEXT,
|
|
497
|
+
status TEXT NOT NULL DEFAULT 'claimed',
|
|
498
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
499
|
+
applied_at TEXT
|
|
500
|
+
)
|
|
501
|
+
`);
|
|
502
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS billing_credits_user_status ON billing_credits(user_id, status)`);
|
|
503
|
+
await db.execute(`
|
|
504
|
+
CREATE TABLE IF NOT EXISTS admin_credit_adjustments (
|
|
505
|
+
reference TEXT PRIMARY KEY,
|
|
506
|
+
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
507
|
+
direction TEXT NOT NULL,
|
|
508
|
+
amount_mc INTEGER NOT NULL,
|
|
509
|
+
reason TEXT NOT NULL,
|
|
510
|
+
actor TEXT NOT NULL,
|
|
511
|
+
status TEXT NOT NULL DEFAULT 'applied',
|
|
512
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
513
|
+
updated_at TEXT
|
|
514
|
+
)
|
|
515
|
+
`);
|
|
516
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS admin_credit_adjustments_user_created ON admin_credit_adjustments(user_id, created_at)`);
|
|
517
|
+
await db.execute(`
|
|
518
|
+
CREATE TABLE IF NOT EXISTS admin_credit_adjustment_events (
|
|
519
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
520
|
+
reference TEXT NOT NULL REFERENCES admin_credit_adjustments(reference),
|
|
521
|
+
seq INTEGER NOT NULL,
|
|
522
|
+
kind TEXT NOT NULL,
|
|
523
|
+
from_amount_mc INTEGER,
|
|
524
|
+
to_amount_mc INTEGER,
|
|
525
|
+
from_reason TEXT,
|
|
526
|
+
to_reason TEXT,
|
|
527
|
+
moved_mc INTEGER NOT NULL DEFAULT 0,
|
|
528
|
+
actor TEXT NOT NULL,
|
|
529
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
530
|
+
UNIQUE (reference, seq)
|
|
531
|
+
)
|
|
532
|
+
`);
|
|
533
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS admin_credit_adjustment_events_reference ON admin_credit_adjustment_events(reference)`);
|
|
484
534
|
await db.execute(`
|
|
485
535
|
CREATE TABLE IF NOT EXISTS site_audit_jobs (
|
|
486
536
|
id TEXT PRIMARY KEY,
|
|
@@ -1994,6 +2044,84 @@ async function creditMc(userId, mc, operation, description, stripePaymentIntent,
|
|
|
1994
2044
|
const res = await db.execute({ sql: "SELECT balance_mc FROM users WHERE id = ?", args: [userId] });
|
|
1995
2045
|
return Number(res.rows[0]?.balance_mc ?? 0);
|
|
1996
2046
|
}
|
|
2047
|
+
async function creditMcIdempotent(userId, mc, operation, description, idempotencyKey, options) {
|
|
2048
|
+
if (!Number.isSafeInteger(mc) || mc <= 0) throw new Error("idempotent credit amount must be a positive integer");
|
|
2049
|
+
if (!idempotencyKey || idempotencyKey.length > 500) throw new Error("idempotent credit key is required and must be bounded");
|
|
2050
|
+
const db = getDb();
|
|
2051
|
+
await ensureMigrated(userId);
|
|
2052
|
+
let neverExpires = options?.neverExpires;
|
|
2053
|
+
if (neverExpires === void 0) {
|
|
2054
|
+
const tier = await db.execute({ sql: "SELECT subscription_tier FROM users WHERE id = ? LIMIT 1", args: [userId] });
|
|
2055
|
+
neverExpires = String(tier.rows[0]?.subscription_tier ?? "").startsWith("sonic");
|
|
2056
|
+
}
|
|
2057
|
+
const expiry = neverExpires ? "9999-12-31 23:59:59" : null;
|
|
2058
|
+
const readBalance = async () => {
|
|
2059
|
+
const res = await db.execute({ sql: "SELECT balance_mc FROM users WHERE id = ?", args: [userId] });
|
|
2060
|
+
return Number(res.rows[0]?.balance_mc ?? 0);
|
|
2061
|
+
};
|
|
2062
|
+
let results;
|
|
2063
|
+
try {
|
|
2064
|
+
results = await db.batch([
|
|
2065
|
+
{
|
|
2066
|
+
sql: "INSERT OR IGNORE INTO billing_credits (idempotency_key, user_id, amount_mc, operation, description, status) VALUES (?, ?, ?, ?, ?, 'claimed')",
|
|
2067
|
+
args: [idempotencyKey, userId, mc, operation, description]
|
|
2068
|
+
},
|
|
2069
|
+
{
|
|
2070
|
+
sql: "UPDATE billing_credits SET status = 'applying' WHERE idempotency_key = ? AND user_id = ? AND amount_mc = ? AND operation = ? AND description = ? AND status = 'claimed'",
|
|
2071
|
+
args: [idempotencyKey, userId, mc, operation, description]
|
|
2072
|
+
},
|
|
2073
|
+
{
|
|
2074
|
+
sql: `INSERT INTO credit_lots (user_id, amount_mc, remaining_mc, source, stripe_pi, expires_at, never_expires)
|
|
2075
|
+
SELECT user_id, amount_mc, amount_mc, operation, NULL, COALESCE(?, datetime('now', ?)), ?
|
|
2076
|
+
FROM billing_credits
|
|
2077
|
+
WHERE idempotency_key = ? AND status = 'applying'`,
|
|
2078
|
+
args: [expiry, CREDIT_LOT_TTL, neverExpires ? 1 : 0, idempotencyKey]
|
|
2079
|
+
},
|
|
2080
|
+
{
|
|
2081
|
+
sql: `UPDATE users
|
|
2082
|
+
SET balance_mc = balance_mc + COALESCE((
|
|
2083
|
+
SELECT amount_mc FROM billing_credits WHERE idempotency_key = ? AND status = 'applying'
|
|
2084
|
+
), 0)
|
|
2085
|
+
WHERE id = ?`,
|
|
2086
|
+
args: [idempotencyKey, userId]
|
|
2087
|
+
},
|
|
2088
|
+
{
|
|
2089
|
+
sql: "INSERT INTO ledger (user_id, amount_mc, operation, description) SELECT user_id, amount_mc, operation, description FROM billing_credits WHERE idempotency_key = ? AND status = 'applying'",
|
|
2090
|
+
args: [idempotencyKey]
|
|
2091
|
+
},
|
|
2092
|
+
{
|
|
2093
|
+
sql: "UPDATE billing_credits SET status = 'applied', applied_at = datetime('now') WHERE idempotency_key = ? AND status = 'applying'",
|
|
2094
|
+
args: [idempotencyKey]
|
|
2095
|
+
}
|
|
2096
|
+
], "write");
|
|
2097
|
+
} catch (error) {
|
|
2098
|
+
try {
|
|
2099
|
+
const recovered = await db.execute({
|
|
2100
|
+
sql: "SELECT user_id, amount_mc, operation, description FROM billing_credits WHERE idempotency_key = ? AND status = 'applied' LIMIT 1",
|
|
2101
|
+
args: [idempotencyKey]
|
|
2102
|
+
});
|
|
2103
|
+
const row2 = recovered.rows[0];
|
|
2104
|
+
if (row2 && Number(row2.user_id) === userId && Number(row2.amount_mc) === mc && String(row2.operation) === operation && String(row2.description) === description) {
|
|
2105
|
+
return { ok: true, balance_mc: await readBalance(), duplicate: true };
|
|
2106
|
+
}
|
|
2107
|
+
} catch {
|
|
2108
|
+
}
|
|
2109
|
+
throw error;
|
|
2110
|
+
}
|
|
2111
|
+
const claim = await db.execute({
|
|
2112
|
+
sql: "SELECT user_id, amount_mc, operation, description, status FROM billing_credits WHERE idempotency_key = ? LIMIT 1",
|
|
2113
|
+
args: [idempotencyKey]
|
|
2114
|
+
});
|
|
2115
|
+
const row = claim.rows[0];
|
|
2116
|
+
if (!row || Number(row.user_id) !== userId || Number(row.amount_mc) !== mc || String(row.operation) !== operation || String(row.description) !== description) {
|
|
2117
|
+
throw new Error("idempotent credit key conflicts with another credit");
|
|
2118
|
+
}
|
|
2119
|
+
return {
|
|
2120
|
+
ok: String(row.status) === "applied",
|
|
2121
|
+
balance_mc: await readBalance(),
|
|
2122
|
+
duplicate: results[1]?.rowsAffected === 0 && String(row.status) === "applied"
|
|
2123
|
+
};
|
|
2124
|
+
}
|
|
1997
2125
|
async function ledgerExistsForOperation(userId, operation) {
|
|
1998
2126
|
const res = await getDb().execute({
|
|
1999
2127
|
sql: "SELECT 1 FROM ledger WHERE user_id = ? AND operation = ? LIMIT 1",
|
|
@@ -2550,6 +2678,7 @@ async function listActiveUserIds() {
|
|
|
2550
2678
|
}
|
|
2551
2679
|
|
|
2552
2680
|
export {
|
|
2681
|
+
CORE_SCHEMA_VERSION,
|
|
2553
2682
|
getDb,
|
|
2554
2683
|
setDbForTesting,
|
|
2555
2684
|
migrate,
|
|
@@ -2645,6 +2774,7 @@ export {
|
|
|
2645
2774
|
expireOldLots,
|
|
2646
2775
|
migrateExistingBalancesToLots,
|
|
2647
2776
|
creditMc,
|
|
2777
|
+
creditMcIdempotent,
|
|
2648
2778
|
ledgerExistsForOperation,
|
|
2649
2779
|
claimMonthlyFreeRefresh,
|
|
2650
2780
|
debitMc,
|
|
@@ -2663,4 +2793,4 @@ export {
|
|
|
2663
2793
|
insertInboxMessage,
|
|
2664
2794
|
listActiveUserIds
|
|
2665
2795
|
};
|
|
2666
|
-
//# sourceMappingURL=chunk-
|
|
2796
|
+
//# sourceMappingURL=chunk-CSCD2HNS.js.map
|