mcp-scraper 0.33.6 → 0.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/bin/api-server.cjs +13558 -12516
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-scraper-cli.cjs +13 -2
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +14 -4
- package/dist/bin/mcp-scraper-cli.js.map +1 -1
- package/dist/bin/mcp-scraper-install.cjs +2 -2
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +117 -7
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +7 -7
- package/dist/bin/paa-harvest.cjs +128 -13
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +4 -4
- package/dist/{chunk-DUS4ENOY.js → chunk-2EDFOQD7.js} +2 -2
- package/dist/{chunk-DUS4ENOY.js.map → chunk-2EDFOQD7.js.map} +1 -1
- package/dist/{chunk-C2OD4ELT.js → chunk-3HBPKR5G.js} +2 -2
- package/dist/{chunk-A54ELKJK.js → chunk-4KQVIYIH.js} +139 -17
- package/dist/chunk-4KQVIYIH.js.map +1 -0
- package/dist/{chunk-AIK7WQD5.js → chunk-62DQAWPF.js} +141 -3
- package/dist/chunk-62DQAWPF.js.map +1 -0
- package/dist/{chunk-V73MPRU6.js → chunk-CB5C3BPB.js} +17 -1
- package/dist/chunk-CB5C3BPB.js.map +1 -0
- package/dist/{chunk-6ZUK4H66.js → chunk-J32XSMJI.js} +117 -8
- package/dist/chunk-J32XSMJI.js.map +1 -0
- package/dist/{chunk-LKIP7VDN.js → chunk-JBSGBSGT.js} +46 -3
- package/dist/chunk-JBSGBSGT.js.map +1 -0
- package/dist/chunk-NFMIHEYE.js +7 -0
- package/dist/chunk-NFMIHEYE.js.map +1 -0
- package/dist/{chunk-XL6YIWJ3.js → chunk-SUOHUXQS.js} +3 -3
- package/dist/{chunk-WJJTGRK4.js → chunk-V36LS5YV.js} +2 -2
- package/dist/{chunk-AT2PFWHW.js → chunk-ZID3WQID.js} +2 -2
- package/dist/{db-5SZQUI7A.js → db-YAI5AQOI.js} +16 -2
- package/dist/{extract-bundle-YNXTFCC3.js → extract-bundle-GPZRLBVY.js} +2 -2
- package/dist/index.cjs +128 -13
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +4 -1
- package/dist/index.d.ts +4 -1
- package/dist/index.js +4 -4
- package/dist/{server-SHOYO52X.js → server-LKVPEQFE.js} +1386 -778
- package/dist/server-LKVPEQFE.js.map +1 -0
- package/dist/{site-extract-repository-ZDTO7ODZ.js → site-extract-repository-JHMVHENZ.js} +4 -4
- package/dist/{worker-BJ64GCDE.js → worker-ZZHSYI3Y.js} +7 -7
- package/docs/mcp-tool-manifest.generated.json +213 -3
- package/package.json +1 -1
- package/dist/chunk-6ZUK4H66.js.map +0 -1
- package/dist/chunk-A54ELKJK.js.map +0 -1
- package/dist/chunk-AIK7WQD5.js.map +0 -1
- package/dist/chunk-F65IIUHI.js +0 -7
- package/dist/chunk-F65IIUHI.js.map +0 -1
- package/dist/chunk-LKIP7VDN.js.map +0 -1
- package/dist/chunk-V73MPRU6.js.map +0 -1
- package/dist/server-SHOYO52X.js.map +0 -1
- /package/dist/{chunk-C2OD4ELT.js.map → chunk-3HBPKR5G.js.map} +0 -0
- /package/dist/{chunk-XL6YIWJ3.js.map → chunk-SUOHUXQS.js.map} +0 -0
- /package/dist/{chunk-WJJTGRK4.js.map → chunk-V36LS5YV.js.map} +0 -0
- /package/dist/{chunk-AT2PFWHW.js.map → chunk-ZID3WQID.js.map} +0 -0
- /package/dist/{db-5SZQUI7A.js.map → db-YAI5AQOI.js.map} +0 -0
- /package/dist/{extract-bundle-YNXTFCC3.js.map → extract-bundle-GPZRLBVY.js.map} +0 -0
- /package/dist/{site-extract-repository-ZDTO7ODZ.js.map → site-extract-repository-JHMVHENZ.js.map} +0 -0
- /package/dist/{worker-BJ64GCDE.js.map → worker-ZZHSYI3Y.js.map} +0 -0
|
@@ -6,16 +6,21 @@ import {
|
|
|
6
6
|
createHarvestAttemptRecorder,
|
|
7
7
|
harvestProblemResponse,
|
|
8
8
|
serializeHarvestProblem
|
|
9
|
-
} from "./chunk-
|
|
9
|
+
} from "./chunk-3HBPKR5G.js";
|
|
10
10
|
import {
|
|
11
|
+
BACKUP_LAST_RESORT_ATTEMPTS,
|
|
11
12
|
BrowserDriver,
|
|
12
13
|
MapsSelectors,
|
|
14
|
+
backupProxyAvailable,
|
|
13
15
|
buildYouTubeChannelVideosUrl,
|
|
16
|
+
cleanupBackupProxyId,
|
|
17
|
+
createBackupProxyIdSafe,
|
|
14
18
|
deleteKernelProxyId,
|
|
15
19
|
harvest,
|
|
20
|
+
resolveBackupKernelProxyId,
|
|
16
21
|
resolveKernelProxyId,
|
|
17
22
|
serpArrivalUrl
|
|
18
|
-
} from "./chunk-
|
|
23
|
+
} from "./chunk-4KQVIYIH.js";
|
|
19
24
|
import {
|
|
20
25
|
HttpMcpToolExecutor,
|
|
21
26
|
assessTranscriptSignal,
|
|
@@ -35,7 +40,7 @@ import {
|
|
|
35
40
|
sanitizeAttempts,
|
|
36
41
|
sanitizeHarvestResult,
|
|
37
42
|
transcribeMediaUrl
|
|
38
|
-
} from "./chunk-
|
|
43
|
+
} from "./chunk-J32XSMJI.js";
|
|
39
44
|
import {
|
|
40
45
|
auditImages,
|
|
41
46
|
buildLinkReport,
|
|
@@ -52,7 +57,7 @@ import {
|
|
|
52
57
|
recordKernelSession,
|
|
53
58
|
recordVendorUsage,
|
|
54
59
|
runWithCostContext
|
|
55
|
-
} from "./chunk-
|
|
60
|
+
} from "./chunk-V36LS5YV.js";
|
|
56
61
|
import {
|
|
57
62
|
csvRecords,
|
|
58
63
|
listWorkflowDefinitions,
|
|
@@ -64,7 +69,7 @@ import {
|
|
|
64
69
|
workflowOutputBaseDir,
|
|
65
70
|
workflowStepCount,
|
|
66
71
|
workflowSupportsSteps
|
|
67
|
-
} from "./chunk-
|
|
72
|
+
} from "./chunk-ZID3WQID.js";
|
|
68
73
|
import "./chunk-K443GQY5.js";
|
|
69
74
|
import {
|
|
70
75
|
DEFAULT_MAPS_PROXY_MODE,
|
|
@@ -75,8 +80,8 @@ import {
|
|
|
75
80
|
RawMapsHoursRowSchema,
|
|
76
81
|
RawMapsOverviewSchema,
|
|
77
82
|
RawMapsReviewStatsSchema
|
|
78
|
-
} from "./chunk-
|
|
79
|
-
import "./chunk-
|
|
83
|
+
} from "./chunk-CB5C3BPB.js";
|
|
84
|
+
import "./chunk-NFMIHEYE.js";
|
|
80
85
|
import {
|
|
81
86
|
completeExtractJob,
|
|
82
87
|
countSuccessfulPages,
|
|
@@ -87,7 +92,7 @@ import {
|
|
|
87
92
|
saveExtractPages,
|
|
88
93
|
setExtractJobTotal,
|
|
89
94
|
settleExtractJob
|
|
90
|
-
} from "./chunk-
|
|
95
|
+
} from "./chunk-SUOHUXQS.js";
|
|
91
96
|
import {
|
|
92
97
|
BROWSER_OPEN_MIN_BALANCE_MC,
|
|
93
98
|
CONCURRENCY_PRICE_ID,
|
|
@@ -122,12 +127,17 @@ import {
|
|
|
122
127
|
browserActiveCostMc,
|
|
123
128
|
calculateConnectedUsageCharge,
|
|
124
129
|
concurrencySlotBillingInfo,
|
|
130
|
+
getPrivateOfferPayload,
|
|
125
131
|
getPublicRatesPayload,
|
|
126
132
|
insufficientBalanceResponse,
|
|
127
133
|
mcToCredits,
|
|
134
|
+
offerClaimPathForTier,
|
|
128
135
|
serpActualCostMc,
|
|
136
|
+
tierAmountChargedUsd,
|
|
137
|
+
tierBillingInterval,
|
|
138
|
+
tierCreditsPerInterval,
|
|
129
139
|
vendorCostUsdToMc
|
|
130
|
-
} from "./chunk-
|
|
140
|
+
} from "./chunk-JBSGBSGT.js";
|
|
131
141
|
import {
|
|
132
142
|
CaptchaError,
|
|
133
143
|
ExtractionError,
|
|
@@ -159,8 +169,10 @@ import {
|
|
|
159
169
|
expireOldLots,
|
|
160
170
|
failJob,
|
|
161
171
|
failWorkflowRunRecord,
|
|
172
|
+
getBillingLedgerSummary,
|
|
162
173
|
getClient,
|
|
163
174
|
getDb,
|
|
175
|
+
getInboxMessage,
|
|
164
176
|
getJob,
|
|
165
177
|
getLedger,
|
|
166
178
|
getMemoryPlan,
|
|
@@ -177,10 +189,13 @@ import {
|
|
|
177
189
|
getWorkflowRun,
|
|
178
190
|
getWorkflowRunStepState,
|
|
179
191
|
getWorkflowSchedule,
|
|
192
|
+
insertInboxMessage,
|
|
180
193
|
ledgerExistsForOperation,
|
|
181
194
|
ledgerExistsForStripePI,
|
|
195
|
+
listActiveUserIds,
|
|
182
196
|
listDueWorkflowSchedules,
|
|
183
197
|
listHarvestAttempts,
|
|
198
|
+
listInboxMessages,
|
|
184
199
|
listJobs,
|
|
185
200
|
listRequestEvents,
|
|
186
201
|
listUsers,
|
|
@@ -188,6 +203,7 @@ import {
|
|
|
188
203
|
listWorkflowRuns,
|
|
189
204
|
listWorkflowSchedules,
|
|
190
205
|
logRequestEvent,
|
|
206
|
+
markInboxMessageOpened,
|
|
191
207
|
markWorkflowRunRunning,
|
|
192
208
|
markWorkflowScheduleRan,
|
|
193
209
|
patchWorkflowSchedule,
|
|
@@ -211,9 +227,10 @@ import {
|
|
|
211
227
|
setSubscriptionTier,
|
|
212
228
|
setUserMemoryCreds,
|
|
213
229
|
settleDebitMcIdempotent,
|
|
230
|
+
softDeleteInboxMessage,
|
|
214
231
|
upsertPageSnapshot,
|
|
215
232
|
verifyPassword
|
|
216
|
-
} from "./chunk-
|
|
233
|
+
} from "./chunk-62DQAWPF.js";
|
|
217
234
|
|
|
218
235
|
// src/blog/registry.ts
|
|
219
236
|
var posts = [
|
|
@@ -5065,7 +5082,7 @@ async function extractSite(opts) {
|
|
|
5065
5082
|
}
|
|
5066
5083
|
|
|
5067
5084
|
// src/api/server.ts
|
|
5068
|
-
import { Hono as
|
|
5085
|
+
import { Hono as Hono26 } from "hono";
|
|
5069
5086
|
import { serve as serveInngest } from "inngest/hono";
|
|
5070
5087
|
|
|
5071
5088
|
// src/inngest/client.ts
|
|
@@ -7371,7 +7388,7 @@ var siteExtractFn = inngest.createFunction(
|
|
|
7371
7388
|
return buildSeoAudit(pages, job.startUrl);
|
|
7372
7389
|
}) : null;
|
|
7373
7390
|
const artifacts = await step.run("finalize", async () => {
|
|
7374
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
7391
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-GPZRLBVY.js");
|
|
7375
7392
|
const stored = await assembleExtractArtifacts(job, { branding, imageAudit, seoAudit });
|
|
7376
7393
|
await completeExtractJob(jobId, stored);
|
|
7377
7394
|
return stored;
|
|
@@ -8606,8 +8623,14 @@ async function provisionMemoryForUser(user) {
|
|
|
8606
8623
|
} catch {
|
|
8607
8624
|
}
|
|
8608
8625
|
}
|
|
8626
|
+
function tierIncludesMemory(subscriptionTier) {
|
|
8627
|
+
if (!subscriptionTier) return false;
|
|
8628
|
+
return !!SUBSCRIPTION_TIER_BY_KEY[subscriptionTier]?.includes_memory;
|
|
8629
|
+
}
|
|
8609
8630
|
function resolveEffectiveMemoryPlan(user) {
|
|
8610
|
-
|
|
8631
|
+
if (user.memory_plan === "team") return "team";
|
|
8632
|
+
if (tierIncludesMemory(user.subscription_tier)) return "pro";
|
|
8633
|
+
return user.memory_plan === "pro" ? "pro" : "free";
|
|
8611
8634
|
}
|
|
8612
8635
|
async function getOrCreateUserMemoryKey(user) {
|
|
8613
8636
|
const creds = await getUserMemoryCreds(user.id);
|
|
@@ -8808,37 +8831,120 @@ internalBillingApp.post("/scheduled-run/void", async (c) => {
|
|
|
8808
8831
|
}
|
|
8809
8832
|
});
|
|
8810
8833
|
|
|
8834
|
+
// src/api/inbox-routes.ts
|
|
8835
|
+
import { Hono as Hono4 } from "hono";
|
|
8836
|
+
import { z as z11 } from "zod";
|
|
8837
|
+
|
|
8838
|
+
// src/api/api-auth.ts
|
|
8839
|
+
import { createMiddleware as createMiddleware2 } from "hono/factory";
|
|
8840
|
+
function createApiKeyAuth() {
|
|
8841
|
+
return createMiddleware2(async (c, next) => {
|
|
8842
|
+
const key = c.req.header("x-api-key");
|
|
8843
|
+
if (!key) return c.json({ error: "Missing API key" }, 401);
|
|
8844
|
+
const user = await getUserByApiKey(key);
|
|
8845
|
+
if (!user) return c.json({ error: "Invalid or inactive API key" }, 401);
|
|
8846
|
+
c.set("user", user);
|
|
8847
|
+
return next();
|
|
8848
|
+
});
|
|
8849
|
+
}
|
|
8850
|
+
|
|
8851
|
+
// src/api/inbox-routes.ts
|
|
8852
|
+
var inboxApp = new Hono4();
|
|
8853
|
+
inboxApp.use("*", createApiKeyAuth());
|
|
8854
|
+
inboxApp.get("/", async (c) => {
|
|
8855
|
+
const messages = await listInboxMessages(c.get("user").id);
|
|
8856
|
+
return c.json({ ok: true, messages });
|
|
8857
|
+
});
|
|
8858
|
+
inboxApp.get("/:id", async (c) => {
|
|
8859
|
+
const message = await getInboxMessage(c.get("user").id, c.req.param("id"));
|
|
8860
|
+
if (!message) return c.json({ ok: false, error: "Message not found" }, 404);
|
|
8861
|
+
return c.json({ ok: true, message });
|
|
8862
|
+
});
|
|
8863
|
+
inboxApp.post("/:id/opened", async (c) => {
|
|
8864
|
+
const openedAt = await markInboxMessageOpened(c.get("user").id, c.req.param("id"));
|
|
8865
|
+
if (openedAt === null) return c.json({ ok: false, error: "Message not found" }, 404);
|
|
8866
|
+
return c.json({ ok: true, opened_at: openedAt });
|
|
8867
|
+
});
|
|
8868
|
+
inboxApp.delete("/:id", async (c) => {
|
|
8869
|
+
const deleted = await softDeleteInboxMessage(c.get("user").id, c.req.param("id"));
|
|
8870
|
+
if (!deleted) return c.json({ ok: false, error: "Message not found" }, 404);
|
|
8871
|
+
return c.json({ ok: true });
|
|
8872
|
+
});
|
|
8873
|
+
var broadcastSchema = z11.object({
|
|
8874
|
+
subject: z11.string().min(1).max(500),
|
|
8875
|
+
body_md: z11.string().min(1).max(2e5),
|
|
8876
|
+
recipients: z11.union([z11.literal("all"), z11.array(z11.string().min(1).max(320)).min(1).max(1e3)])
|
|
8877
|
+
});
|
|
8878
|
+
function internalRequestAuthorized2(header) {
|
|
8879
|
+
const expected = process.env.INTERNAL_BILLING_SECRET ?? process.env.INTERNAL_MEMORY_DEBIT_SECRET;
|
|
8880
|
+
return Boolean(expected && header && header === expected);
|
|
8881
|
+
}
|
|
8882
|
+
var internalInboxApp = new Hono4();
|
|
8883
|
+
internalInboxApp.use("*", async (c, next) => {
|
|
8884
|
+
if (!internalRequestAuthorized2(c.req.header("x-internal-secret"))) {
|
|
8885
|
+
return c.json({ ok: false, error_code: "unauthorized", error: "unauthorized" }, 401);
|
|
8886
|
+
}
|
|
8887
|
+
await next();
|
|
8888
|
+
});
|
|
8889
|
+
internalInboxApp.post("/broadcast", async (c) => {
|
|
8890
|
+
const parsed = broadcastSchema.safeParse(await c.req.json().catch(() => null));
|
|
8891
|
+
if (!parsed.success) {
|
|
8892
|
+
return c.json({ ok: false, error_code: "invalid_request", error: "invalid inbox broadcast request" }, 400);
|
|
8893
|
+
}
|
|
8894
|
+
const { subject, body_md, recipients } = parsed.data;
|
|
8895
|
+
const skipped = [];
|
|
8896
|
+
const userIds = [];
|
|
8897
|
+
if (recipients === "all") {
|
|
8898
|
+
userIds.push(...await listActiveUserIds());
|
|
8899
|
+
} else {
|
|
8900
|
+
const seen = /* @__PURE__ */ new Set();
|
|
8901
|
+
for (const email of recipients) {
|
|
8902
|
+
const user = await getUserByEmail(email.trim().toLowerCase());
|
|
8903
|
+
if (!user || seen.has(user.id)) {
|
|
8904
|
+
skipped.push(email);
|
|
8905
|
+
continue;
|
|
8906
|
+
}
|
|
8907
|
+
seen.add(user.id);
|
|
8908
|
+
userIds.push(user.id);
|
|
8909
|
+
}
|
|
8910
|
+
}
|
|
8911
|
+
for (const userId of userIds) {
|
|
8912
|
+
await insertInboxMessage(userId, subject, body_md, "operator");
|
|
8913
|
+
}
|
|
8914
|
+
return c.json({ ok: true, delivered: userIds.length, skipped });
|
|
8915
|
+
});
|
|
8916
|
+
|
|
8811
8917
|
// src/api/connected-usage-billing.ts
|
|
8812
8918
|
import { createHash as createHash3, randomUUID as randomUUID2 } from "crypto";
|
|
8813
|
-
import { z as
|
|
8919
|
+
import { z as z12 } from "zod";
|
|
8814
8920
|
var CONNECTED_USAGE_BILLING_CLASS = "connected_usage";
|
|
8815
8921
|
var CONNECTED_USAGE_DEFAULT_SOURCE_SURFACE = "mastra_scheduler";
|
|
8816
|
-
var ConnectedUsageSafeMetadataSchema =
|
|
8817
|
-
toolName:
|
|
8818
|
-
operationKind:
|
|
8819
|
-
outcome:
|
|
8820
|
-
requestId:
|
|
8821
|
-
occurrenceId:
|
|
8822
|
-
scheduleId:
|
|
8823
|
-
nangoOperationId:
|
|
8824
|
-
startedAt:
|
|
8825
|
-
completedAt:
|
|
8922
|
+
var ConnectedUsageSafeMetadataSchema = z12.object({
|
|
8923
|
+
toolName: z12.string().trim().min(1).max(200).optional(),
|
|
8924
|
+
operationKind: z12.enum(["read", "action", "sync", "export", "proxy", "other"]).optional(),
|
|
8925
|
+
outcome: z12.enum(["success", "error", "partial"]).optional(),
|
|
8926
|
+
requestId: z12.string().trim().min(1).max(200).optional(),
|
|
8927
|
+
occurrenceId: z12.string().trim().min(1).max(200).optional(),
|
|
8928
|
+
scheduleId: z12.string().trim().min(1).max(200).optional(),
|
|
8929
|
+
nangoOperationId: z12.string().trim().min(1).max(200).optional(),
|
|
8930
|
+
startedAt: z12.string().datetime({ offset: true }).optional(),
|
|
8931
|
+
completedAt: z12.string().datetime({ offset: true }).optional()
|
|
8826
8932
|
}).strict();
|
|
8827
|
-
var ConnectedUsageSettlementInputSchema =
|
|
8828
|
-
identity:
|
|
8829
|
-
idempotencyKey:
|
|
8830
|
-
ratePolicyVersion:
|
|
8831
|
-
connectionId:
|
|
8832
|
-
providerConfigKey:
|
|
8833
|
-
functionRuns:
|
|
8834
|
-
proxyRequests:
|
|
8835
|
-
computeMilliseconds:
|
|
8836
|
-
sourceSurface:
|
|
8933
|
+
var ConnectedUsageSettlementInputSchema = z12.object({
|
|
8934
|
+
identity: z12.string().trim().min(1).max(320),
|
|
8935
|
+
idempotencyKey: z12.string().trim().min(1).max(500),
|
|
8936
|
+
ratePolicyVersion: z12.string().trim().min(1).max(100),
|
|
8937
|
+
connectionId: z12.string().trim().min(1).max(500),
|
|
8938
|
+
providerConfigKey: z12.string().trim().min(1).max(200),
|
|
8939
|
+
functionRuns: z12.number().int().safe().nonnegative(),
|
|
8940
|
+
proxyRequests: z12.number().int().safe().nonnegative(),
|
|
8941
|
+
computeMilliseconds: z12.number().int().safe().nonnegative(),
|
|
8942
|
+
sourceSurface: z12.enum(["main_mcp", "mastra_scheduler", "nango_reconciliation"]).default(CONNECTED_USAGE_DEFAULT_SOURCE_SURFACE),
|
|
8837
8943
|
metadata: ConnectedUsageSafeMetadataSchema.optional()
|
|
8838
8944
|
}).strict();
|
|
8839
|
-
var ConnectedUsagePreflightInputSchema =
|
|
8840
|
-
identity:
|
|
8841
|
-
ratePolicyVersion:
|
|
8945
|
+
var ConnectedUsagePreflightInputSchema = z12.object({
|
|
8946
|
+
identity: z12.string().trim().min(1).max(320),
|
|
8947
|
+
ratePolicyVersion: z12.string().trim().min(1).max(100)
|
|
8842
8948
|
}).strict();
|
|
8843
8949
|
var ConnectedUsageBillingError = class extends Error {
|
|
8844
8950
|
constructor(code, message, httpStatus, details = {}) {
|
|
@@ -9214,8 +9320,249 @@ async function listConnectedUsageHistory(userId, limit = 100) {
|
|
|
9214
9320
|
return history;
|
|
9215
9321
|
}
|
|
9216
9322
|
|
|
9323
|
+
// src/api/connected-account-billing.ts
|
|
9324
|
+
var DEFAULT_CONNECTED_ACCOUNT_PRICE_ID = "price_1TtYPAS8aAcsk3TGHOgrDZiQ";
|
|
9325
|
+
async function ensureConnectedAccountBillingSchema() {
|
|
9326
|
+
await getDb().execute(`CREATE TABLE IF NOT EXISTS connected_account_billing (
|
|
9327
|
+
user_id INTEGER PRIMARY KEY,
|
|
9328
|
+
stripe_subscription_id TEXT,
|
|
9329
|
+
stripe_subscription_item_id TEXT,
|
|
9330
|
+
price_id TEXT,
|
|
9331
|
+
quantity INTEGER NOT NULL DEFAULT 0,
|
|
9332
|
+
status TEXT NOT NULL DEFAULT 'pending',
|
|
9333
|
+
last_error_code TEXT,
|
|
9334
|
+
synced_at TEXT,
|
|
9335
|
+
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
9336
|
+
)`);
|
|
9337
|
+
}
|
|
9338
|
+
async function getConnectedAccountBillingState(userId) {
|
|
9339
|
+
await ensureConnectedAccountBillingSchema();
|
|
9340
|
+
const result = await getDb().execute({
|
|
9341
|
+
sql: "SELECT * FROM connected_account_billing WHERE user_id = ? LIMIT 1",
|
|
9342
|
+
args: [Number(userId)]
|
|
9343
|
+
});
|
|
9344
|
+
return result.rows[0] ? result.rows[0] : null;
|
|
9345
|
+
}
|
|
9346
|
+
async function setConnectedAccountBillingState(input) {
|
|
9347
|
+
await ensureConnectedAccountBillingSchema();
|
|
9348
|
+
await getDb().execute({
|
|
9349
|
+
sql: `INSERT INTO connected_account_billing
|
|
9350
|
+
(user_id, stripe_subscription_id, stripe_subscription_item_id, price_id, quantity, status, last_error_code, synced_at, updated_at)
|
|
9351
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, CASE WHEN ? THEN datetime('now') ELSE NULL END, datetime('now'))
|
|
9352
|
+
ON CONFLICT(user_id) DO UPDATE SET
|
|
9353
|
+
stripe_subscription_id = excluded.stripe_subscription_id,
|
|
9354
|
+
stripe_subscription_item_id = excluded.stripe_subscription_item_id,
|
|
9355
|
+
price_id = excluded.price_id,
|
|
9356
|
+
quantity = excluded.quantity,
|
|
9357
|
+
status = excluded.status,
|
|
9358
|
+
last_error_code = excluded.last_error_code,
|
|
9359
|
+
synced_at = CASE WHEN ? THEN datetime('now') ELSE connected_account_billing.synced_at END,
|
|
9360
|
+
updated_at = datetime('now')`,
|
|
9361
|
+
args: [
|
|
9362
|
+
Number(input.userId),
|
|
9363
|
+
input.stripeSubscriptionId,
|
|
9364
|
+
input.stripeSubscriptionItemId,
|
|
9365
|
+
input.priceId,
|
|
9366
|
+
input.quantity,
|
|
9367
|
+
input.status,
|
|
9368
|
+
input.lastErrorCode,
|
|
9369
|
+
input.synced ? 1 : 0,
|
|
9370
|
+
input.synced ? 1 : 0
|
|
9371
|
+
]
|
|
9372
|
+
});
|
|
9373
|
+
const state = await getConnectedAccountBillingState(input.userId);
|
|
9374
|
+
if (!state) throw new Error("connected account billing state was not persisted");
|
|
9375
|
+
return state;
|
|
9376
|
+
}
|
|
9377
|
+
var ConnectedAccountBillingError = class extends Error {
|
|
9378
|
+
constructor(message, code, status = 409, details) {
|
|
9379
|
+
super(message);
|
|
9380
|
+
this.code = code;
|
|
9381
|
+
this.status = status;
|
|
9382
|
+
this.details = details;
|
|
9383
|
+
this.name = "ConnectedAccountBillingError";
|
|
9384
|
+
}
|
|
9385
|
+
code;
|
|
9386
|
+
status;
|
|
9387
|
+
details;
|
|
9388
|
+
};
|
|
9389
|
+
function connectedAccountPriceId() {
|
|
9390
|
+
const configured = process.env.CONNECTED_ACCOUNT_PRICE_ID?.trim();
|
|
9391
|
+
if (configured) return configured;
|
|
9392
|
+
return DEFAULT_CONNECTED_ACCOUNT_PRICE_ID || null;
|
|
9393
|
+
}
|
|
9394
|
+
function connectedAccountBillingView(actualQuantity, state) {
|
|
9395
|
+
const quantity = Math.max(0, Math.round(actualQuantity));
|
|
9396
|
+
return {
|
|
9397
|
+
provider: "nango",
|
|
9398
|
+
billingMode: "flat_recurring_credits",
|
|
9399
|
+
priceId: connectedAccountPriceId(),
|
|
9400
|
+
unitAmountCredits: CONNECTED_ACTIVE_CONNECTION_MONTHLY_CREDITS,
|
|
9401
|
+
interval: "month",
|
|
9402
|
+
quantity,
|
|
9403
|
+
projectedMonthlyCredits: quantity * CONNECTED_ACTIVE_CONNECTION_MONTHLY_CREDITS,
|
|
9404
|
+
status: state?.status ?? "pending",
|
|
9405
|
+
lastErrorCode: state?.last_error_code ?? null,
|
|
9406
|
+
syncedAt: state?.synced_at ?? null,
|
|
9407
|
+
usage: CONNECTED_USAGE_RATE_POLICY.usage
|
|
9408
|
+
};
|
|
9409
|
+
}
|
|
9410
|
+
function findBasePlanItem(subscription) {
|
|
9411
|
+
return subscription.items.data.find((item) => !!item.price?.id && item.price.id in SUBSCRIPTION_TIERS);
|
|
9412
|
+
}
|
|
9413
|
+
function safeErrorCode(error) {
|
|
9414
|
+
if (error instanceof ConnectedAccountBillingError) return error.code;
|
|
9415
|
+
return "stripe_connection_billing_failed";
|
|
9416
|
+
}
|
|
9417
|
+
async function persistFailure(user, quantityToPersist, error) {
|
|
9418
|
+
await setConnectedAccountBillingState({
|
|
9419
|
+
userId: user.id,
|
|
9420
|
+
stripeSubscriptionId: user.subscription_id,
|
|
9421
|
+
stripeSubscriptionItemId: null,
|
|
9422
|
+
priceId: connectedAccountPriceId(),
|
|
9423
|
+
quantity: quantityToPersist,
|
|
9424
|
+
status: error instanceof ConnectedAccountBillingError && error.code !== "stripe_connection_billing_failed" ? "blocked" : "error",
|
|
9425
|
+
lastErrorCode: safeErrorCode(error),
|
|
9426
|
+
synced: false
|
|
9427
|
+
});
|
|
9428
|
+
}
|
|
9429
|
+
async function reconcileConnectedAccountBilling(user, actualQuantity) {
|
|
9430
|
+
if (!Number.isSafeInteger(actualQuantity) || actualQuantity < 0) {
|
|
9431
|
+
throw new Error("actualQuantity must be a non-negative safe integer");
|
|
9432
|
+
}
|
|
9433
|
+
const priceId = connectedAccountPriceId();
|
|
9434
|
+
if (!priceId) {
|
|
9435
|
+
const error = new ConnectedAccountBillingError(
|
|
9436
|
+
"Connected-account billing is not configured.",
|
|
9437
|
+
"connected_account_price_not_configured",
|
|
9438
|
+
503
|
|
9439
|
+
);
|
|
9440
|
+
await persistFailure(user, actualQuantity, error);
|
|
9441
|
+
throw error;
|
|
9442
|
+
}
|
|
9443
|
+
if (!user.subscription_id || !(user.subscription_tier ?? "") || !Object.values(SUBSCRIPTION_TIERS).some((t) => t.tier === user.subscription_tier)) {
|
|
9444
|
+
if (actualQuantity === 0) {
|
|
9445
|
+
const state = await setConnectedAccountBillingState({
|
|
9446
|
+
userId: user.id,
|
|
9447
|
+
stripeSubscriptionId: null,
|
|
9448
|
+
stripeSubscriptionItemId: null,
|
|
9449
|
+
priceId,
|
|
9450
|
+
quantity: 0,
|
|
9451
|
+
status: "synced",
|
|
9452
|
+
lastErrorCode: null,
|
|
9453
|
+
synced: true
|
|
9454
|
+
});
|
|
9455
|
+
return connectedAccountBillingView(0, state);
|
|
9456
|
+
}
|
|
9457
|
+
const error = new ConnectedAccountBillingError(
|
|
9458
|
+
"An active paid plan is required before connected accounts can be billed.",
|
|
9459
|
+
"paid_plan_required",
|
|
9460
|
+
403
|
|
9461
|
+
);
|
|
9462
|
+
await persistFailure(user, actualQuantity, error);
|
|
9463
|
+
throw error;
|
|
9464
|
+
}
|
|
9465
|
+
const existingState = await getConnectedAccountBillingState(user.id);
|
|
9466
|
+
const currentQuantity = Math.max(0, existingState?.quantity ?? 0);
|
|
9467
|
+
try {
|
|
9468
|
+
if (actualQuantity > currentQuantity) {
|
|
9469
|
+
const newConnections = actualQuantity - currentQuantity;
|
|
9470
|
+
const feeMc = newConnections * CONNECTED_ACTIVE_CONNECTION_MONTHLY_MC;
|
|
9471
|
+
const debit = await debitMc(
|
|
9472
|
+
user.id,
|
|
9473
|
+
feeMc,
|
|
9474
|
+
LedgerOperation.CONNECTED_ACCOUNT_FEE,
|
|
9475
|
+
`${newConnections} connected account${newConnections === 1 ? "" : "s"} at ${CONNECTED_ACTIVE_CONNECTION_MONTHLY_CREDITS} credits/month`
|
|
9476
|
+
);
|
|
9477
|
+
if (!debit.ok) {
|
|
9478
|
+
const requiredMc = feeMc;
|
|
9479
|
+
const error = new ConnectedAccountBillingError(
|
|
9480
|
+
`Insufficient credits to connect. Balance: ${mcToCredits(debit.balance_mc)} credits. Connecting requires ${mcToCredits(requiredMc)} credits (${mcToCredits(requiredMc - debit.balance_mc)} short). Upgrade your plan or wait for your credits to refresh.`,
|
|
9481
|
+
"insufficient_credits",
|
|
9482
|
+
402,
|
|
9483
|
+
{
|
|
9484
|
+
balanceCredits: mcToCredits(debit.balance_mc),
|
|
9485
|
+
requiredCredits: mcToCredits(requiredMc),
|
|
9486
|
+
shortByCredits: mcToCredits(Math.max(0, requiredMc - debit.balance_mc))
|
|
9487
|
+
}
|
|
9488
|
+
);
|
|
9489
|
+
await persistFailure(user, currentQuantity, error);
|
|
9490
|
+
throw error;
|
|
9491
|
+
}
|
|
9492
|
+
}
|
|
9493
|
+
const state = await setConnectedAccountBillingState({
|
|
9494
|
+
userId: user.id,
|
|
9495
|
+
stripeSubscriptionId: user.subscription_id,
|
|
9496
|
+
stripeSubscriptionItemId: null,
|
|
9497
|
+
priceId,
|
|
9498
|
+
quantity: actualQuantity,
|
|
9499
|
+
status: "synced",
|
|
9500
|
+
lastErrorCode: null,
|
|
9501
|
+
synced: true
|
|
9502
|
+
});
|
|
9503
|
+
return connectedAccountBillingView(actualQuantity, state);
|
|
9504
|
+
} catch (error) {
|
|
9505
|
+
if (error instanceof ConnectedAccountBillingError) throw error;
|
|
9506
|
+
await persistFailure(user, currentQuantity, error);
|
|
9507
|
+
throw new ConnectedAccountBillingError(
|
|
9508
|
+
"Unable to update connected-account billing.",
|
|
9509
|
+
"stripe_connection_billing_failed",
|
|
9510
|
+
503
|
|
9511
|
+
);
|
|
9512
|
+
}
|
|
9513
|
+
}
|
|
9514
|
+
async function currentConnectedAccountBillingView(userId, actualQuantity) {
|
|
9515
|
+
return connectedAccountBillingView(actualQuantity, await getConnectedAccountBillingState(userId));
|
|
9516
|
+
}
|
|
9517
|
+
|
|
9518
|
+
// src/api/billing-summary.ts
|
|
9519
|
+
var allOperations = Object.values(LedgerOperation);
|
|
9520
|
+
var LEDGER_OPERATION_CLASSIFICATION = {
|
|
9521
|
+
refund: allOperations.filter((op) => op === LedgerOperation.REFUND || op.endsWith("_refund")),
|
|
9522
|
+
grant: [LedgerOperation.SUBSCRIPTION, LedgerOperation.SIGNUP_GRANT, LedgerOperation.MONTHLY_REFRESH],
|
|
9523
|
+
purchase: [LedgerOperation.TOPUP]
|
|
9524
|
+
};
|
|
9525
|
+
function resolveBillingSummaryWindow(raw, now = /* @__PURE__ */ new Date()) {
|
|
9526
|
+
if (raw === "30d") {
|
|
9527
|
+
return { window: "30d", since: new Date(now.getTime() - 30 * 24 * 60 * 60 * 1e3) };
|
|
9528
|
+
}
|
|
9529
|
+
return { window: "month", since: new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1)) };
|
|
9530
|
+
}
|
|
9531
|
+
function toSqliteUtc(date) {
|
|
9532
|
+
return date.toISOString().slice(0, 19).replace("T", " ");
|
|
9533
|
+
}
|
|
9534
|
+
async function billingSummaryHandler(c) {
|
|
9535
|
+
const raw = c.req.query("window");
|
|
9536
|
+
if (raw !== void 0 && raw !== "month" && raw !== "30d") {
|
|
9537
|
+
return c.json({ ok: false, error: "window must be 'month' or '30d'" }, 400);
|
|
9538
|
+
}
|
|
9539
|
+
const user = c.get("user");
|
|
9540
|
+
const { window: window2, since } = resolveBillingSummaryWindow(raw);
|
|
9541
|
+
const [summary, connectedState] = await Promise.all([
|
|
9542
|
+
getBillingLedgerSummary(user.id, toSqliteUtc(since), LEDGER_OPERATION_CLASSIFICATION),
|
|
9543
|
+
getConnectedAccountBillingState(user.id)
|
|
9544
|
+
]);
|
|
9545
|
+
const connectedQuantity = Math.max(0, Math.round(Number(connectedState?.quantity ?? 0)));
|
|
9546
|
+
c.header("Cache-Control", "no-store");
|
|
9547
|
+
return c.json({
|
|
9548
|
+
ok: true,
|
|
9549
|
+
window: window2,
|
|
9550
|
+
since: since.toISOString(),
|
|
9551
|
+
balance_mc: user.balance_mc,
|
|
9552
|
+
totals: {
|
|
9553
|
+
spent_mc: summary.spent_mc,
|
|
9554
|
+
refunded_mc: summary.refunded_mc,
|
|
9555
|
+
granted_mc: summary.granted_mc,
|
|
9556
|
+
purchased_mc: summary.purchased_mc,
|
|
9557
|
+
net_mc: summary.net_mc
|
|
9558
|
+
},
|
|
9559
|
+
by_operation: summary.by_operation,
|
|
9560
|
+
connected_monthly_mc: connectedQuantity * CONNECTED_ACTIVE_CONNECTION_MONTHLY_MC
|
|
9561
|
+
});
|
|
9562
|
+
}
|
|
9563
|
+
|
|
9217
9564
|
// src/api/youtube-routes.ts
|
|
9218
|
-
import { Hono as
|
|
9565
|
+
import { Hono as Hono5 } from "hono";
|
|
9219
9566
|
|
|
9220
9567
|
// src/youtube/youtube-harvest.ts
|
|
9221
9568
|
import { promises as fs } from "fs";
|
|
@@ -9223,17 +9570,17 @@ import path4 from "path";
|
|
|
9223
9570
|
import Papa from "papaparse";
|
|
9224
9571
|
|
|
9225
9572
|
// src/youtube/schemas.ts
|
|
9226
|
-
import { z as
|
|
9227
|
-
var YouTubeHarvestOptionsSchema =
|
|
9228
|
-
mode:
|
|
9229
|
-
query:
|
|
9230
|
-
channelHandle:
|
|
9231
|
-
maxVideos:
|
|
9232
|
-
downloadMp3:
|
|
9233
|
-
outputDir:
|
|
9234
|
-
kernelApiKey:
|
|
9235
|
-
kernelProxyId:
|
|
9236
|
-
headless:
|
|
9573
|
+
import { z as z13 } from "zod";
|
|
9574
|
+
var YouTubeHarvestOptionsSchema = z13.object({
|
|
9575
|
+
mode: z13.enum(["search", "channel"]),
|
|
9576
|
+
query: z13.string().optional(),
|
|
9577
|
+
channelHandle: z13.string().optional(),
|
|
9578
|
+
maxVideos: z13.number().int().min(1).max(5e3).default(100),
|
|
9579
|
+
downloadMp3: z13.boolean().default(false),
|
|
9580
|
+
outputDir: z13.string().default("./yt-output"),
|
|
9581
|
+
kernelApiKey: z13.string().optional(),
|
|
9582
|
+
kernelProxyId: z13.string().optional(),
|
|
9583
|
+
headless: z13.boolean().default(false)
|
|
9237
9584
|
}).refine(
|
|
9238
9585
|
(o) => o.mode === "search" ? !!o.query : !!o.channelHandle,
|
|
9239
9586
|
{ message: "search mode requires query; channel mode requires channelHandle" }
|
|
@@ -10005,78 +10352,66 @@ async function fetchCaptions(videoId) {
|
|
|
10005
10352
|
);
|
|
10006
10353
|
}
|
|
10007
10354
|
|
|
10008
|
-
// src/api/api-auth.ts
|
|
10009
|
-
import { createMiddleware as createMiddleware2 } from "hono/factory";
|
|
10010
|
-
function createApiKeyAuth() {
|
|
10011
|
-
return createMiddleware2(async (c, next) => {
|
|
10012
|
-
const key = c.req.header("x-api-key");
|
|
10013
|
-
if (!key) return c.json({ error: "Missing API key" }, 401);
|
|
10014
|
-
const user = await getUserByApiKey(key);
|
|
10015
|
-
if (!user) return c.json({ error: "Invalid or inactive API key" }, 401);
|
|
10016
|
-
c.set("user", user);
|
|
10017
|
-
return next();
|
|
10018
|
-
});
|
|
10019
|
-
}
|
|
10020
|
-
|
|
10021
10355
|
// src/api/server-schemas.ts
|
|
10022
|
-
import { z as
|
|
10023
|
-
var HarvestBodySchema =
|
|
10024
|
-
query:
|
|
10025
|
-
location:
|
|
10026
|
-
depth:
|
|
10027
|
-
maxQuestions:
|
|
10028
|
-
callback_url:
|
|
10029
|
-
gl:
|
|
10030
|
-
hl:
|
|
10031
|
-
device:
|
|
10032
|
-
proxyMode:
|
|
10033
|
-
proxyZip:
|
|
10034
|
-
debug:
|
|
10035
|
-
serpOnly:
|
|
10036
|
-
pages:
|
|
10037
|
-
|
|
10038
|
-
|
|
10039
|
-
|
|
10040
|
-
|
|
10041
|
-
|
|
10042
|
-
|
|
10043
|
-
|
|
10044
|
-
|
|
10045
|
-
|
|
10046
|
-
|
|
10047
|
-
|
|
10048
|
-
|
|
10049
|
-
|
|
10050
|
-
|
|
10051
|
-
|
|
10052
|
-
|
|
10053
|
-
|
|
10054
|
-
|
|
10055
|
-
|
|
10056
|
-
|
|
10057
|
-
|
|
10058
|
-
|
|
10059
|
-
|
|
10060
|
-
|
|
10061
|
-
|
|
10062
|
-
|
|
10063
|
-
|
|
10064
|
-
|
|
10065
|
-
|
|
10066
|
-
|
|
10067
|
-
|
|
10068
|
-
|
|
10069
|
-
|
|
10070
|
-
|
|
10071
|
-
|
|
10072
|
-
|
|
10073
|
-
|
|
10074
|
-
|
|
10075
|
-
|
|
10076
|
-
|
|
10077
|
-
|
|
10078
|
-
|
|
10079
|
-
|
|
10356
|
+
import { z as z14 } from "zod";
|
|
10357
|
+
var HarvestBodySchema = z14.object({
|
|
10358
|
+
query: z14.string().min(1, "query is required"),
|
|
10359
|
+
location: z14.string().optional(),
|
|
10360
|
+
depth: z14.number().int().min(1).max(4).optional(),
|
|
10361
|
+
maxQuestions: z14.number().int().min(1).optional(),
|
|
10362
|
+
callback_url: z14.string().url().optional(),
|
|
10363
|
+
gl: z14.string().optional(),
|
|
10364
|
+
hl: z14.string().optional(),
|
|
10365
|
+
device: z14.enum(["desktop", "mobile"]).optional(),
|
|
10366
|
+
proxyMode: z14.enum(["configured", "none"]).optional(),
|
|
10367
|
+
proxyZip: z14.string().regex(/^\d{5}$/).optional(),
|
|
10368
|
+
debug: z14.boolean().optional(),
|
|
10369
|
+
serpOnly: z14.boolean().optional(),
|
|
10370
|
+
pages: z14.number().int().min(1).max(2).optional(),
|
|
10371
|
+
recency: z14.enum(["day", "week", "month", "year"]).optional()
|
|
10372
|
+
});
|
|
10373
|
+
var ExtractUrlBodySchema = z14.object({
|
|
10374
|
+
url: z14.string().min(1, "url is required"),
|
|
10375
|
+
screenshot: z14.boolean().optional(),
|
|
10376
|
+
screenshotDevice: z14.enum(["desktop", "mobile"]).optional(),
|
|
10377
|
+
extractBranding: z14.boolean().optional(),
|
|
10378
|
+
downloadMedia: z14.boolean().optional(),
|
|
10379
|
+
mediaTypes: z14.array(z14.enum(["image", "video", "audio"])).optional(),
|
|
10380
|
+
allowLocal: z14.boolean().optional(),
|
|
10381
|
+
depositToVault: z14.boolean().optional(),
|
|
10382
|
+
vaultName: z14.string().trim().min(1).max(120).optional()
|
|
10383
|
+
});
|
|
10384
|
+
var DiffPageBodySchema = z14.object({
|
|
10385
|
+
url: z14.string().min(1, "url is required"),
|
|
10386
|
+
allowLocal: z14.boolean().optional(),
|
|
10387
|
+
resetBaseline: z14.boolean().optional()
|
|
10388
|
+
});
|
|
10389
|
+
var MapUrlsBodySchema = z14.object({
|
|
10390
|
+
url: z14.string().min(1, "url is required"),
|
|
10391
|
+
maxUrls: z14.number().int().min(1).max(2e3).optional(),
|
|
10392
|
+
concurrency: z14.number().int().min(1).max(20).optional(),
|
|
10393
|
+
browserFallback: z14.boolean().optional(),
|
|
10394
|
+
kernelFallback: z14.boolean().optional()
|
|
10395
|
+
});
|
|
10396
|
+
var ExtractSiteBodySchema = z14.object({
|
|
10397
|
+
url: z14.string().min(1, "url is required"),
|
|
10398
|
+
maxPages: z14.number().int().min(1).max(1e4).optional(),
|
|
10399
|
+
browserFallback: z14.boolean().optional(),
|
|
10400
|
+
kernelFallback: z14.boolean().optional(),
|
|
10401
|
+
rotateProxies: z14.boolean().optional(),
|
|
10402
|
+
rotateProxyEvery: z14.number().int().min(1).max(100).optional(),
|
|
10403
|
+
background: z14.boolean().optional(),
|
|
10404
|
+
downloadImages: z14.boolean().optional(),
|
|
10405
|
+
formats: z14.array(z14.enum(["markdown", "links", "json", "images", "branding", "issues"])).optional()
|
|
10406
|
+
});
|
|
10407
|
+
var YoutubeHarvestBodySchema = z14.object({
|
|
10408
|
+
mode: z14.enum(["search", "channel"]),
|
|
10409
|
+
query: z14.string().optional(),
|
|
10410
|
+
channelHandle: z14.string().optional(),
|
|
10411
|
+
maxVideos: z14.number().int().min(1).max(500).optional()
|
|
10412
|
+
});
|
|
10413
|
+
var YoutubeTranscribeBodySchema = z14.object({
|
|
10414
|
+
videoId: z14.string().min(1, "videoId is required")
|
|
10080
10415
|
});
|
|
10081
10416
|
|
|
10082
10417
|
// src/api/concurrency-gates.ts
|
|
@@ -10254,7 +10589,7 @@ async function extendConcurrencyGate(lockId, ttlSeconds = DEFAULT_LOCK_TTL_SECON
|
|
|
10254
10589
|
}
|
|
10255
10590
|
|
|
10256
10591
|
// src/api/youtube-routes.ts
|
|
10257
|
-
var youtubeApp = new
|
|
10592
|
+
var youtubeApp = new Hono5();
|
|
10258
10593
|
youtubeApp.post("/harvest", createApiKeyAuth(), async (c) => {
|
|
10259
10594
|
const raw = await c.req.json().catch(() => ({}));
|
|
10260
10595
|
const parsed = YoutubeHarvestBodySchema.safeParse(raw);
|
|
@@ -10413,14 +10748,14 @@ function fmtTs(secs) {
|
|
|
10413
10748
|
}
|
|
10414
10749
|
|
|
10415
10750
|
// src/api/screenshot-routes.ts
|
|
10416
|
-
import { Hono as
|
|
10417
|
-
import { z as
|
|
10418
|
-
var ScreenshotBodySchema =
|
|
10419
|
-
url:
|
|
10420
|
-
device:
|
|
10421
|
-
allowLocal:
|
|
10751
|
+
import { Hono as Hono6 } from "hono";
|
|
10752
|
+
import { z as z15 } from "zod";
|
|
10753
|
+
var ScreenshotBodySchema = z15.object({
|
|
10754
|
+
url: z15.string().trim().min(1, "url is required"),
|
|
10755
|
+
device: z15.string().trim().optional(),
|
|
10756
|
+
allowLocal: z15.boolean().optional()
|
|
10422
10757
|
});
|
|
10423
|
-
var screenshotApp = new
|
|
10758
|
+
var screenshotApp = new Hono6();
|
|
10424
10759
|
screenshotApp.use("*", createApiKeyAuth());
|
|
10425
10760
|
screenshotApp.post("/", async (c) => {
|
|
10426
10761
|
const raw = await c.req.json().catch(() => ({}));
|
|
@@ -10477,8 +10812,8 @@ screenshotApp.post("/", async (c) => {
|
|
|
10477
10812
|
});
|
|
10478
10813
|
|
|
10479
10814
|
// src/api/facebook-ad-routes.ts
|
|
10480
|
-
import { Hono as
|
|
10481
|
-
import { z as
|
|
10815
|
+
import { Hono as Hono7 } from "hono";
|
|
10816
|
+
import { z as z16 } from "zod";
|
|
10482
10817
|
|
|
10483
10818
|
// src/extractor/FacebookAdExtractor.ts
|
|
10484
10819
|
var SPRITE_Y_TO_PLATFORM = {
|
|
@@ -11371,35 +11706,35 @@ async function extractFacebookOrganicVideoFromPage(page, sourceUrl, quality = "b
|
|
|
11371
11706
|
|
|
11372
11707
|
// src/api/facebook-ad-routes.ts
|
|
11373
11708
|
import { fal } from "@fal-ai/client";
|
|
11374
|
-
var FacebookAdBodySchema =
|
|
11375
|
-
url:
|
|
11376
|
-
libraryId:
|
|
11377
|
-
openModal:
|
|
11709
|
+
var FacebookAdBodySchema = z16.object({
|
|
11710
|
+
url: z16.string().trim().optional(),
|
|
11711
|
+
libraryId: z16.string().trim().optional(),
|
|
11712
|
+
openModal: z16.boolean().optional()
|
|
11378
11713
|
}).refine((d) => !!d.url || !!d.libraryId, { message: "url or libraryId is required" });
|
|
11379
|
-
var FacebookPageIntelBodySchema =
|
|
11380
|
-
pageId:
|
|
11381
|
-
query:
|
|
11382
|
-
libraryId:
|
|
11383
|
-
maxAds:
|
|
11384
|
-
country:
|
|
11714
|
+
var FacebookPageIntelBodySchema = z16.object({
|
|
11715
|
+
pageId: z16.string().trim().optional(),
|
|
11716
|
+
query: z16.string().trim().optional(),
|
|
11717
|
+
libraryId: z16.string().trim().optional(),
|
|
11718
|
+
maxAds: z16.number().int().min(1).max(200).optional(),
|
|
11719
|
+
country: z16.string().trim().toUpperCase().optional()
|
|
11385
11720
|
}).refine((d) => !!d.pageId || !!d.query || !!d.libraryId, {
|
|
11386
11721
|
message: "pageId, libraryId, or query is required"
|
|
11387
11722
|
});
|
|
11388
|
-
var FacebookTranscribeBodySchema =
|
|
11389
|
-
videoUrl:
|
|
11723
|
+
var FacebookTranscribeBodySchema = z16.object({
|
|
11724
|
+
videoUrl: z16.string().trim().min(1, "videoUrl is required")
|
|
11390
11725
|
});
|
|
11391
|
-
var FacebookVideoTranscribeBodySchema =
|
|
11392
|
-
url:
|
|
11393
|
-
quality:
|
|
11726
|
+
var FacebookVideoTranscribeBodySchema = z16.object({
|
|
11727
|
+
url: z16.string().trim().min(1, "url is required"),
|
|
11728
|
+
quality: z16.enum(["best", "hd", "sd"]).default("best")
|
|
11394
11729
|
});
|
|
11395
|
-
var FacebookSearchBodySchema =
|
|
11396
|
-
query:
|
|
11397
|
-
country:
|
|
11398
|
-
maxResults:
|
|
11730
|
+
var FacebookSearchBodySchema = z16.object({
|
|
11731
|
+
query: z16.string().trim().min(1, "query is required"),
|
|
11732
|
+
country: z16.string().trim().toUpperCase().optional(),
|
|
11733
|
+
maxResults: z16.number().int().min(1).max(20).optional()
|
|
11399
11734
|
});
|
|
11400
|
-
var FacebookMediaBodySchema =
|
|
11401
|
-
url:
|
|
11402
|
-
filename:
|
|
11735
|
+
var FacebookMediaBodySchema = z16.object({
|
|
11736
|
+
url: z16.string().trim().min(1, "url is required"),
|
|
11737
|
+
filename: z16.string().trim().optional()
|
|
11403
11738
|
});
|
|
11404
11739
|
function invalidRequest(message) {
|
|
11405
11740
|
return { error_code: "invalid_request", message };
|
|
@@ -11490,7 +11825,7 @@ async function kernelLaunchOptsResidential() {
|
|
|
11490
11825
|
}
|
|
11491
11826
|
return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
|
|
11492
11827
|
}
|
|
11493
|
-
var facebookAdApp = new
|
|
11828
|
+
var facebookAdApp = new Hono7();
|
|
11494
11829
|
facebookAdApp.post("/ad", createApiKeyAuth(), async (c) => {
|
|
11495
11830
|
const raw = await c.req.json().catch(() => ({}));
|
|
11496
11831
|
const parsed = FacebookAdBodySchema.safeParse(raw);
|
|
@@ -11832,8 +12167,8 @@ facebookAdApp.post("/media", createApiKeyAuth(), async (c) => {
|
|
|
11832
12167
|
});
|
|
11833
12168
|
|
|
11834
12169
|
// src/api/google-ads-routes.ts
|
|
11835
|
-
import { Hono as
|
|
11836
|
-
import { z as
|
|
12170
|
+
import { Hono as Hono8 } from "hono";
|
|
12171
|
+
import { z as z17 } from "zod";
|
|
11837
12172
|
|
|
11838
12173
|
// src/extractor/GoogleAdsExtractor.ts
|
|
11839
12174
|
var ORIGIN = "https://adstransparency.google.com";
|
|
@@ -12003,19 +12338,19 @@ var GoogleAdsExtractor = class {
|
|
|
12003
12338
|
|
|
12004
12339
|
// src/api/google-ads-routes.ts
|
|
12005
12340
|
import { fal as fal2 } from "@fal-ai/client";
|
|
12006
|
-
var GoogleAdsSearchBodySchema =
|
|
12007
|
-
query:
|
|
12008
|
-
region:
|
|
12009
|
-
maxResults:
|
|
12010
|
-
});
|
|
12011
|
-
var GoogleAdsPageIntelBodySchema =
|
|
12012
|
-
advertiserId:
|
|
12013
|
-
domain:
|
|
12014
|
-
region:
|
|
12015
|
-
maxAds:
|
|
12341
|
+
var GoogleAdsSearchBodySchema = z17.object({
|
|
12342
|
+
query: z17.string().trim().min(1, "query is required"),
|
|
12343
|
+
region: z17.string().trim().toUpperCase().length(2).optional(),
|
|
12344
|
+
maxResults: z17.number().int().min(1).max(20).optional()
|
|
12345
|
+
});
|
|
12346
|
+
var GoogleAdsPageIntelBodySchema = z17.object({
|
|
12347
|
+
advertiserId: z17.string().trim().optional(),
|
|
12348
|
+
domain: z17.string().trim().optional(),
|
|
12349
|
+
region: z17.string().trim().toUpperCase().length(2).optional(),
|
|
12350
|
+
maxAds: z17.number().int().min(1).max(200).optional()
|
|
12016
12351
|
}).refine((d) => !!d.advertiserId || !!d.domain, { message: "advertiserId or domain is required" });
|
|
12017
|
-
var GoogleAdsTranscribeBodySchema =
|
|
12018
|
-
videoUrl:
|
|
12352
|
+
var GoogleAdsTranscribeBodySchema = z17.object({
|
|
12353
|
+
videoUrl: z17.string().trim().min(1, "videoUrl is required")
|
|
12019
12354
|
});
|
|
12020
12355
|
function invalidRequest2(message) {
|
|
12021
12356
|
return { error_code: "invalid_request", message };
|
|
@@ -12041,7 +12376,7 @@ async function kernelLaunchOptsResidential2() {
|
|
|
12041
12376
|
function isBlockedMessage(msg) {
|
|
12042
12377
|
return msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha");
|
|
12043
12378
|
}
|
|
12044
|
-
var googleAdsApp = new
|
|
12379
|
+
var googleAdsApp = new Hono8();
|
|
12045
12380
|
googleAdsApp.post("/search", createApiKeyAuth(), async (c) => {
|
|
12046
12381
|
const raw = await c.req.json().catch(() => ({}));
|
|
12047
12382
|
const parsed = GoogleAdsSearchBodySchema.safeParse(raw);
|
|
@@ -12157,8 +12492,8 @@ googleAdsApp.post("/transcribe", createApiKeyAuth(), async (c) => {
|
|
|
12157
12492
|
});
|
|
12158
12493
|
|
|
12159
12494
|
// src/api/instagram-routes.ts
|
|
12160
|
-
import { Hono as
|
|
12161
|
-
import { z as
|
|
12495
|
+
import { Hono as Hono9 } from "hono";
|
|
12496
|
+
import { z as z18 } from "zod";
|
|
12162
12497
|
import { mkdtempSync, readFileSync as readFileSync2, rmSync, writeFileSync } from "fs";
|
|
12163
12498
|
import { tmpdir } from "os";
|
|
12164
12499
|
import { join as join3 } from "path";
|
|
@@ -12520,32 +12855,32 @@ async function extractInstagramMediaFromPage(page, sourceUrl, capturedMediaUrls
|
|
|
12520
12855
|
}
|
|
12521
12856
|
|
|
12522
12857
|
// src/api/instagram-routes.ts
|
|
12523
|
-
var InstagramProfileContentBodySchema =
|
|
12524
|
-
handle:
|
|
12525
|
-
url:
|
|
12526
|
-
profile:
|
|
12527
|
-
saveProfileChanges:
|
|
12528
|
-
save_profile_changes:
|
|
12529
|
-
maxItems:
|
|
12530
|
-
maxScrolls:
|
|
12531
|
-
scrollDelayMs:
|
|
12532
|
-
stableScrollLimit:
|
|
12858
|
+
var InstagramProfileContentBodySchema = z18.object({
|
|
12859
|
+
handle: z18.string().trim().optional(),
|
|
12860
|
+
url: z18.string().trim().optional(),
|
|
12861
|
+
profile: z18.string().trim().min(1).optional(),
|
|
12862
|
+
saveProfileChanges: z18.boolean().optional(),
|
|
12863
|
+
save_profile_changes: z18.boolean().optional(),
|
|
12864
|
+
maxItems: z18.number().int().min(1).max(2e3).default(50),
|
|
12865
|
+
maxScrolls: z18.number().int().min(0).max(250).default(10),
|
|
12866
|
+
scrollDelayMs: z18.number().int().min(250).max(5e3).default(1200),
|
|
12867
|
+
stableScrollLimit: z18.number().int().min(1).max(10).default(4)
|
|
12533
12868
|
}).refine((d) => !!d.handle || !!d.url, {
|
|
12534
12869
|
message: "handle or url is required"
|
|
12535
12870
|
});
|
|
12536
|
-
var InstagramMediaTypeSchema =
|
|
12537
|
-
var InstagramMediaDownloadBodySchema =
|
|
12538
|
-
url:
|
|
12539
|
-
profile:
|
|
12540
|
-
saveProfileChanges:
|
|
12541
|
-
save_profile_changes:
|
|
12542
|
-
mediaTypes:
|
|
12543
|
-
downloadMedia:
|
|
12544
|
-
downloadAllTracks:
|
|
12545
|
-
includeTranscript:
|
|
12546
|
-
mux:
|
|
12547
|
-
});
|
|
12548
|
-
var instagramApp = new
|
|
12871
|
+
var InstagramMediaTypeSchema = z18.enum(["image", "video", "audio"]);
|
|
12872
|
+
var InstagramMediaDownloadBodySchema = z18.object({
|
|
12873
|
+
url: z18.string().trim().min(1, "url is required"),
|
|
12874
|
+
profile: z18.string().trim().min(1).optional(),
|
|
12875
|
+
saveProfileChanges: z18.boolean().optional(),
|
|
12876
|
+
save_profile_changes: z18.boolean().optional(),
|
|
12877
|
+
mediaTypes: z18.array(InstagramMediaTypeSchema).default(["image", "video", "audio"]),
|
|
12878
|
+
downloadMedia: z18.boolean().default(true),
|
|
12879
|
+
downloadAllTracks: z18.boolean().default(false),
|
|
12880
|
+
includeTranscript: z18.boolean().default(false),
|
|
12881
|
+
mux: z18.boolean().default(true)
|
|
12882
|
+
});
|
|
12883
|
+
var instagramApp = new Hono9();
|
|
12549
12884
|
function invalidRequest3(message) {
|
|
12550
12885
|
return { error_code: "invalid_request", message };
|
|
12551
12886
|
}
|
|
@@ -12825,11 +13160,168 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
|
|
|
12825
13160
|
});
|
|
12826
13161
|
|
|
12827
13162
|
// src/api/reddit-routes.ts
|
|
12828
|
-
import { Hono as
|
|
12829
|
-
import { z as
|
|
12830
|
-
|
|
12831
|
-
|
|
12832
|
-
|
|
13163
|
+
import { Hono as Hono10 } from "hono";
|
|
13164
|
+
import { z as z19 } from "zod";
|
|
13165
|
+
|
|
13166
|
+
// src/api/reddit-trending.ts
|
|
13167
|
+
function buildSerpQuery(topic, subreddit) {
|
|
13168
|
+
const scope = subreddit ? `site:reddit.com/r/${subreddit}` : "site:reddit.com";
|
|
13169
|
+
return `${scope} ${topic}`.trim();
|
|
13170
|
+
}
|
|
13171
|
+
function subredditFromUrl(url) {
|
|
13172
|
+
const m = url.match(/\/r\/([^/]+)\//i);
|
|
13173
|
+
return m ? m[1] : "";
|
|
13174
|
+
}
|
|
13175
|
+
function extractRedditThreadUrls(organicResults, limit) {
|
|
13176
|
+
if (!Array.isArray(organicResults)) return [];
|
|
13177
|
+
const seen = /* @__PURE__ */ new Set();
|
|
13178
|
+
const out = [];
|
|
13179
|
+
for (const raw of organicResults) {
|
|
13180
|
+
if (!raw || typeof raw !== "object") continue;
|
|
13181
|
+
const r = raw;
|
|
13182
|
+
const url = canonicalThreadUrl(r.url);
|
|
13183
|
+
if (!url || seen.has(url)) continue;
|
|
13184
|
+
seen.add(url);
|
|
13185
|
+
out.push({
|
|
13186
|
+
title: typeof r.title === "string" ? r.title.trim() : "",
|
|
13187
|
+
url,
|
|
13188
|
+
subreddit: subredditFromUrl(url)
|
|
13189
|
+
});
|
|
13190
|
+
if (out.length >= limit) break;
|
|
13191
|
+
}
|
|
13192
|
+
return out;
|
|
13193
|
+
}
|
|
13194
|
+
async function mapWithConcurrency(items, concurrency, fn) {
|
|
13195
|
+
const results = new Array(items.length);
|
|
13196
|
+
let next = 0;
|
|
13197
|
+
const workers = Array.from({ length: Math.max(1, Math.min(concurrency, items.length)) }, async () => {
|
|
13198
|
+
while (next < items.length) {
|
|
13199
|
+
const i = next++;
|
|
13200
|
+
results[i] = await fn(items[i], i);
|
|
13201
|
+
}
|
|
13202
|
+
});
|
|
13203
|
+
await Promise.all(workers);
|
|
13204
|
+
return results;
|
|
13205
|
+
}
|
|
13206
|
+
function parseCountText(text2) {
|
|
13207
|
+
if (typeof text2 !== "string") return 0;
|
|
13208
|
+
const m = text2.replace(/,/g, "").match(/\d+/);
|
|
13209
|
+
return m ? Number(m[0]) : 0;
|
|
13210
|
+
}
|
|
13211
|
+
function canonicalThreadUrl(href) {
|
|
13212
|
+
if (typeof href !== "string" || !href.trim()) return null;
|
|
13213
|
+
try {
|
|
13214
|
+
const u = new URL(href.trim(), "https://old.reddit.com");
|
|
13215
|
+
if (!/(^|\.)reddit\.com$/.test(u.hostname.toLowerCase())) return null;
|
|
13216
|
+
if (!/\/comments\//.test(u.pathname)) return null;
|
|
13217
|
+
u.hostname = "www.reddit.com";
|
|
13218
|
+
u.protocol = "https:";
|
|
13219
|
+
u.search = "";
|
|
13220
|
+
u.hash = "";
|
|
13221
|
+
return u.toString();
|
|
13222
|
+
} catch {
|
|
13223
|
+
return null;
|
|
13224
|
+
}
|
|
13225
|
+
}
|
|
13226
|
+
function engagementScore(score, commentCount) {
|
|
13227
|
+
return score + 2 * commentCount;
|
|
13228
|
+
}
|
|
13229
|
+
var INTERROGATIVE_STARTERS = /* @__PURE__ */ new Set([
|
|
13230
|
+
"what",
|
|
13231
|
+
"whats",
|
|
13232
|
+
"how",
|
|
13233
|
+
"why",
|
|
13234
|
+
"when",
|
|
13235
|
+
"where",
|
|
13236
|
+
"who",
|
|
13237
|
+
"whom",
|
|
13238
|
+
"whose",
|
|
13239
|
+
"which",
|
|
13240
|
+
"is",
|
|
13241
|
+
"are",
|
|
13242
|
+
"am",
|
|
13243
|
+
"was",
|
|
13244
|
+
"were",
|
|
13245
|
+
"do",
|
|
13246
|
+
"does",
|
|
13247
|
+
"did",
|
|
13248
|
+
"can",
|
|
13249
|
+
"could",
|
|
13250
|
+
"should",
|
|
13251
|
+
"would",
|
|
13252
|
+
"will",
|
|
13253
|
+
"shall",
|
|
13254
|
+
"has",
|
|
13255
|
+
"have",
|
|
13256
|
+
"had",
|
|
13257
|
+
"anyone",
|
|
13258
|
+
"any",
|
|
13259
|
+
"may",
|
|
13260
|
+
"might"
|
|
13261
|
+
]);
|
|
13262
|
+
function normalizeQuestionKey(s) {
|
|
13263
|
+
return s.toLowerCase().replace(/[?.!\s]+$/g, "").replace(/\s+/g, " ").trim();
|
|
13264
|
+
}
|
|
13265
|
+
function isQuestionSentence(s) {
|
|
13266
|
+
if (s.endsWith("?")) return true;
|
|
13267
|
+
const first = s.replace(/^[^a-zA-Z]+/, "").split(/\s+/)[0]?.toLowerCase() ?? "";
|
|
13268
|
+
return INTERROGATIVE_STARTERS.has(first.replace(/[^a-z]/g, ""));
|
|
13269
|
+
}
|
|
13270
|
+
function extractQuestions(texts, max = 5) {
|
|
13271
|
+
const out = [];
|
|
13272
|
+
const seen = /* @__PURE__ */ new Set();
|
|
13273
|
+
for (const text2 of texts) {
|
|
13274
|
+
if (out.length >= max) break;
|
|
13275
|
+
if (!text2) continue;
|
|
13276
|
+
const sentences = text2.split(/(?<=[.!?])\s+|\n+/);
|
|
13277
|
+
for (const raw of sentences) {
|
|
13278
|
+
const s = raw.trim();
|
|
13279
|
+
if (s.length < 8 || s.length > 300) continue;
|
|
13280
|
+
if (!isQuestionSentence(s)) continue;
|
|
13281
|
+
const key = normalizeQuestionKey(s);
|
|
13282
|
+
if (!key || seen.has(key)) continue;
|
|
13283
|
+
seen.add(key);
|
|
13284
|
+
out.push(s);
|
|
13285
|
+
if (out.length >= max) break;
|
|
13286
|
+
}
|
|
13287
|
+
}
|
|
13288
|
+
return out;
|
|
13289
|
+
}
|
|
13290
|
+
function trendingTotals(threads) {
|
|
13291
|
+
let upvotes = 0;
|
|
13292
|
+
let comments = 0;
|
|
13293
|
+
for (const t of threads) {
|
|
13294
|
+
upvotes += t.score;
|
|
13295
|
+
comments += t.commentCount;
|
|
13296
|
+
}
|
|
13297
|
+
return { threads: threads.length, upvotes, comments };
|
|
13298
|
+
}
|
|
13299
|
+
function flattenQuestions(threads) {
|
|
13300
|
+
const seen = /* @__PURE__ */ new Set();
|
|
13301
|
+
const out = [];
|
|
13302
|
+
for (const t of threads) {
|
|
13303
|
+
for (const q of t.topQuestions) {
|
|
13304
|
+
const key = normalizeQuestionKey(q);
|
|
13305
|
+
if (!key || seen.has(key)) continue;
|
|
13306
|
+
seen.add(key);
|
|
13307
|
+
out.push({ question: q, threadUrl: t.url });
|
|
13308
|
+
}
|
|
13309
|
+
}
|
|
13310
|
+
return out;
|
|
13311
|
+
}
|
|
13312
|
+
|
|
13313
|
+
// src/api/reddit-routes.ts
|
|
13314
|
+
var RedditThreadBodySchema = z19.object({
|
|
13315
|
+
url: z19.string().trim().min(1, "url is required"),
|
|
13316
|
+
maxComments: z19.number().int().min(1).max(2e3).optional()
|
|
13317
|
+
});
|
|
13318
|
+
var RedditTrendingBodySchema = z19.object({
|
|
13319
|
+
topic: z19.string().trim().min(1, "topic is required"),
|
|
13320
|
+
subreddit: z19.string().trim().regex(/^(r\/)?[A-Za-z0-9_]{2,24}$/i, "subreddit must be a bare name like SEO").optional(),
|
|
13321
|
+
window: z19.enum(["week", "month"]).default("month"),
|
|
13322
|
+
maxThreads: z19.number().int().min(1).max(40).default(20),
|
|
13323
|
+
includeComments: z19.boolean().default(true),
|
|
13324
|
+
maxCommentsPerThread: z19.number().int().min(1).max(200).default(50)
|
|
12833
13325
|
});
|
|
12834
13326
|
function invalidRequest4(message) {
|
|
12835
13327
|
return { error_code: "invalid_request", message };
|
|
@@ -12885,7 +13377,36 @@ var PARSE_REDDIT = `(() => {
|
|
|
12885
13377
|
});
|
|
12886
13378
|
return { title: title, author: author, score: score, postBody: postBody, blocked: blocked, numComments: comments.length, comments: comments };
|
|
12887
13379
|
})()`;
|
|
12888
|
-
|
|
13380
|
+
async function scrapeOldRedditWithRetries(url, script, accept) {
|
|
13381
|
+
for (let attempt = 0; attempt < 4; attempt++) {
|
|
13382
|
+
const backupAttempt = attempt === 3;
|
|
13383
|
+
if (backupAttempt && !backupProxyAvailable()) break;
|
|
13384
|
+
const proxyId = backupAttempt ? await createBackupProxyIdSafe(browserServiceApiKey(), attempt) : await residentialProxyId(attempt);
|
|
13385
|
+
if (backupAttempt && !proxyId) break;
|
|
13386
|
+
const driver = new BrowserDriver();
|
|
13387
|
+
try {
|
|
13388
|
+
await driver.launch({
|
|
13389
|
+
headless: true,
|
|
13390
|
+
headlessMode: "headful",
|
|
13391
|
+
kernelApiKey: browserServiceApiKey(),
|
|
13392
|
+
kernelProxyId: proxyId,
|
|
13393
|
+
viewport: { width: 1280, height: 900 },
|
|
13394
|
+
locale: "en-US"
|
|
13395
|
+
});
|
|
13396
|
+
const page = driver.getPage();
|
|
13397
|
+
await page.goto(url, { waitUntil: "domcontentloaded", timeout: attempt === 0 ? 3e4 : 45e3 });
|
|
13398
|
+
await page.waitForTimeout(2500);
|
|
13399
|
+
const data = await page.evaluate(script);
|
|
13400
|
+
if (accept(data)) return { data, attempts: attempt + 1, proxied: Boolean(proxyId) };
|
|
13401
|
+
} catch {
|
|
13402
|
+
} finally {
|
|
13403
|
+
await driver.close();
|
|
13404
|
+
if (backupAttempt) await cleanupBackupProxyId(browserServiceApiKey(), proxyId);
|
|
13405
|
+
}
|
|
13406
|
+
}
|
|
13407
|
+
return null;
|
|
13408
|
+
}
|
|
13409
|
+
var redditApp = new Hono10();
|
|
12889
13410
|
redditApp.post("/thread", createApiKeyAuth(), async (c) => {
|
|
12890
13411
|
const raw = await c.req.json().catch(() => ({}));
|
|
12891
13412
|
const parsed = RedditThreadBodySchema.safeParse(raw);
|
|
@@ -12907,29 +13428,13 @@ redditApp.post("/thread", createApiKeyAuth(), async (c) => {
|
|
|
12907
13428
|
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.reddit_thread), 402);
|
|
12908
13429
|
debited = true;
|
|
12909
13430
|
let result = null;
|
|
12910
|
-
|
|
12911
|
-
|
|
12912
|
-
|
|
12913
|
-
|
|
12914
|
-
|
|
12915
|
-
|
|
12916
|
-
|
|
12917
|
-
kernelApiKey: browserServiceApiKey(),
|
|
12918
|
-
kernelProxyId: proxyId,
|
|
12919
|
-
viewport: { width: 1280, height: 900 },
|
|
12920
|
-
locale: "en-US"
|
|
12921
|
-
});
|
|
12922
|
-
const page = driver.getPage();
|
|
12923
|
-
await page.goto(oldUrl, { waitUntil: "domcontentloaded", timeout: attempt === 0 ? 3e4 : 45e3 });
|
|
12924
|
-
await page.waitForTimeout(2500);
|
|
12925
|
-
const data = await page.evaluate(PARSE_REDDIT);
|
|
12926
|
-
if (!data.blocked && (data.postBody || data.comments.length > 0)) {
|
|
12927
|
-
result = { ...data, sourceUrl: body.url, oldRedditUrl: oldUrl, attempts: attempt + 1, proxied: Boolean(proxyId) };
|
|
12928
|
-
}
|
|
12929
|
-
} catch {
|
|
12930
|
-
} finally {
|
|
12931
|
-
await driver.close();
|
|
12932
|
-
}
|
|
13431
|
+
const scraped = await scrapeOldRedditWithRetries(
|
|
13432
|
+
oldUrl,
|
|
13433
|
+
PARSE_REDDIT,
|
|
13434
|
+
(data) => !data.blocked && (Boolean(data.postBody) || data.comments.length > 0)
|
|
13435
|
+
);
|
|
13436
|
+
if (scraped) {
|
|
13437
|
+
result = { ...scraped.data, sourceUrl: body.url, oldRedditUrl: oldUrl, attempts: scraped.attempts, proxied: scraped.proxied };
|
|
12933
13438
|
}
|
|
12934
13439
|
if (!result) {
|
|
12935
13440
|
await creditMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD_REFUND, "reddit blocked/empty after retries");
|
|
@@ -12965,17 +13470,152 @@ redditApp.post("/thread", createApiKeyAuth(), async (c) => {
|
|
|
12965
13470
|
await releaseConcurrencyGate(gate.lockId);
|
|
12966
13471
|
}
|
|
12967
13472
|
});
|
|
13473
|
+
var TRENDING_SOFT_DEADLINE_MS = 24e4;
|
|
13474
|
+
var TRENDING_SCRAPE_CONCURRENCY = 4;
|
|
13475
|
+
redditApp.post("/trending", createApiKeyAuth(), async (c) => {
|
|
13476
|
+
const raw = await c.req.json().catch(() => ({}));
|
|
13477
|
+
const parsed = RedditTrendingBodySchema.safeParse(raw);
|
|
13478
|
+
if (!parsed.success) return c.json(invalidRequest4(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
|
|
13479
|
+
const body = parsed.data;
|
|
13480
|
+
const subreddit = body.subreddit ? body.subreddit.replace(/^r\//i, "") : void 0;
|
|
13481
|
+
const serpQuery = buildSerpQuery(body.topic, subreddit);
|
|
13482
|
+
const user = c.get("user");
|
|
13483
|
+
const gate = await acquireConcurrencyGate(user, "reddit_trending", {
|
|
13484
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
13485
|
+
metadata: { topic: body.topic }
|
|
13486
|
+
});
|
|
13487
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
13488
|
+
const softDeadlineMs = Date.now() + TRENDING_SOFT_DEADLINE_MS;
|
|
13489
|
+
let discoveryDebited = false;
|
|
13490
|
+
let discoveryRefunded = false;
|
|
13491
|
+
const outstanding = { thread: 0, comment: 0 };
|
|
13492
|
+
try {
|
|
13493
|
+
const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD, serpQuery);
|
|
13494
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.reddit_thread), 402);
|
|
13495
|
+
discoveryDebited = true;
|
|
13496
|
+
let candidates = [];
|
|
13497
|
+
try {
|
|
13498
|
+
const serp = await harvest({
|
|
13499
|
+
query: serpQuery,
|
|
13500
|
+
serpOnly: true,
|
|
13501
|
+
proxyMode: "none",
|
|
13502
|
+
pages: 2,
|
|
13503
|
+
recency: body.window,
|
|
13504
|
+
kernelApiKey: browserServiceApiKey(),
|
|
13505
|
+
format: "json",
|
|
13506
|
+
outputDir: "/tmp/reddit-trending-serp",
|
|
13507
|
+
softDeadlineMs
|
|
13508
|
+
});
|
|
13509
|
+
candidates = extractRedditThreadUrls(serp.organicResults, body.maxThreads);
|
|
13510
|
+
} catch {
|
|
13511
|
+
candidates = [];
|
|
13512
|
+
}
|
|
13513
|
+
if (candidates.length === 0) {
|
|
13514
|
+
await creditMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD_REFUND, "no reddit threads discovered");
|
|
13515
|
+
discoveryRefunded = true;
|
|
13516
|
+
await logRequestEvent({ userId: user.id, source: "reddit_trending", status: "failed", query: body.topic, error: "no reddit threads discovered" });
|
|
13517
|
+
return c.json({ error: "No Reddit threads found for that topic and window (refunded)" }, 404);
|
|
13518
|
+
}
|
|
13519
|
+
let ranked;
|
|
13520
|
+
if (!body.includeComments) {
|
|
13521
|
+
ranked = candidates.map((cand) => ({
|
|
13522
|
+
title: cand.title,
|
|
13523
|
+
url: cand.url,
|
|
13524
|
+
subreddit: cand.subreddit,
|
|
13525
|
+
score: 0,
|
|
13526
|
+
commentCount: 0,
|
|
13527
|
+
engagementScore: 0,
|
|
13528
|
+
ageText: "",
|
|
13529
|
+
topQuestions: []
|
|
13530
|
+
}));
|
|
13531
|
+
} else {
|
|
13532
|
+
let balanceExhausted = false;
|
|
13533
|
+
const scrapedThreads = await mapWithConcurrency(candidates, TRENDING_SCRAPE_CONCURRENCY, async (cand) => {
|
|
13534
|
+
if (balanceExhausted || Date.now() >= softDeadlineMs) return null;
|
|
13535
|
+
const oldUrl = toOldReddit(cand.url);
|
|
13536
|
+
if (!oldUrl) return null;
|
|
13537
|
+
const threadDebit = await debitMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD, cand.url);
|
|
13538
|
+
if (!threadDebit.ok) {
|
|
13539
|
+
balanceExhausted = true;
|
|
13540
|
+
return null;
|
|
13541
|
+
}
|
|
13542
|
+
outstanding.thread += MC_COSTS.reddit_thread;
|
|
13543
|
+
const scraped = await scrapeOldRedditWithRetries(
|
|
13544
|
+
oldUrl,
|
|
13545
|
+
PARSE_REDDIT,
|
|
13546
|
+
(data) => !data.blocked && (Boolean(data.postBody) || data.comments.length > 0)
|
|
13547
|
+
);
|
|
13548
|
+
if (!scraped) {
|
|
13549
|
+
await creditMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD_REFUND, "reddit blocked/empty after retries");
|
|
13550
|
+
outstanding.thread -= MC_COSTS.reddit_thread;
|
|
13551
|
+
return null;
|
|
13552
|
+
}
|
|
13553
|
+
const score = parseCountText(scraped.data.score);
|
|
13554
|
+
let comments = Array.isArray(scraped.data.comments) ? scraped.data.comments : [];
|
|
13555
|
+
const commentCount = comments.length;
|
|
13556
|
+
comments = comments.slice(0, body.maxCommentsPerThread);
|
|
13557
|
+
let commentMc = comments.length * MC_COSTS.reddit_comment;
|
|
13558
|
+
if (commentMc > 0) {
|
|
13559
|
+
const commentDebit = await debitMc(user.id, commentMc, LedgerOperation.REDDIT_COMMENT, cand.url);
|
|
13560
|
+
if (!commentDebit.ok) {
|
|
13561
|
+
balanceExhausted = true;
|
|
13562
|
+
const affordable = Math.floor((commentDebit.balance_mc ?? 0) / MC_COSTS.reddit_comment);
|
|
13563
|
+
comments = comments.slice(0, affordable);
|
|
13564
|
+
commentMc = affordable * MC_COSTS.reddit_comment;
|
|
13565
|
+
if (commentMc > 0) await debitMc(user.id, commentMc, LedgerOperation.REDDIT_COMMENT, cand.url);
|
|
13566
|
+
}
|
|
13567
|
+
outstanding.comment += commentMc;
|
|
13568
|
+
}
|
|
13569
|
+
const topLevelBodies = comments.filter((cm) => cm && cm.depth === 0).map((cm) => cm.body);
|
|
13570
|
+
return {
|
|
13571
|
+
title: cand.title || scraped.data.title || "",
|
|
13572
|
+
url: cand.url,
|
|
13573
|
+
subreddit: cand.subreddit,
|
|
13574
|
+
score,
|
|
13575
|
+
commentCount,
|
|
13576
|
+
engagementScore: engagementScore(score, commentCount),
|
|
13577
|
+
ageText: "",
|
|
13578
|
+
topQuestions: extractQuestions([scraped.data.postBody, ...topLevelBodies])
|
|
13579
|
+
};
|
|
13580
|
+
});
|
|
13581
|
+
ranked = scrapedThreads.filter((t) => t !== null).sort((a, b) => b.engagementScore - a.engagementScore);
|
|
13582
|
+
}
|
|
13583
|
+
const result = {
|
|
13584
|
+
topic: body.topic,
|
|
13585
|
+
subreddit: subreddit ?? null,
|
|
13586
|
+
window: body.window === "week" ? "7d" : "30d",
|
|
13587
|
+
totals: trendingTotals(ranked),
|
|
13588
|
+
rankedThreads: ranked,
|
|
13589
|
+
questions: flattenQuestions(ranked),
|
|
13590
|
+
threadsScraped: body.includeComments ? ranked.length : 0,
|
|
13591
|
+
candidatesFound: candidates.length,
|
|
13592
|
+
partial: body.includeComments && ranked.length < candidates.length,
|
|
13593
|
+
searchQuery: serpQuery
|
|
13594
|
+
};
|
|
13595
|
+
await logRequestEvent({ userId: user.id, source: "reddit_trending", status: "done", query: body.topic, resultCount: ranked.length, result });
|
|
13596
|
+
return c.json(result);
|
|
13597
|
+
} catch (err) {
|
|
13598
|
+
if (discoveryDebited && !discoveryRefunded) await creditMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD_REFUND, "failed call");
|
|
13599
|
+
if (outstanding.thread > 0) await creditMc(user.id, outstanding.thread, LedgerOperation.REDDIT_THREAD_REFUND, "failed call");
|
|
13600
|
+
if (outstanding.comment > 0) await creditMc(user.id, outstanding.comment, LedgerOperation.REDDIT_COMMENT_REFUND, "failed call");
|
|
13601
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
13602
|
+
await logRequestEvent({ userId: user.id, source: "reddit_trending", status: "failed", query: body.topic, error: msg });
|
|
13603
|
+
return c.json({ error: msg }, 500);
|
|
13604
|
+
} finally {
|
|
13605
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
13606
|
+
}
|
|
13607
|
+
});
|
|
12968
13608
|
|
|
12969
13609
|
// src/api/video-routes.ts
|
|
12970
|
-
import { Hono as
|
|
12971
|
-
import { z as
|
|
12972
|
-
var videoApp = new
|
|
12973
|
-
var AnalyzeBodySchema =
|
|
12974
|
-
sourceUrl:
|
|
12975
|
-
intervalS:
|
|
12976
|
-
maxFrames:
|
|
12977
|
-
detail:
|
|
12978
|
-
vault:
|
|
13610
|
+
import { Hono as Hono11 } from "hono";
|
|
13611
|
+
import { z as z20 } from "zod";
|
|
13612
|
+
var videoApp = new Hono11();
|
|
13613
|
+
var AnalyzeBodySchema = z20.object({
|
|
13614
|
+
sourceUrl: z20.string().trim().url("sourceUrl must be a direct video file URL"),
|
|
13615
|
+
intervalS: z20.number().min(1).max(30).optional(),
|
|
13616
|
+
maxFrames: z20.number().int().min(1).max(480).optional(),
|
|
13617
|
+
detail: z20.enum(["fast", "standard", "deep"]).optional(),
|
|
13618
|
+
vault: z20.string().trim().min(1).optional()
|
|
12979
13619
|
});
|
|
12980
13620
|
function invalidRequest5(message) {
|
|
12981
13621
|
return { error_code: "invalid_request", message };
|
|
@@ -13079,7 +13719,7 @@ videoApp.post("/status", createApiKeyAuth(), async (c) => {
|
|
|
13079
13719
|
});
|
|
13080
13720
|
|
|
13081
13721
|
// src/api/maps-routes.ts
|
|
13082
|
-
import { Hono as
|
|
13722
|
+
import { Hono as Hono12 } from "hono";
|
|
13083
13723
|
|
|
13084
13724
|
// src/extractor/GoogleBusinessPanel.ts
|
|
13085
13725
|
function clickNamedBusinessCardInBrowser(businessName) {
|
|
@@ -13805,7 +14445,7 @@ var MapsReviewCollector = class {
|
|
|
13805
14445
|
};
|
|
13806
14446
|
|
|
13807
14447
|
// src/extractor/MapsExtractor.ts
|
|
13808
|
-
import { z as
|
|
14448
|
+
import { z as z21 } from "zod";
|
|
13809
14449
|
var MapsExtractor = class {
|
|
13810
14450
|
constructor(driver) {
|
|
13811
14451
|
this.driver = driver;
|
|
@@ -14032,7 +14672,7 @@ var MapsExtractor = class {
|
|
|
14032
14672
|
});
|
|
14033
14673
|
return rows;
|
|
14034
14674
|
}, { hoursTable: MapsSelectors.hoursTable, hoursTableAlt: MapsSelectors.hoursTableAlt });
|
|
14035
|
-
const result =
|
|
14675
|
+
const result = z21.array(RawMapsHoursRowSchema).safeParse(raw);
|
|
14036
14676
|
if (!result.success) {
|
|
14037
14677
|
console.warn("[MapsExtractor] hours parse failed", result.error.flatten());
|
|
14038
14678
|
return [];
|
|
@@ -14096,7 +14736,7 @@ var MapsExtractor = class {
|
|
|
14096
14736
|
});
|
|
14097
14737
|
return results;
|
|
14098
14738
|
});
|
|
14099
|
-
const result =
|
|
14739
|
+
const result = z21.array(RawMapsAboutAttributeSchema).safeParse(raw);
|
|
14100
14740
|
if (!result.success) {
|
|
14101
14741
|
console.warn("[MapsExtractor] about parse failed", result.error.flatten());
|
|
14102
14742
|
return [];
|
|
@@ -14197,7 +14837,8 @@ async function cleanupDisposableProxy(kernelApiKey, proxyId, eventName) {
|
|
|
14197
14837
|
async function runMapsSearchWithRotation(options) {
|
|
14198
14838
|
if (options.proxyMode === "location") options = { ...options, proxyMode: "none" };
|
|
14199
14839
|
const attempts = [];
|
|
14200
|
-
const
|
|
14840
|
+
const baseMaxAttempts = mapsSearchMaxAttemptsForProxyMode(options.proxyMode) + 1;
|
|
14841
|
+
const maxAttempts = baseMaxAttempts + (backupProxyAvailable() ? BACKUP_LAST_RESORT_ATTEMPTS : 0);
|
|
14201
14842
|
const started = Date.now();
|
|
14202
14843
|
let lastError = null;
|
|
14203
14844
|
for (let attemptIndex = 0; attemptIndex < maxAttempts; attemptIndex += 1) {
|
|
@@ -14207,8 +14848,13 @@ async function runMapsSearchWithRotation(options) {
|
|
|
14207
14848
|
const proxyZip = options.proxyZipForAttempt?.(attemptIndex) ?? options.proxyZip;
|
|
14208
14849
|
let resolution = null;
|
|
14209
14850
|
let disposableProxyId;
|
|
14851
|
+
const isBackupAttempt = attemptIndex >= baseMaxAttempts;
|
|
14210
14852
|
try {
|
|
14211
|
-
resolution = await
|
|
14853
|
+
resolution = isBackupAttempt ? await resolveBackupKernelProxyId({
|
|
14854
|
+
kernelApiKey: options.kernelApiKey,
|
|
14855
|
+
proxyMode: options.proxyMode,
|
|
14856
|
+
attemptIndex
|
|
14857
|
+
}) : await resolveKernelProxyId({
|
|
14212
14858
|
kernelApiKey: options.kernelApiKey,
|
|
14213
14859
|
proxyMode: options.proxyMode,
|
|
14214
14860
|
configuredKernelProxyId: options.kernelProxyId ?? options.configuredKernelProxyId,
|
|
@@ -14216,18 +14862,20 @@ async function runMapsSearchWithRotation(options) {
|
|
|
14216
14862
|
proxyZip,
|
|
14217
14863
|
gl: options.gl,
|
|
14218
14864
|
attemptIndex,
|
|
14219
|
-
fresh: options.proxyMode === "location"
|
|
14865
|
+
fresh: options.proxyMode === "location",
|
|
14866
|
+
backupOnExhaustion: true
|
|
14220
14867
|
});
|
|
14868
|
+
if (!resolution) break;
|
|
14221
14869
|
disposableProxyId = resolution.disposableProxyId;
|
|
14222
14870
|
const extractArgs = {
|
|
14223
14871
|
...options,
|
|
14224
14872
|
proxyZip,
|
|
14225
|
-
kernelProxyId: options.proxyMode === "none" ? void 0 : resolution.kernelProxyId,
|
|
14873
|
+
kernelProxyId: options.proxyMode === "none" && !isBackupAttempt ? void 0 : resolution.kernelProxyId,
|
|
14226
14874
|
kernelProxyResolution: resolution.resolution,
|
|
14227
|
-
serpRedirect: options.proxyMode === "none" && attemptIndex >= 2 && attemptIndex < 6,
|
|
14228
|
-
forceDirectEgress: options.proxyMode === "none" && attemptIndex >= 4
|
|
14875
|
+
serpRedirect: options.proxyMode === "none" && !isBackupAttempt && attemptIndex >= 2 && attemptIndex < 6,
|
|
14876
|
+
forceDirectEgress: options.proxyMode === "none" && !isBackupAttempt && attemptIndex >= 4
|
|
14229
14877
|
};
|
|
14230
|
-
const isHeadfulEscalation = attemptIndex
|
|
14878
|
+
const isHeadfulEscalation = attemptIndex >= baseMaxAttempts - 1;
|
|
14231
14879
|
const baseCtx = currentCostContext();
|
|
14232
14880
|
const result = isHeadfulEscalation && baseCtx ? await runWithCostContext({ ...baseCtx, forceHeadful: true }, () => extractor.extract(extractArgs)) : await extractor.extract(extractArgs);
|
|
14233
14881
|
const attempt = attemptFromDebug({
|
|
@@ -14314,7 +14962,7 @@ function mapsErrorResponse(c, err, errorCode) {
|
|
|
14314
14962
|
attempts: rotationError?.attempts ?? void 0
|
|
14315
14963
|
}, retryable ? 503 : 500);
|
|
14316
14964
|
}
|
|
14317
|
-
var mapsApp = new
|
|
14965
|
+
var mapsApp = new Hono12();
|
|
14318
14966
|
mapsApp.post("/search", createApiKeyAuth(), async (c) => {
|
|
14319
14967
|
const user = c.get("user");
|
|
14320
14968
|
const body = await c.req.json().catch(() => ({}));
|
|
@@ -14452,11 +15100,11 @@ mapsApp.post("/place", createApiKeyAuth(), async (c) => {
|
|
|
14452
15100
|
});
|
|
14453
15101
|
|
|
14454
15102
|
// src/api/trustpilot-routes.ts
|
|
14455
|
-
import { Hono as
|
|
14456
|
-
import { z as
|
|
14457
|
-
var TrustpilotReviewsBodySchema =
|
|
14458
|
-
domain:
|
|
14459
|
-
maxPages:
|
|
15103
|
+
import { Hono as Hono13 } from "hono";
|
|
15104
|
+
import { z as z22 } from "zod";
|
|
15105
|
+
var TrustpilotReviewsBodySchema = z22.object({
|
|
15106
|
+
domain: z22.string().trim().min(1, "domain is required"),
|
|
15107
|
+
maxPages: z22.number().int().min(1).max(50).default(5)
|
|
14460
15108
|
});
|
|
14461
15109
|
function invalidRequest6(message) {
|
|
14462
15110
|
return { error_code: "invalid_request", message };
|
|
@@ -14521,7 +15169,7 @@ async function extractReviewCards(page) {
|
|
|
14521
15169
|
});
|
|
14522
15170
|
});
|
|
14523
15171
|
}
|
|
14524
|
-
var trustpilotApp = new
|
|
15172
|
+
var trustpilotApp = new Hono13();
|
|
14525
15173
|
trustpilotApp.post("/reviews", createApiKeyAuth(), async (c) => {
|
|
14526
15174
|
const raw = await c.req.json().catch(() => ({}));
|
|
14527
15175
|
const parsed = TrustpilotReviewsBodySchema.safeParse(raw);
|
|
@@ -14602,11 +15250,11 @@ trustpilotApp.post("/reviews", createApiKeyAuth(), async (c) => {
|
|
|
14602
15250
|
});
|
|
14603
15251
|
|
|
14604
15252
|
// src/api/g2-routes.ts
|
|
14605
|
-
import { Hono as
|
|
14606
|
-
import { z as
|
|
14607
|
-
var G2ReviewsBodySchema =
|
|
14608
|
-
product:
|
|
14609
|
-
maxPages:
|
|
15253
|
+
import { Hono as Hono14 } from "hono";
|
|
15254
|
+
import { z as z23 } from "zod";
|
|
15255
|
+
var G2ReviewsBodySchema = z23.object({
|
|
15256
|
+
product: z23.string().trim().min(1, "product is required"),
|
|
15257
|
+
maxPages: z23.number().int().min(1).max(50).default(5)
|
|
14610
15258
|
});
|
|
14611
15259
|
function invalidRequest7(message) {
|
|
14612
15260
|
return { error_code: "invalid_request", message };
|
|
@@ -14711,7 +15359,7 @@ async function extractG2ReviewCards(page) {
|
|
|
14711
15359
|
});
|
|
14712
15360
|
});
|
|
14713
15361
|
}
|
|
14714
|
-
var g2App = new
|
|
15362
|
+
var g2App = new Hono14();
|
|
14715
15363
|
g2App.post("/reviews", createApiKeyAuth(), async (c) => {
|
|
14716
15364
|
const raw = await c.req.json().catch(() => ({}));
|
|
14717
15365
|
const parsed = G2ReviewsBodySchema.safeParse(raw);
|
|
@@ -14734,8 +15382,11 @@ g2App.post("/reviews", createApiKeyAuth(), async (c) => {
|
|
|
14734
15382
|
const allReviews = [];
|
|
14735
15383
|
let pagesFetched = 0;
|
|
14736
15384
|
let blocked = false;
|
|
14737
|
-
for (let attempt = 0; attempt <
|
|
14738
|
-
const
|
|
15385
|
+
for (let attempt = 0; attempt < 4 && pagesFetched === 0; attempt++) {
|
|
15386
|
+
const backupAttempt = attempt === 3;
|
|
15387
|
+
if (backupAttempt && !backupProxyAvailable()) break;
|
|
15388
|
+
const proxyId = backupAttempt ? await createBackupProxyIdSafe(browserServiceApiKey(), attempt) : await residentialProxyId2(attempt);
|
|
15389
|
+
if (backupAttempt && !proxyId) break;
|
|
14739
15390
|
const driver = new BrowserDriver();
|
|
14740
15391
|
try {
|
|
14741
15392
|
await driver.launch({
|
|
@@ -14768,6 +15419,7 @@ g2App.post("/reviews", createApiKeyAuth(), async (c) => {
|
|
|
14768
15419
|
blocked = true;
|
|
14769
15420
|
} finally {
|
|
14770
15421
|
await driver.close();
|
|
15422
|
+
if (backupAttempt) await cleanupBackupProxyId(browserServiceApiKey(), proxyId);
|
|
14771
15423
|
}
|
|
14772
15424
|
}
|
|
14773
15425
|
if (pagesFetched === 0 || allReviews.length === 0) {
|
|
@@ -14808,12 +15460,12 @@ g2App.post("/reviews", createApiKeyAuth(), async (c) => {
|
|
|
14808
15460
|
});
|
|
14809
15461
|
|
|
14810
15462
|
// src/api/directory-routes.ts
|
|
14811
|
-
import { Hono as
|
|
15463
|
+
import { Hono as Hono15 } from "hono";
|
|
14812
15464
|
|
|
14813
15465
|
// src/directory/directory-workflow.ts
|
|
14814
15466
|
import { mkdir as mkdir2, writeFile } from "fs/promises";
|
|
14815
15467
|
import { join as join4 } from "path";
|
|
14816
|
-
import { z as
|
|
15468
|
+
import { z as z24 } from "zod";
|
|
14817
15469
|
|
|
14818
15470
|
// src/directory/location-db.ts
|
|
14819
15471
|
import { access, readFile } from "fs/promises";
|
|
@@ -14974,24 +15626,24 @@ async function resolveDirectoryMarkets(options) {
|
|
|
14974
15626
|
}
|
|
14975
15627
|
|
|
14976
15628
|
// src/directory/directory-workflow.ts
|
|
14977
|
-
var DirectoryWorkflowOptionsSchema =
|
|
14978
|
-
query:
|
|
14979
|
-
state:
|
|
14980
|
-
minPopulation:
|
|
14981
|
-
populationYear:
|
|
14982
|
-
maxCities:
|
|
14983
|
-
maxResultsPerCity:
|
|
14984
|
-
concurrency:
|
|
14985
|
-
includeZipGroups:
|
|
14986
|
-
usZipsCsvPath:
|
|
14987
|
-
saveCsv:
|
|
14988
|
-
gl:
|
|
14989
|
-
hl:
|
|
14990
|
-
proxyMode:
|
|
14991
|
-
proxyZip:
|
|
14992
|
-
debug:
|
|
14993
|
-
headless:
|
|
14994
|
-
kernelApiKey:
|
|
15629
|
+
var DirectoryWorkflowOptionsSchema = z24.object({
|
|
15630
|
+
query: z24.string().min(1),
|
|
15631
|
+
state: z24.string().min(2).default("TN"),
|
|
15632
|
+
minPopulation: z24.number().int().min(0).default(1e5),
|
|
15633
|
+
populationYear: z24.union(POPULATION_YEARS.map((year) => z24.literal(year))).default(2025),
|
|
15634
|
+
maxCities: z24.number().int().min(1).max(100).default(25),
|
|
15635
|
+
maxResultsPerCity: z24.number().int().min(1).max(50).default(50),
|
|
15636
|
+
concurrency: z24.number().int().min(1).max(5).default(5),
|
|
15637
|
+
includeZipGroups: z24.boolean().default(true),
|
|
15638
|
+
usZipsCsvPath: z24.string().optional(),
|
|
15639
|
+
saveCsv: z24.boolean().default(true),
|
|
15640
|
+
gl: z24.string().length(2).default("us"),
|
|
15641
|
+
hl: z24.string().length(2).default("en"),
|
|
15642
|
+
proxyMode: z24.enum(["configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
15643
|
+
proxyZip: z24.string().regex(/^\d{5}$/).optional(),
|
|
15644
|
+
debug: z24.boolean().default(false),
|
|
15645
|
+
headless: z24.boolean().default(true),
|
|
15646
|
+
kernelApiKey: z24.string().optional()
|
|
14995
15647
|
});
|
|
14996
15648
|
function errorMessage(err) {
|
|
14997
15649
|
return err instanceof Error ? err.message : String(err);
|
|
@@ -15213,7 +15865,7 @@ async function runDirectoryWorkflowFromPlan(options, plan) {
|
|
|
15213
15865
|
}
|
|
15214
15866
|
|
|
15215
15867
|
// src/api/directory-routes.ts
|
|
15216
|
-
var directoryApp = new
|
|
15868
|
+
var directoryApp = new Hono15();
|
|
15217
15869
|
directoryApp.post("/run", createApiKeyAuth(), async (c) => {
|
|
15218
15870
|
const user = c.get("user");
|
|
15219
15871
|
const body = await c.req.json().catch(() => ({}));
|
|
@@ -15286,35 +15938,35 @@ directoryApp.post("/run", createApiKeyAuth(), async (c) => {
|
|
|
15286
15938
|
// src/api/workflow-routes.ts
|
|
15287
15939
|
import { createHmac as createHmac3 } from "crypto";
|
|
15288
15940
|
import { readFile as readFile2 } from "fs/promises";
|
|
15289
|
-
import { Hono as
|
|
15290
|
-
import { z as
|
|
15291
|
-
var workflowApp = new
|
|
15292
|
-
var WorkflowInputSchema =
|
|
15293
|
-
var WorkflowIdSchema =
|
|
15294
|
-
var CadenceSchema =
|
|
15295
|
-
var ScheduleStatusSchema =
|
|
15296
|
-
var RunBodySchema =
|
|
15941
|
+
import { Hono as Hono16 } from "hono";
|
|
15942
|
+
import { z as z25 } from "zod";
|
|
15943
|
+
var workflowApp = new Hono16();
|
|
15944
|
+
var WorkflowInputSchema = z25.record(z25.unknown()).default({});
|
|
15945
|
+
var WorkflowIdSchema = z25.string().min(1);
|
|
15946
|
+
var CadenceSchema = z25.enum(["daily", "weekly", "monthly"]);
|
|
15947
|
+
var ScheduleStatusSchema = z25.enum(["active", "paused"]);
|
|
15948
|
+
var RunBodySchema = z25.object({
|
|
15297
15949
|
workflowId: WorkflowIdSchema,
|
|
15298
15950
|
input: WorkflowInputSchema,
|
|
15299
|
-
webhookUrl:
|
|
15951
|
+
webhookUrl: z25.string().url().optional()
|
|
15300
15952
|
});
|
|
15301
|
-
var ScheduleCreateSchema =
|
|
15953
|
+
var ScheduleCreateSchema = z25.object({
|
|
15302
15954
|
workflowId: WorkflowIdSchema,
|
|
15303
|
-
name:
|
|
15955
|
+
name: z25.string().min(1).max(120).optional(),
|
|
15304
15956
|
input: WorkflowInputSchema,
|
|
15305
15957
|
cadence: CadenceSchema.default("weekly"),
|
|
15306
|
-
timezone:
|
|
15307
|
-
webhookUrl:
|
|
15308
|
-
nextRunAt:
|
|
15958
|
+
timezone: z25.string().min(1).max(64).default("UTC"),
|
|
15959
|
+
webhookUrl: z25.string().url().optional(),
|
|
15960
|
+
nextRunAt: z25.string().datetime().optional()
|
|
15309
15961
|
});
|
|
15310
|
-
var SchedulePatchSchema =
|
|
15311
|
-
name:
|
|
15962
|
+
var SchedulePatchSchema = z25.object({
|
|
15963
|
+
name: z25.string().min(1).max(120).optional(),
|
|
15312
15964
|
status: ScheduleStatusSchema.optional(),
|
|
15313
15965
|
input: WorkflowInputSchema.optional(),
|
|
15314
15966
|
cadence: CadenceSchema.optional(),
|
|
15315
|
-
timezone:
|
|
15316
|
-
webhookUrl:
|
|
15317
|
-
nextRunAt:
|
|
15967
|
+
timezone: z25.string().min(1).max(64).optional(),
|
|
15968
|
+
webhookUrl: z25.string().url().nullable().optional(),
|
|
15969
|
+
nextRunAt: z25.string().datetime().nullable().optional()
|
|
15318
15970
|
});
|
|
15319
15971
|
function hostedWorkflowOutputDir() {
|
|
15320
15972
|
return workflowOutputBaseDir(process.env.MCP_SCRAPER_WORKFLOW_OUTPUT_DIR?.trim() || "/tmp/mcp-scraper-workflows");
|
|
@@ -15818,7 +16470,7 @@ workflowApp.post("/cron/dispatch", async (c) => {
|
|
|
15818
16470
|
});
|
|
15819
16471
|
|
|
15820
16472
|
// src/api/serp-intelligence-routes.ts
|
|
15821
|
-
import { Hono as
|
|
16473
|
+
import { Hono as Hono17 } from "hono";
|
|
15822
16474
|
import { createHash as createHash5, randomUUID as randomUUID4 } from "crypto";
|
|
15823
16475
|
|
|
15824
16476
|
// src/serp-intelligence/page-snapshot-extractor.ts
|
|
@@ -16131,7 +16783,7 @@ async function capturePageSnapshots(targets, options = {}) {
|
|
|
16131
16783
|
}
|
|
16132
16784
|
|
|
16133
16785
|
// src/serp-intelligence/schemas.ts
|
|
16134
|
-
import { z as
|
|
16786
|
+
import { z as z26 } from "zod";
|
|
16135
16787
|
var SerpIntelligenceDeviceValues = ["desktop", "mobile"];
|
|
16136
16788
|
var SerpIntelligenceProxyModeValues = ["location", "configured", "none"];
|
|
16137
16789
|
var SerpIntelligenceAttemptOutcomeValues = [
|
|
@@ -16193,181 +16845,181 @@ function isPublicHttpUrl(value) {
|
|
|
16193
16845
|
return false;
|
|
16194
16846
|
}
|
|
16195
16847
|
}
|
|
16196
|
-
var SerpIntelligencePublicHttpUrlSchema =
|
|
16197
|
-
var SerpIntelligenceCaptureBodySchema =
|
|
16198
|
-
query:
|
|
16199
|
-
location:
|
|
16200
|
-
gl:
|
|
16201
|
-
hl:
|
|
16202
|
-
device:
|
|
16203
|
-
proxyMode:
|
|
16204
|
-
proxyZip:
|
|
16205
|
-
pages:
|
|
16206
|
-
debug:
|
|
16207
|
-
includePageSnapshots:
|
|
16208
|
-
pageSnapshotLimit:
|
|
16848
|
+
var SerpIntelligencePublicHttpUrlSchema = z26.string().url().refine(isPublicHttpUrl, "url must be a public HTTP or HTTPS URL");
|
|
16849
|
+
var SerpIntelligenceCaptureBodySchema = z26.object({
|
|
16850
|
+
query: z26.string().trim().min(1, "query is required"),
|
|
16851
|
+
location: z26.string().trim().min(1).optional(),
|
|
16852
|
+
gl: z26.string().trim().length(2).default("us"),
|
|
16853
|
+
hl: z26.string().trim().length(2).default("en"),
|
|
16854
|
+
device: z26.enum(SerpIntelligenceDeviceValues).default("desktop"),
|
|
16855
|
+
proxyMode: z26.enum(SerpIntelligenceProxyModeValues).default(DEFAULT_PROXY_MODE),
|
|
16856
|
+
proxyZip: z26.string().regex(/^\d{5}$/).optional(),
|
|
16857
|
+
pages: z26.number().int().min(1).max(2).default(1),
|
|
16858
|
+
debug: z26.boolean().default(false),
|
|
16859
|
+
includePageSnapshots: z26.boolean().default(false),
|
|
16860
|
+
pageSnapshotLimit: z26.number().int().min(0).max(10).default(0)
|
|
16209
16861
|
}).strict();
|
|
16210
|
-
var SerpIntelligencePageSnapshotRequestSchema =
|
|
16862
|
+
var SerpIntelligencePageSnapshotRequestSchema = z26.object({
|
|
16211
16863
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
16212
|
-
sourceKind:
|
|
16213
|
-
sourcePosition:
|
|
16864
|
+
sourceKind: z26.enum(SerpPageSnapshotSourceKindValues).default("configured_target"),
|
|
16865
|
+
sourcePosition: z26.number().int().min(1).optional()
|
|
16214
16866
|
}).strict();
|
|
16215
|
-
var SerpIntelligencePageSnapshotsBodySchema =
|
|
16216
|
-
urls:
|
|
16217
|
-
targets:
|
|
16218
|
-
maxConcurrency:
|
|
16219
|
-
timeoutMs:
|
|
16220
|
-
debug:
|
|
16867
|
+
var SerpIntelligencePageSnapshotsBodySchema = z26.object({
|
|
16868
|
+
urls: z26.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25).optional(),
|
|
16869
|
+
targets: z26.array(SerpIntelligencePageSnapshotRequestSchema).min(1).max(25).optional(),
|
|
16870
|
+
maxConcurrency: z26.number().int().min(1).max(5).default(2),
|
|
16871
|
+
timeoutMs: z26.number().int().min(1e3).max(6e4).default(15e3),
|
|
16872
|
+
debug: z26.boolean().default(false)
|
|
16221
16873
|
}).strict().superRefine((body, ctx) => {
|
|
16222
16874
|
const hasUrls = Boolean(body.urls?.length);
|
|
16223
16875
|
const hasTargets = Boolean(body.targets?.length);
|
|
16224
16876
|
if (hasUrls === hasTargets) {
|
|
16225
16877
|
ctx.addIssue({
|
|
16226
|
-
code:
|
|
16878
|
+
code: z26.ZodIssueCode.custom,
|
|
16227
16879
|
message: "provide exactly one of urls or targets",
|
|
16228
16880
|
path: hasUrls ? ["targets"] : ["urls"]
|
|
16229
16881
|
});
|
|
16230
16882
|
}
|
|
16231
16883
|
});
|
|
16232
|
-
var SerpIntelligenceAICitationSchema =
|
|
16233
|
-
text:
|
|
16234
|
-
href:
|
|
16884
|
+
var SerpIntelligenceAICitationSchema = z26.object({
|
|
16885
|
+
text: z26.string(),
|
|
16886
|
+
href: z26.string()
|
|
16235
16887
|
}).strict();
|
|
16236
|
-
var SerpIntelligenceOrganicResultSchema =
|
|
16237
|
-
position:
|
|
16238
|
-
title:
|
|
16239
|
-
url:
|
|
16240
|
-
domain:
|
|
16241
|
-
cite:
|
|
16242
|
-
snippet:
|
|
16243
|
-
isRedditStyle:
|
|
16244
|
-
inlineRating:
|
|
16245
|
-
value:
|
|
16246
|
-
count:
|
|
16888
|
+
var SerpIntelligenceOrganicResultSchema = z26.object({
|
|
16889
|
+
position: z26.number().int().min(1),
|
|
16890
|
+
title: z26.string(),
|
|
16891
|
+
url: z26.string(),
|
|
16892
|
+
domain: z26.string(),
|
|
16893
|
+
cite: z26.string().nullable(),
|
|
16894
|
+
snippet: z26.string().nullable(),
|
|
16895
|
+
isRedditStyle: z26.boolean(),
|
|
16896
|
+
inlineRating: z26.object({
|
|
16897
|
+
value: z26.string(),
|
|
16898
|
+
count: z26.string()
|
|
16247
16899
|
}).strict().nullable()
|
|
16248
16900
|
}).strict();
|
|
16249
|
-
var SerpIntelligenceLocationEvidenceSchema =
|
|
16250
|
-
status:
|
|
16251
|
-
expected:
|
|
16252
|
-
city:
|
|
16253
|
-
regionCode:
|
|
16254
|
-
canonicalLocation:
|
|
16901
|
+
var SerpIntelligenceLocationEvidenceSchema = z26.object({
|
|
16902
|
+
status: z26.enum(SerpIntelligenceLocalizationStatusValues),
|
|
16903
|
+
expected: z26.object({
|
|
16904
|
+
city: z26.string(),
|
|
16905
|
+
regionCode: z26.string().nullable(),
|
|
16906
|
+
canonicalLocation: z26.string()
|
|
16255
16907
|
}).strict().nullable(),
|
|
16256
|
-
candidates:
|
|
16257
|
-
city:
|
|
16258
|
-
regionCode:
|
|
16259
|
-
count:
|
|
16260
|
-
examples:
|
|
16908
|
+
candidates: z26.array(z26.object({
|
|
16909
|
+
city: z26.string(),
|
|
16910
|
+
regionCode: z26.string(),
|
|
16911
|
+
count: z26.number().int().min(0),
|
|
16912
|
+
examples: z26.array(z26.string())
|
|
16261
16913
|
}).strict())
|
|
16262
16914
|
}).strict();
|
|
16263
|
-
var SerpIntelligenceHarvestResultSchema =
|
|
16264
|
-
seed:
|
|
16265
|
-
location:
|
|
16266
|
-
extractedAt:
|
|
16267
|
-
totalQuestions:
|
|
16268
|
-
surface:
|
|
16269
|
-
aiOverview:
|
|
16270
|
-
detected:
|
|
16271
|
-
text:
|
|
16272
|
-
citations:
|
|
16273
|
-
expanded:
|
|
16274
|
-
fullyExpanded:
|
|
16275
|
-
sections:
|
|
16915
|
+
var SerpIntelligenceHarvestResultSchema = z26.object({
|
|
16916
|
+
seed: z26.string(),
|
|
16917
|
+
location: z26.string().nullable(),
|
|
16918
|
+
extractedAt: z26.string(),
|
|
16919
|
+
totalQuestions: z26.number().int().min(0),
|
|
16920
|
+
surface: z26.enum(["web", "aim", "unknown"]),
|
|
16921
|
+
aiOverview: z26.object({
|
|
16922
|
+
detected: z26.boolean(),
|
|
16923
|
+
text: z26.string().nullable(),
|
|
16924
|
+
citations: z26.array(SerpIntelligenceAICitationSchema),
|
|
16925
|
+
expanded: z26.boolean().optional(),
|
|
16926
|
+
fullyExpanded: z26.boolean().optional(),
|
|
16927
|
+
sections: z26.array(z26.string()).optional()
|
|
16276
16928
|
}).strict(),
|
|
16277
|
-
aiMode:
|
|
16278
|
-
detected:
|
|
16279
|
-
text:
|
|
16280
|
-
citations:
|
|
16929
|
+
aiMode: z26.object({
|
|
16930
|
+
detected: z26.boolean(),
|
|
16931
|
+
text: z26.string().nullable(),
|
|
16932
|
+
citations: z26.array(SerpIntelligenceAICitationSchema)
|
|
16281
16933
|
}).strict(),
|
|
16282
|
-
tree:
|
|
16283
|
-
flat:
|
|
16284
|
-
videos:
|
|
16285
|
-
forums:
|
|
16286
|
-
organicResults:
|
|
16287
|
-
localPack:
|
|
16288
|
-
entityIds:
|
|
16289
|
-
entities:
|
|
16290
|
-
name:
|
|
16291
|
-
kgId:
|
|
16292
|
-
cid:
|
|
16293
|
-
gcid:
|
|
16934
|
+
tree: z26.array(z26.unknown()),
|
|
16935
|
+
flat: z26.array(z26.unknown()),
|
|
16936
|
+
videos: z26.array(z26.unknown()),
|
|
16937
|
+
forums: z26.array(z26.unknown()),
|
|
16938
|
+
organicResults: z26.array(SerpIntelligenceOrganicResultSchema),
|
|
16939
|
+
localPack: z26.array(z26.unknown()),
|
|
16940
|
+
entityIds: z26.object({
|
|
16941
|
+
entities: z26.array(z26.object({
|
|
16942
|
+
name: z26.string(),
|
|
16943
|
+
kgId: z26.string().nullable(),
|
|
16944
|
+
cid: z26.string().nullable(),
|
|
16945
|
+
gcid: z26.string().nullable()
|
|
16294
16946
|
}).strict()),
|
|
16295
|
-
kgIds:
|
|
16296
|
-
cids:
|
|
16297
|
-
gcids:
|
|
16947
|
+
kgIds: z26.array(z26.string()),
|
|
16948
|
+
cids: z26.array(z26.string()),
|
|
16949
|
+
gcids: z26.array(z26.string())
|
|
16298
16950
|
}).strict(),
|
|
16299
|
-
stats:
|
|
16300
|
-
seed:
|
|
16301
|
-
totalQuestions:
|
|
16302
|
-
maxDepthReached:
|
|
16303
|
-
durationMs:
|
|
16304
|
-
errorCount:
|
|
16951
|
+
stats: z26.object({
|
|
16952
|
+
seed: z26.string(),
|
|
16953
|
+
totalQuestions: z26.number().int().min(0),
|
|
16954
|
+
maxDepthReached: z26.number().int().min(0),
|
|
16955
|
+
durationMs: z26.number().min(0),
|
|
16956
|
+
errorCount: z26.number().int().min(0)
|
|
16305
16957
|
}).strict(),
|
|
16306
|
-
diagnostics:
|
|
16307
|
-
completionStatus:
|
|
16308
|
-
problem:
|
|
16309
|
-
warnings:
|
|
16310
|
-
debug:
|
|
16958
|
+
diagnostics: z26.object({
|
|
16959
|
+
completionStatus: z26.enum(["paa_found", "no_paa", "serp_only"]),
|
|
16960
|
+
problem: z26.null(),
|
|
16961
|
+
warnings: z26.array(z26.unknown()).optional(),
|
|
16962
|
+
debug: z26.object({
|
|
16311
16963
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.optional()
|
|
16312
16964
|
}).passthrough().optional()
|
|
16313
16965
|
}).passthrough(),
|
|
16314
|
-
whatPeopleSaying:
|
|
16966
|
+
whatPeopleSaying: z26.array(z26.unknown())
|
|
16315
16967
|
}).strict();
|
|
16316
|
-
var SerpIntelligenceCaptureAttemptSchema =
|
|
16317
|
-
attemptNumber:
|
|
16318
|
-
outcome:
|
|
16319
|
-
startedAt:
|
|
16320
|
-
completedAt:
|
|
16321
|
-
durationMs:
|
|
16322
|
-
problemCode:
|
|
16323
|
-
message:
|
|
16324
|
-
kernelSessionId:
|
|
16325
|
-
cleanupSucceeded:
|
|
16968
|
+
var SerpIntelligenceCaptureAttemptSchema = z26.object({
|
|
16969
|
+
attemptNumber: z26.number().int().min(1),
|
|
16970
|
+
outcome: z26.enum(SerpIntelligenceAttemptOutcomeValues),
|
|
16971
|
+
startedAt: z26.string().optional(),
|
|
16972
|
+
completedAt: z26.string().optional(),
|
|
16973
|
+
durationMs: z26.number().min(0).optional(),
|
|
16974
|
+
problemCode: z26.string().optional(),
|
|
16975
|
+
message: z26.string().optional(),
|
|
16976
|
+
kernelSessionId: z26.string().nullable().optional(),
|
|
16977
|
+
cleanupSucceeded: z26.boolean().nullable().optional()
|
|
16326
16978
|
}).strict();
|
|
16327
|
-
var SerpPageSnapshotCaptureSchema =
|
|
16979
|
+
var SerpPageSnapshotCaptureSchema = z26.object({
|
|
16328
16980
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
16329
16981
|
requestedUrl: SerpIntelligencePublicHttpUrlSchema,
|
|
16330
16982
|
finalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
16331
|
-
sourceKind:
|
|
16332
|
-
sourcePosition:
|
|
16333
|
-
status:
|
|
16334
|
-
fetchedVia:
|
|
16335
|
-
httpStatus:
|
|
16336
|
-
contentType:
|
|
16337
|
-
title:
|
|
16983
|
+
sourceKind: z26.enum(SerpPageSnapshotSourceKindValues),
|
|
16984
|
+
sourcePosition: z26.number().int().min(1).nullable(),
|
|
16985
|
+
status: z26.enum(SerpPageFetchStatusValues),
|
|
16986
|
+
fetchedVia: z26.enum(SerpPageFetchedViaValues).nullable(),
|
|
16987
|
+
httpStatus: z26.number().int().min(100).max(599).nullable(),
|
|
16988
|
+
contentType: z26.string().nullable(),
|
|
16989
|
+
title: z26.string().nullable(),
|
|
16338
16990
|
canonicalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
16339
|
-
metaDescription:
|
|
16340
|
-
headings:
|
|
16341
|
-
level:
|
|
16342
|
-
text:
|
|
16991
|
+
metaDescription: z26.string().nullable(),
|
|
16992
|
+
headings: z26.array(z26.object({
|
|
16993
|
+
level: z26.number().int().min(1).max(6),
|
|
16994
|
+
text: z26.string()
|
|
16343
16995
|
}).strict()).default([]),
|
|
16344
|
-
artifact:
|
|
16345
|
-
htmlBlobUrl:
|
|
16346
|
-
textBlobUrl:
|
|
16347
|
-
markdownBlobUrl:
|
|
16348
|
-
screenshotBlobUrl:
|
|
16349
|
-
contentSha256:
|
|
16350
|
-
capturedAt:
|
|
16996
|
+
artifact: z26.object({
|
|
16997
|
+
htmlBlobUrl: z26.string().url().nullable(),
|
|
16998
|
+
textBlobUrl: z26.string().url().nullable(),
|
|
16999
|
+
markdownBlobUrl: z26.string().url().nullable(),
|
|
17000
|
+
screenshotBlobUrl: z26.string().url().nullable(),
|
|
17001
|
+
contentSha256: z26.string().nullable(),
|
|
17002
|
+
capturedAt: z26.string().nullable()
|
|
16351
17003
|
}).strict(),
|
|
16352
|
-
error:
|
|
16353
|
-
code:
|
|
16354
|
-
message:
|
|
17004
|
+
error: z26.object({
|
|
17005
|
+
code: z26.string(),
|
|
17006
|
+
message: z26.string()
|
|
16355
17007
|
}).strict().nullable()
|
|
16356
17008
|
}).strict();
|
|
16357
|
-
var SerpIntelligenceCaptureResponseSchema =
|
|
17009
|
+
var SerpIntelligenceCaptureResponseSchema = z26.object({
|
|
16358
17010
|
harvestResult: SerpIntelligenceHarvestResultSchema,
|
|
16359
|
-
attempts:
|
|
17011
|
+
attempts: z26.array(SerpIntelligenceCaptureAttemptSchema),
|
|
16360
17012
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.nullable(),
|
|
16361
|
-
pageSnapshotArtifacts:
|
|
16362
|
-
billing:
|
|
16363
|
-
creditsUsed:
|
|
16364
|
-
requestId:
|
|
16365
|
-
jobId:
|
|
17013
|
+
pageSnapshotArtifacts: z26.array(SerpPageSnapshotCaptureSchema),
|
|
17014
|
+
billing: z26.object({
|
|
17015
|
+
creditsUsed: z26.number().min(0).optional(),
|
|
17016
|
+
requestId: z26.string().optional(),
|
|
17017
|
+
jobId: z26.string().optional()
|
|
16366
17018
|
}).strict().optional()
|
|
16367
17019
|
}).strict();
|
|
16368
|
-
var SerpIntelligencePageSnapshotsResponseSchema =
|
|
16369
|
-
pageSnapshotArtifacts:
|
|
16370
|
-
attempts:
|
|
17020
|
+
var SerpIntelligencePageSnapshotsResponseSchema = z26.object({
|
|
17021
|
+
pageSnapshotArtifacts: z26.array(SerpPageSnapshotCaptureSchema),
|
|
17022
|
+
attempts: z26.array(SerpIntelligenceCaptureAttemptSchema).default([])
|
|
16371
17023
|
}).strict();
|
|
16372
17024
|
|
|
16373
17025
|
// src/serp-intelligence/serp-capture-service.ts
|
|
@@ -16550,7 +17202,7 @@ var SERP_INTELLIGENCE_RATE_WINDOW_SECONDS = 60;
|
|
|
16550
17202
|
var POST_CAPTURE_ROUTE_LABEL = "POST /capture";
|
|
16551
17203
|
var POST_PAGE_SNAPSHOTS_ROUTE_LABEL = "POST /page-snapshots";
|
|
16552
17204
|
var SERP_CAPTURE_BILLING_SOURCE = "serp_intelligence_capture";
|
|
16553
|
-
var serpIntelligenceApp = new
|
|
17205
|
+
var serpIntelligenceApp = new Hono17();
|
|
16554
17206
|
serpIntelligenceApp.use("*", createApiKeyAuth());
|
|
16555
17207
|
function structuredError(input) {
|
|
16556
17208
|
return {
|
|
@@ -16826,7 +17478,7 @@ serpIntelligenceApp.post("/page-snapshots", async (c) => {
|
|
|
16826
17478
|
});
|
|
16827
17479
|
|
|
16828
17480
|
// src/mcp/mcp-routes.ts
|
|
16829
|
-
import { Hono as
|
|
17481
|
+
import { Hono as Hono18 } from "hono";
|
|
16830
17482
|
import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js";
|
|
16831
17483
|
|
|
16832
17484
|
// src/mcp/memory-mcp-tool-executor.ts
|
|
@@ -17477,7 +18129,7 @@ async function requireMcpCallerKey(c) {
|
|
|
17477
18129
|
if (!user) return mcpAuthError();
|
|
17478
18130
|
return { key: callerKey, user };
|
|
17479
18131
|
}
|
|
17480
|
-
var mcpApp = new
|
|
18132
|
+
var mcpApp = new Hono18();
|
|
17481
18133
|
mcpApp.all("/", async (c) => {
|
|
17482
18134
|
try {
|
|
17483
18135
|
const keyOrError = await requireMcpCallerKey(c);
|
|
@@ -17504,7 +18156,7 @@ mcpApp.all("/", async (c) => {
|
|
|
17504
18156
|
|
|
17505
18157
|
// src/api/browser-agent-routes.ts
|
|
17506
18158
|
import { randomUUID as randomUUID6 } from "crypto";
|
|
17507
|
-
import { Hono as
|
|
18159
|
+
import { Hono as Hono19 } from "hono";
|
|
17508
18160
|
|
|
17509
18161
|
// src/api/browser-agent-db.ts
|
|
17510
18162
|
import { randomUUID as randomUUID5 } from "crypto";
|
|
@@ -17705,6 +18357,57 @@ async function listAuthConnectionsByProfile(profile) {
|
|
|
17705
18357
|
});
|
|
17706
18358
|
return res.rows;
|
|
17707
18359
|
}
|
|
18360
|
+
async function listProfileSummaries(userId) {
|
|
18361
|
+
const db = getDb();
|
|
18362
|
+
const res = await db.execute({
|
|
18363
|
+
sql: `SELECT c.profile AS profile,
|
|
18364
|
+
l.label AS label,
|
|
18365
|
+
COUNT(DISTINCT c.domain) AS domains,
|
|
18366
|
+
MAX(COALESCE(c.updated_at, c.created_at)) AS last_used_at
|
|
18367
|
+
FROM browser_auth_connections c
|
|
18368
|
+
JOIN browser_agent_sessions s ON s.id = c.browser_agent_session_id
|
|
18369
|
+
LEFT JOIN browser_profile_labels l ON l.user_id = s.user_id AND l.profile = c.profile
|
|
18370
|
+
WHERE s.user_id = ? AND c.profile IS NOT NULL
|
|
18371
|
+
GROUP BY c.profile, l.label
|
|
18372
|
+
ORDER BY MAX(COALESCE(c.updated_at, c.created_at)) DESC`,
|
|
18373
|
+
args: [userId]
|
|
18374
|
+
});
|
|
18375
|
+
return res.rows.map((row) => ({
|
|
18376
|
+
profile: String(row.profile),
|
|
18377
|
+
label: row.label == null ? null : String(row.label),
|
|
18378
|
+
domains: Number(row.domains ?? 0),
|
|
18379
|
+
last_used_at: row.last_used_at == null ? null : String(row.last_used_at)
|
|
18380
|
+
}));
|
|
18381
|
+
}
|
|
18382
|
+
async function userHasProfile(userId, profile) {
|
|
18383
|
+
if (!profile) return false;
|
|
18384
|
+
const db = getDb();
|
|
18385
|
+
const res = await db.execute({
|
|
18386
|
+
sql: `SELECT 1 AS present
|
|
18387
|
+
FROM browser_auth_connections c
|
|
18388
|
+
JOIN browser_agent_sessions s ON s.id = c.browser_agent_session_id
|
|
18389
|
+
WHERE s.user_id = ? AND c.profile = ?
|
|
18390
|
+
LIMIT 1`,
|
|
18391
|
+
args: [userId, profile]
|
|
18392
|
+
});
|
|
18393
|
+
return res.rows.length > 0;
|
|
18394
|
+
}
|
|
18395
|
+
async function upsertProfileLabel(userId, profile, label) {
|
|
18396
|
+
const db = getDb();
|
|
18397
|
+
await db.execute({
|
|
18398
|
+
sql: `INSERT INTO browser_profile_labels (user_id, profile, label)
|
|
18399
|
+
VALUES (?, ?, ?)
|
|
18400
|
+
ON CONFLICT(user_id, profile) DO UPDATE SET label = excluded.label, updated_at = datetime('now')`,
|
|
18401
|
+
args: [userId, profile, label]
|
|
18402
|
+
});
|
|
18403
|
+
}
|
|
18404
|
+
async function deleteProfileLabel(userId, profile) {
|
|
18405
|
+
const db = getDb();
|
|
18406
|
+
await db.execute({
|
|
18407
|
+
sql: `DELETE FROM browser_profile_labels WHERE user_id = ? AND profile = ?`,
|
|
18408
|
+
args: [userId, profile]
|
|
18409
|
+
});
|
|
18410
|
+
}
|
|
17708
18411
|
async function createSessionRow(input) {
|
|
17709
18412
|
const db = getDb();
|
|
17710
18413
|
const id = `bas_${randomUUID5().replace(/-/g, "").slice(0, 20)}`;
|
|
@@ -19006,12 +19709,55 @@ async function loadOpenSession(id, userId) {
|
|
|
19006
19709
|
return row;
|
|
19007
19710
|
}
|
|
19008
19711
|
function buildBrowserAgentRoutes() {
|
|
19009
|
-
const app2 = new
|
|
19712
|
+
const app2 = new Hono19();
|
|
19010
19713
|
app2.use("*", async (c, next) => {
|
|
19011
19714
|
await migrateBrowserAgent();
|
|
19012
19715
|
return next();
|
|
19013
19716
|
});
|
|
19014
19717
|
app2.use("*", auth);
|
|
19718
|
+
app2.get("/profiles", async (c) => {
|
|
19719
|
+
const user = c.get("user");
|
|
19720
|
+
try {
|
|
19721
|
+
const rows = await listProfileSummaries(user.id);
|
|
19722
|
+
return c.json({
|
|
19723
|
+
ok: true,
|
|
19724
|
+
profiles: rows.map((row) => ({
|
|
19725
|
+
profile: row.profile,
|
|
19726
|
+
label: row.label,
|
|
19727
|
+
domains: row.domains,
|
|
19728
|
+
last_used_at: row.last_used_at
|
|
19729
|
+
}))
|
|
19730
|
+
});
|
|
19731
|
+
} catch (err) {
|
|
19732
|
+
return c.json(failure(err), 502);
|
|
19733
|
+
}
|
|
19734
|
+
});
|
|
19735
|
+
app2.patch("/profiles/label", async (c) => {
|
|
19736
|
+
const user = c.get("user");
|
|
19737
|
+
const body = await c.req.json().catch(() => ({}));
|
|
19738
|
+
const profileName = typeof body.profile === "string" ? body.profile.trim() : "";
|
|
19739
|
+
if (!profileName) return c.json({ error: "profile is required" }, 400);
|
|
19740
|
+
const rawLabel = body.label;
|
|
19741
|
+
if (rawLabel !== null && typeof rawLabel !== "string") {
|
|
19742
|
+
return c.json({ error: "label must be a string of 1-60 characters, or null to clear the display label" }, 400);
|
|
19743
|
+
}
|
|
19744
|
+
const label = typeof rawLabel === "string" ? rawLabel.trim() : null;
|
|
19745
|
+
if (rawLabel !== null && (!label || label.length > 60)) {
|
|
19746
|
+
return c.json({ error: "label must be a string of 1-60 characters, or null to clear the display label" }, 400);
|
|
19747
|
+
}
|
|
19748
|
+
try {
|
|
19749
|
+
const owned = await userHasProfile(user.id, profileName);
|
|
19750
|
+
if (!owned) return c.json({ error: "profile not found" }, 404);
|
|
19751
|
+
if (label == null) {
|
|
19752
|
+
await deleteProfileLabel(user.id, profileName);
|
|
19753
|
+
} else {
|
|
19754
|
+
await upsertProfileLabel(user.id, profileName, label);
|
|
19755
|
+
}
|
|
19756
|
+
return c.json({ ok: true, profile: profileName, profile_name_immutable: true, label });
|
|
19757
|
+
} catch (err) {
|
|
19758
|
+
return c.json(failure(err), 502);
|
|
19759
|
+
}
|
|
19760
|
+
});
|
|
19015
19761
|
app2.post("/profiles/onboard", async (c) => {
|
|
19016
19762
|
const user = c.get("user");
|
|
19017
19763
|
if (Number(user.balance_mc ?? 0) < BROWSER_OPEN_MIN_BALANCE_MC) {
|
|
@@ -19682,320 +20428,123 @@ async function selectSession(id) {
|
|
|
19682
20428
|
async function openSession() {
|
|
19683
20429
|
const r = await api('POST', '/sessions', { label: 'console', disable_default_proxy: true });
|
|
19684
20430
|
if (r.ok) { await refreshSessions(); selectSession(r.data.session_id); }
|
|
19685
|
-
else alert('Open failed: ' + JSON.stringify(r.data));
|
|
19686
|
-
}
|
|
19687
|
-
|
|
19688
|
-
async function closeCurrent() {
|
|
19689
|
-
if (!state.current) return;
|
|
19690
|
-
await api('DELETE', '/sessions/' + state.current);
|
|
19691
|
-
await refreshSessions();
|
|
19692
|
-
}
|
|
19693
|
-
|
|
19694
|
-
async function downloadReplay(replayId) {
|
|
19695
|
-
if (!state.current || !replayId) return;
|
|
19696
|
-
const res = await fetch('/agent/sessions/' + encodeURIComponent(state.current) + '/replays/' + encodeURIComponent(replayId) + '/download', {
|
|
19697
|
-
headers: { 'x-api-key': state.key },
|
|
19698
|
-
});
|
|
19699
|
-
if (!res.ok) {
|
|
19700
|
-
alert('Replay download failed: ' + await res.text());
|
|
19701
|
-
return;
|
|
19702
|
-
}
|
|
19703
|
-
const blob = await res.blob();
|
|
19704
|
-
const url = URL.createObjectURL(blob);
|
|
19705
|
-
const a = document.createElement('a');
|
|
19706
|
-
a.href = url;
|
|
19707
|
-
a.download = state.current + '-' + replayId + '.mp4';
|
|
19708
|
-
document.body.appendChild(a);
|
|
19709
|
-
a.click();
|
|
19710
|
-
a.remove();
|
|
19711
|
-
URL.revokeObjectURL(url);
|
|
19712
|
-
}
|
|
19713
|
-
|
|
19714
|
-
function frameSrc() {
|
|
19715
|
-
if (!state.liveUrl) return null;
|
|
19716
|
-
const sep = state.liveUrl.includes('?') ? '&' : '?';
|
|
19717
|
-
return state.readOnly ? state.liveUrl + sep + 'readOnly=true' : state.liveUrl;
|
|
19718
|
-
}
|
|
19719
|
-
|
|
19720
|
-
function saveKey(v) { state.key = v.trim(); localStorage.setItem(KEY_STORE, state.key); render(); if (state.key) { refreshSessions(); if (state.current) selectSession(state.current); } }
|
|
19721
|
-
|
|
19722
|
-
function h(html) { const t = document.createElement('template'); t.innerHTML = html.trim(); return t.content.firstChild; }
|
|
19723
|
-
function esc(s) { return String(s == null ? '' : s).replace(/[&<>"]/g, c => ({ '&': '&', '<': '<', '>': '>', '"': '"' }[c])); }
|
|
19724
|
-
|
|
19725
|
-
function render() {
|
|
19726
|
-
const app = document.getElementById('app');
|
|
19727
|
-
app.innerHTML = '';
|
|
19728
|
-
if (!state.key) {
|
|
19729
|
-
app.appendChild(h('<div class="gate"><h2>Browser Agent Console</h2><p>Paste your API key to watch and control browser sessions.</p><input id="k" type="password" placeholder="API key" /><button id="kb">Continue</button></div>'));
|
|
19730
|
-
document.getElementById('kb').onclick = () => saveKey(document.getElementById('k').value);
|
|
19731
|
-
document.getElementById('k').onkeydown = e => { if (e.key === 'Enter') saveKey(e.target.value); };
|
|
19732
|
-
return;
|
|
19733
|
-
}
|
|
19734
|
-
const header = h('<header><h1>Browser Agent</h1><span class="muted">live control + replays</span><span class="spacer"></span><button id="open">+ New Session</button><button class="ghost" id="logout">Forget key</button></header>');
|
|
19735
|
-
app.appendChild(header);
|
|
19736
|
-
document.getElementById('open').onclick = openSession;
|
|
19737
|
-
document.getElementById('logout').onclick = () => saveKey('');
|
|
19738
|
-
|
|
19739
|
-
const layout = h('<div class="layout"></div>');
|
|
19740
|
-
const aside = h('<aside><h2>Sessions</h2></aside>');
|
|
19741
|
-
if (!state.sessions.length) aside.appendChild(h('<div class="muted" style="padding:4px">No sessions yet.</div>'));
|
|
19742
|
-
for (const s of state.sessions) {
|
|
19743
|
-
const el = h('<div class="sess ' + (s.session_id === state.current ? 'active' : '') + '"><div class="id">' + esc(s.session_id) + '</div><div class="meta"><span class="dot ' + esc(s.status) + '"></span>' + esc(s.status) + (s.label ? ' \xB7 ' + esc(s.label) : '') + '</div></div>');
|
|
19744
|
-
el.onclick = () => selectSession(s.session_id);
|
|
19745
|
-
aside.appendChild(el);
|
|
19746
|
-
}
|
|
19747
|
-
layout.appendChild(aside);
|
|
19748
|
-
|
|
19749
|
-
const main = h('<main></main>');
|
|
19750
|
-
if (!state.current) {
|
|
19751
|
-
main.appendChild(h('<div class="empty"><div>Select or open a session to watch.</div></div>'));
|
|
19752
|
-
} else {
|
|
19753
|
-
const tb = h('<div class="toolbar"><label><input type="checkbox" id="ro" ' + (state.readOnly ? 'checked' : '') + ' /> Read-only (uncheck to take control)</label><span class="spacer"></span><button class="ghost" id="reload">Reload view</button><button class="ghost" id="close">Close session</button></div>');
|
|
19754
|
-
main.appendChild(tb);
|
|
19755
|
-
const stage = h('<div class="stage"></div>');
|
|
19756
|
-
const src = frameSrc();
|
|
19757
|
-
if (src) {
|
|
19758
|
-
const f = h('<iframe allow="autoplay; clipboard-read; clipboard-write" src="' + esc(src) + '"></iframe>');
|
|
19759
|
-
stage.appendChild(f);
|
|
19760
|
-
} else {
|
|
19761
|
-
stage.appendChild(h('<div class="empty"><div>Live view unavailable for this session.</div><div class="muted">It may be closed or still starting.</div></div>'));
|
|
19762
|
-
}
|
|
19763
|
-
main.appendChild(stage);
|
|
19764
|
-
|
|
19765
|
-
const rep = h('<div class="replays"><h3>Replays</h3></div>');
|
|
19766
|
-
if (!state.replays.length) rep.appendChild(h('<div class="muted">No replays recorded.</div>'));
|
|
19767
|
-
for (const r of state.replays) {
|
|
19768
|
-
const status = r.stopped_at ? 'ready' : 'recording...';
|
|
19769
|
-
const link = r.view_url ? '<a href="' + esc(r.view_url) + '" target="_blank" rel="noopener">view mp4</a>' : '<span class="muted">' + status + '</span>';
|
|
19770
|
-
const download = r.stopped_at ? '<button class="linkish replay-download" data-rid="' + esc(r.replay_id) + '">download mp4</button>' : '';
|
|
19771
|
-
rep.appendChild(h('<div class="replay"><span class="muted">' + esc(r.started_at || '') + '</span><span class="spacer"></span>' + link + download + '</div>'));
|
|
19772
|
-
}
|
|
19773
|
-
main.appendChild(rep);
|
|
19774
|
-
|
|
19775
|
-
layout.appendChild(main);
|
|
19776
|
-
}
|
|
19777
|
-
app.appendChild(layout);
|
|
19778
|
-
|
|
19779
|
-
const ro = document.getElementById('ro');
|
|
19780
|
-
if (ro) ro.onchange = e => { state.readOnly = e.target.checked; render(); };
|
|
19781
|
-
const reload = document.getElementById('reload');
|
|
19782
|
-
if (reload) reload.onclick = () => selectSession(state.current);
|
|
19783
|
-
const close = document.getElementById('close');
|
|
19784
|
-
if (close) close.onclick = closeCurrent;
|
|
19785
|
-
document.querySelectorAll('.replay-download').forEach(btn => {
|
|
19786
|
-
btn.onclick = () => downloadReplay(btn.getAttribute('data-rid'));
|
|
19787
|
-
});
|
|
19788
|
-
}
|
|
19789
|
-
|
|
19790
|
-
render();
|
|
19791
|
-
if (state.key) { refreshSessions(); if (state.current) selectSession(state.current); }
|
|
19792
|
-
</script>
|
|
19793
|
-
</body>
|
|
19794
|
-
</html>`;
|
|
19795
|
-
}
|
|
19796
|
-
|
|
19797
|
-
// src/api/stripe-routes.ts
|
|
19798
|
-
import Stripe from "stripe";
|
|
19799
|
-
import { Hono as Hono19 } from "hono";
|
|
19800
|
-
|
|
19801
|
-
// src/api/connected-account-billing.ts
|
|
19802
|
-
var DEFAULT_CONNECTED_ACCOUNT_PRICE_ID = "price_1TtYPAS8aAcsk3TGHOgrDZiQ";
|
|
19803
|
-
async function ensureConnectedAccountBillingSchema() {
|
|
19804
|
-
await getDb().execute(`CREATE TABLE IF NOT EXISTS connected_account_billing (
|
|
19805
|
-
user_id INTEGER PRIMARY KEY,
|
|
19806
|
-
stripe_subscription_id TEXT,
|
|
19807
|
-
stripe_subscription_item_id TEXT,
|
|
19808
|
-
price_id TEXT,
|
|
19809
|
-
quantity INTEGER NOT NULL DEFAULT 0,
|
|
19810
|
-
status TEXT NOT NULL DEFAULT 'pending',
|
|
19811
|
-
last_error_code TEXT,
|
|
19812
|
-
synced_at TEXT,
|
|
19813
|
-
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
19814
|
-
)`);
|
|
19815
|
-
}
|
|
19816
|
-
async function getConnectedAccountBillingState(userId) {
|
|
19817
|
-
await ensureConnectedAccountBillingSchema();
|
|
19818
|
-
const result = await getDb().execute({
|
|
19819
|
-
sql: "SELECT * FROM connected_account_billing WHERE user_id = ? LIMIT 1",
|
|
19820
|
-
args: [Number(userId)]
|
|
19821
|
-
});
|
|
19822
|
-
return result.rows[0] ? result.rows[0] : null;
|
|
19823
|
-
}
|
|
19824
|
-
async function setConnectedAccountBillingState(input) {
|
|
19825
|
-
await ensureConnectedAccountBillingSchema();
|
|
19826
|
-
await getDb().execute({
|
|
19827
|
-
sql: `INSERT INTO connected_account_billing
|
|
19828
|
-
(user_id, stripe_subscription_id, stripe_subscription_item_id, price_id, quantity, status, last_error_code, synced_at, updated_at)
|
|
19829
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, CASE WHEN ? THEN datetime('now') ELSE NULL END, datetime('now'))
|
|
19830
|
-
ON CONFLICT(user_id) DO UPDATE SET
|
|
19831
|
-
stripe_subscription_id = excluded.stripe_subscription_id,
|
|
19832
|
-
stripe_subscription_item_id = excluded.stripe_subscription_item_id,
|
|
19833
|
-
price_id = excluded.price_id,
|
|
19834
|
-
quantity = excluded.quantity,
|
|
19835
|
-
status = excluded.status,
|
|
19836
|
-
last_error_code = excluded.last_error_code,
|
|
19837
|
-
synced_at = CASE WHEN ? THEN datetime('now') ELSE connected_account_billing.synced_at END,
|
|
19838
|
-
updated_at = datetime('now')`,
|
|
19839
|
-
args: [
|
|
19840
|
-
Number(input.userId),
|
|
19841
|
-
input.stripeSubscriptionId,
|
|
19842
|
-
input.stripeSubscriptionItemId,
|
|
19843
|
-
input.priceId,
|
|
19844
|
-
input.quantity,
|
|
19845
|
-
input.status,
|
|
19846
|
-
input.lastErrorCode,
|
|
19847
|
-
input.synced ? 1 : 0,
|
|
19848
|
-
input.synced ? 1 : 0
|
|
19849
|
-
]
|
|
19850
|
-
});
|
|
19851
|
-
const state = await getConnectedAccountBillingState(input.userId);
|
|
19852
|
-
if (!state) throw new Error("connected account billing state was not persisted");
|
|
19853
|
-
return state;
|
|
19854
|
-
}
|
|
19855
|
-
var ConnectedAccountBillingError = class extends Error {
|
|
19856
|
-
constructor(message, code, status = 409, details) {
|
|
19857
|
-
super(message);
|
|
19858
|
-
this.code = code;
|
|
19859
|
-
this.status = status;
|
|
19860
|
-
this.details = details;
|
|
19861
|
-
this.name = "ConnectedAccountBillingError";
|
|
19862
|
-
}
|
|
19863
|
-
code;
|
|
19864
|
-
status;
|
|
19865
|
-
details;
|
|
19866
|
-
};
|
|
19867
|
-
function connectedAccountPriceId() {
|
|
19868
|
-
const configured = process.env.CONNECTED_ACCOUNT_PRICE_ID?.trim();
|
|
19869
|
-
if (configured) return configured;
|
|
19870
|
-
return DEFAULT_CONNECTED_ACCOUNT_PRICE_ID || null;
|
|
19871
|
-
}
|
|
19872
|
-
function connectedAccountBillingView(actualQuantity, state) {
|
|
19873
|
-
const quantity = Math.max(0, Math.round(actualQuantity));
|
|
19874
|
-
return {
|
|
19875
|
-
provider: "nango",
|
|
19876
|
-
billingMode: "flat_recurring_credits",
|
|
19877
|
-
priceId: connectedAccountPriceId(),
|
|
19878
|
-
unitAmountCredits: CONNECTED_ACTIVE_CONNECTION_MONTHLY_CREDITS,
|
|
19879
|
-
interval: "month",
|
|
19880
|
-
quantity,
|
|
19881
|
-
projectedMonthlyCredits: quantity * CONNECTED_ACTIVE_CONNECTION_MONTHLY_CREDITS,
|
|
19882
|
-
status: state?.status ?? "pending",
|
|
19883
|
-
lastErrorCode: state?.last_error_code ?? null,
|
|
19884
|
-
syncedAt: state?.synced_at ?? null,
|
|
19885
|
-
usage: CONNECTED_USAGE_RATE_POLICY.usage
|
|
19886
|
-
};
|
|
19887
|
-
}
|
|
19888
|
-
function findBasePlanItem(subscription) {
|
|
19889
|
-
return subscription.items.data.find((item) => !!item.price?.id && item.price.id in SUBSCRIPTION_TIERS);
|
|
19890
|
-
}
|
|
19891
|
-
function safeErrorCode(error) {
|
|
19892
|
-
if (error instanceof ConnectedAccountBillingError) return error.code;
|
|
19893
|
-
return "stripe_connection_billing_failed";
|
|
20431
|
+
else alert('Open failed: ' + JSON.stringify(r.data));
|
|
19894
20432
|
}
|
|
19895
|
-
|
|
19896
|
-
|
|
19897
|
-
|
|
19898
|
-
|
|
19899
|
-
|
|
19900
|
-
|
|
19901
|
-
|
|
19902
|
-
|
|
19903
|
-
|
|
19904
|
-
|
|
20433
|
+
|
|
20434
|
+
async function closeCurrent() {
|
|
20435
|
+
if (!state.current) return;
|
|
20436
|
+
await api('DELETE', '/sessions/' + state.current);
|
|
20437
|
+
await refreshSessions();
|
|
20438
|
+
}
|
|
20439
|
+
|
|
20440
|
+
async function downloadReplay(replayId) {
|
|
20441
|
+
if (!state.current || !replayId) return;
|
|
20442
|
+
const res = await fetch('/agent/sessions/' + encodeURIComponent(state.current) + '/replays/' + encodeURIComponent(replayId) + '/download', {
|
|
20443
|
+
headers: { 'x-api-key': state.key },
|
|
19905
20444
|
});
|
|
20445
|
+
if (!res.ok) {
|
|
20446
|
+
alert('Replay download failed: ' + await res.text());
|
|
20447
|
+
return;
|
|
20448
|
+
}
|
|
20449
|
+
const blob = await res.blob();
|
|
20450
|
+
const url = URL.createObjectURL(blob);
|
|
20451
|
+
const a = document.createElement('a');
|
|
20452
|
+
a.href = url;
|
|
20453
|
+
a.download = state.current + '-' + replayId + '.mp4';
|
|
20454
|
+
document.body.appendChild(a);
|
|
20455
|
+
a.click();
|
|
20456
|
+
a.remove();
|
|
20457
|
+
URL.revokeObjectURL(url);
|
|
19906
20458
|
}
|
|
19907
|
-
|
|
19908
|
-
|
|
19909
|
-
|
|
20459
|
+
|
|
20460
|
+
function frameSrc() {
|
|
20461
|
+
if (!state.liveUrl) return null;
|
|
20462
|
+
const sep = state.liveUrl.includes('?') ? '&' : '?';
|
|
20463
|
+
return state.readOnly ? state.liveUrl + sep + 'readOnly=true' : state.liveUrl;
|
|
20464
|
+
}
|
|
20465
|
+
|
|
20466
|
+
function saveKey(v) { state.key = v.trim(); localStorage.setItem(KEY_STORE, state.key); render(); if (state.key) { refreshSessions(); if (state.current) selectSession(state.current); } }
|
|
20467
|
+
|
|
20468
|
+
function h(html) { const t = document.createElement('template'); t.innerHTML = html.trim(); return t.content.firstChild; }
|
|
20469
|
+
function esc(s) { return String(s == null ? '' : s).replace(/[&<>"]/g, c => ({ '&': '&', '<': '<', '>': '>', '"': '"' }[c])); }
|
|
20470
|
+
|
|
20471
|
+
function render() {
|
|
20472
|
+
const app = document.getElementById('app');
|
|
20473
|
+
app.innerHTML = '';
|
|
20474
|
+
if (!state.key) {
|
|
20475
|
+
app.appendChild(h('<div class="gate"><h2>Browser Agent Console</h2><p>Paste your API key to watch and control browser sessions.</p><input id="k" type="password" placeholder="API key" /><button id="kb">Continue</button></div>'));
|
|
20476
|
+
document.getElementById('kb').onclick = () => saveKey(document.getElementById('k').value);
|
|
20477
|
+
document.getElementById('k').onkeydown = e => { if (e.key === 'Enter') saveKey(e.target.value); };
|
|
20478
|
+
return;
|
|
19910
20479
|
}
|
|
19911
|
-
const
|
|
19912
|
-
|
|
19913
|
-
|
|
19914
|
-
|
|
19915
|
-
|
|
19916
|
-
|
|
19917
|
-
|
|
19918
|
-
|
|
19919
|
-
|
|
20480
|
+
const header = h('<header><h1>Browser Agent</h1><span class="muted">live control + replays</span><span class="spacer"></span><button id="open">+ New Session</button><button class="ghost" id="logout">Forget key</button></header>');
|
|
20481
|
+
app.appendChild(header);
|
|
20482
|
+
document.getElementById('open').onclick = openSession;
|
|
20483
|
+
document.getElementById('logout').onclick = () => saveKey('');
|
|
20484
|
+
|
|
20485
|
+
const layout = h('<div class="layout"></div>');
|
|
20486
|
+
const aside = h('<aside><h2>Sessions</h2></aside>');
|
|
20487
|
+
if (!state.sessions.length) aside.appendChild(h('<div class="muted" style="padding:4px">No sessions yet.</div>'));
|
|
20488
|
+
for (const s of state.sessions) {
|
|
20489
|
+
const el = h('<div class="sess ' + (s.session_id === state.current ? 'active' : '') + '"><div class="id">' + esc(s.session_id) + '</div><div class="meta"><span class="dot ' + esc(s.status) + '"></span>' + esc(s.status) + (s.label ? ' \xB7 ' + esc(s.label) : '') + '</div></div>');
|
|
20490
|
+
el.onclick = () => selectSession(s.session_id);
|
|
20491
|
+
aside.appendChild(el);
|
|
19920
20492
|
}
|
|
19921
|
-
|
|
19922
|
-
|
|
19923
|
-
|
|
19924
|
-
|
|
19925
|
-
|
|
19926
|
-
|
|
19927
|
-
|
|
19928
|
-
|
|
19929
|
-
|
|
19930
|
-
|
|
19931
|
-
|
|
19932
|
-
|
|
19933
|
-
|
|
20493
|
+
layout.appendChild(aside);
|
|
20494
|
+
|
|
20495
|
+
const main = h('<main></main>');
|
|
20496
|
+
if (!state.current) {
|
|
20497
|
+
main.appendChild(h('<div class="empty"><div>Select or open a session to watch.</div></div>'));
|
|
20498
|
+
} else {
|
|
20499
|
+
const tb = h('<div class="toolbar"><label><input type="checkbox" id="ro" ' + (state.readOnly ? 'checked' : '') + ' /> Read-only (uncheck to take control)</label><span class="spacer"></span><button class="ghost" id="reload">Reload view</button><button class="ghost" id="close">Close session</button></div>');
|
|
20500
|
+
main.appendChild(tb);
|
|
20501
|
+
const stage = h('<div class="stage"></div>');
|
|
20502
|
+
const src = frameSrc();
|
|
20503
|
+
if (src) {
|
|
20504
|
+
const f = h('<iframe allow="autoplay; clipboard-read; clipboard-write" src="' + esc(src) + '"></iframe>');
|
|
20505
|
+
stage.appendChild(f);
|
|
20506
|
+
} else {
|
|
20507
|
+
stage.appendChild(h('<div class="empty"><div>Live view unavailable for this session.</div><div class="muted">It may be closed or still starting.</div></div>'));
|
|
19934
20508
|
}
|
|
19935
|
-
|
|
19936
|
-
|
|
19937
|
-
|
|
19938
|
-
|
|
19939
|
-
)
|
|
19940
|
-
|
|
19941
|
-
|
|
19942
|
-
|
|
19943
|
-
|
|
19944
|
-
const currentQuantity = Math.max(0, existingState?.quantity ?? 0);
|
|
19945
|
-
try {
|
|
19946
|
-
if (actualQuantity > currentQuantity) {
|
|
19947
|
-
const newConnections = actualQuantity - currentQuantity;
|
|
19948
|
-
const feeMc = newConnections * CONNECTED_ACTIVE_CONNECTION_MONTHLY_MC;
|
|
19949
|
-
const debit = await debitMc(
|
|
19950
|
-
user.id,
|
|
19951
|
-
feeMc,
|
|
19952
|
-
LedgerOperation.CONNECTED_ACCOUNT_FEE,
|
|
19953
|
-
`${newConnections} connected account${newConnections === 1 ? "" : "s"} at ${CONNECTED_ACTIVE_CONNECTION_MONTHLY_CREDITS} credits/month`
|
|
19954
|
-
);
|
|
19955
|
-
if (!debit.ok) {
|
|
19956
|
-
const requiredMc = feeMc;
|
|
19957
|
-
const error = new ConnectedAccountBillingError(
|
|
19958
|
-
`Insufficient credits to connect. Balance: ${mcToCredits(debit.balance_mc)} credits. Connecting requires ${mcToCredits(requiredMc)} credits (${mcToCredits(requiredMc - debit.balance_mc)} short). Upgrade your plan or wait for your credits to refresh.`,
|
|
19959
|
-
"insufficient_credits",
|
|
19960
|
-
402,
|
|
19961
|
-
{
|
|
19962
|
-
balanceCredits: mcToCredits(debit.balance_mc),
|
|
19963
|
-
requiredCredits: mcToCredits(requiredMc),
|
|
19964
|
-
shortByCredits: mcToCredits(Math.max(0, requiredMc - debit.balance_mc))
|
|
19965
|
-
}
|
|
19966
|
-
);
|
|
19967
|
-
await persistFailure(user, currentQuantity, error);
|
|
19968
|
-
throw error;
|
|
19969
|
-
}
|
|
20509
|
+
main.appendChild(stage);
|
|
20510
|
+
|
|
20511
|
+
const rep = h('<div class="replays"><h3>Replays</h3></div>');
|
|
20512
|
+
if (!state.replays.length) rep.appendChild(h('<div class="muted">No replays recorded.</div>'));
|
|
20513
|
+
for (const r of state.replays) {
|
|
20514
|
+
const status = r.stopped_at ? 'ready' : 'recording...';
|
|
20515
|
+
const link = r.view_url ? '<a href="' + esc(r.view_url) + '" target="_blank" rel="noopener">view mp4</a>' : '<span class="muted">' + status + '</span>';
|
|
20516
|
+
const download = r.stopped_at ? '<button class="linkish replay-download" data-rid="' + esc(r.replay_id) + '">download mp4</button>' : '';
|
|
20517
|
+
rep.appendChild(h('<div class="replay"><span class="muted">' + esc(r.started_at || '') + '</span><span class="spacer"></span>' + link + download + '</div>'));
|
|
19970
20518
|
}
|
|
19971
|
-
|
|
19972
|
-
|
|
19973
|
-
|
|
19974
|
-
stripeSubscriptionItemId: null,
|
|
19975
|
-
priceId,
|
|
19976
|
-
quantity: actualQuantity,
|
|
19977
|
-
status: "synced",
|
|
19978
|
-
lastErrorCode: null,
|
|
19979
|
-
synced: true
|
|
19980
|
-
});
|
|
19981
|
-
return connectedAccountBillingView(actualQuantity, state);
|
|
19982
|
-
} catch (error) {
|
|
19983
|
-
if (error instanceof ConnectedAccountBillingError) throw error;
|
|
19984
|
-
await persistFailure(user, currentQuantity, error);
|
|
19985
|
-
throw new ConnectedAccountBillingError(
|
|
19986
|
-
"Unable to update connected-account billing.",
|
|
19987
|
-
"stripe_connection_billing_failed",
|
|
19988
|
-
503
|
|
19989
|
-
);
|
|
20519
|
+
main.appendChild(rep);
|
|
20520
|
+
|
|
20521
|
+
layout.appendChild(main);
|
|
19990
20522
|
}
|
|
20523
|
+
app.appendChild(layout);
|
|
20524
|
+
|
|
20525
|
+
const ro = document.getElementById('ro');
|
|
20526
|
+
if (ro) ro.onchange = e => { state.readOnly = e.target.checked; render(); };
|
|
20527
|
+
const reload = document.getElementById('reload');
|
|
20528
|
+
if (reload) reload.onclick = () => selectSession(state.current);
|
|
20529
|
+
const close = document.getElementById('close');
|
|
20530
|
+
if (close) close.onclick = closeCurrent;
|
|
20531
|
+
document.querySelectorAll('.replay-download').forEach(btn => {
|
|
20532
|
+
btn.onclick = () => downloadReplay(btn.getAttribute('data-rid'));
|
|
20533
|
+
});
|
|
19991
20534
|
}
|
|
19992
|
-
|
|
19993
|
-
|
|
20535
|
+
|
|
20536
|
+
render();
|
|
20537
|
+
if (state.key) { refreshSessions(); if (state.current) selectSession(state.current); }
|
|
20538
|
+
</script>
|
|
20539
|
+
</body>
|
|
20540
|
+
</html>`;
|
|
19994
20541
|
}
|
|
19995
20542
|
|
|
19996
20543
|
// src/api/stripe-routes.ts
|
|
20544
|
+
import Stripe from "stripe";
|
|
20545
|
+
import { Hono as Hono20 } from "hono";
|
|
19997
20546
|
var stripe = new Stripe(process.env.STRIPE_SECRET_KEY, { apiVersion: "2026-02-25.clover" });
|
|
19998
|
-
var stripeApp = new
|
|
20547
|
+
var stripeApp = new Hono20();
|
|
19999
20548
|
stripeApp.post("/webhooks", async (c) => {
|
|
20000
20549
|
const sig = c.req.header("stripe-signature");
|
|
20001
20550
|
const body = await c.req.text();
|
|
@@ -20026,9 +20575,16 @@ stripeApp.post("/webhooks", async (c) => {
|
|
|
20026
20575
|
await creditMc(user.id, tier.credits_mc, LedgerOperation.SUBSCRIPTION, `${tier.label} subscription credits`, invoice.id, { neverExpires: tier.credits_never_expire });
|
|
20027
20576
|
await setSubscriptionTier(user.id, tier.tier, tier.concurrency, subId ?? user.subscription_id);
|
|
20028
20577
|
if (tier.includes_memory) {
|
|
20029
|
-
const
|
|
20030
|
-
|
|
20031
|
-
|
|
20578
|
+
const standaloneMemorySub = user.memory_subscription_id;
|
|
20579
|
+
if (standaloneMemorySub) {
|
|
20580
|
+
try {
|
|
20581
|
+
await stripe.subscriptions.update(standaloneMemorySub, { cancel_at_period_end: true });
|
|
20582
|
+
} catch (err) {
|
|
20583
|
+
console.error("[stripe] could not wind down standalone memory subscription", standaloneMemorySub, err instanceof Error ? err.message : err);
|
|
20584
|
+
}
|
|
20585
|
+
}
|
|
20586
|
+
await setMemoryPlan(user.id, "pro", null);
|
|
20587
|
+
await syncMemoryKeyPlan(user, resolveEffectiveMemoryPlan({ ...user, memory_plan: "pro", subscription_tier: tier.tier }));
|
|
20032
20588
|
}
|
|
20033
20589
|
const credentials = await syncScheduledActionCredentials(user);
|
|
20034
20590
|
if (!credentials.ok) console.warn("[stripe] scheduled-action credential sync failed:", credentials.error ?? "unknown error");
|
|
@@ -20139,7 +20695,7 @@ async function resolveUser2(customerId, emailFallback) {
|
|
|
20139
20695
|
}
|
|
20140
20696
|
|
|
20141
20697
|
// src/api/oauth-routes.ts
|
|
20142
|
-
import { Hono as
|
|
20698
|
+
import { Hono as Hono21 } from "hono";
|
|
20143
20699
|
import { getCookie, setCookie } from "hono/cookie";
|
|
20144
20700
|
import { createHash as createHash6, randomBytes as randomBytes2, randomUUID as randomUUID7 } from "crypto";
|
|
20145
20701
|
import { importPKCS8, exportJWK, SignJWT } from "jose";
|
|
@@ -20308,7 +20864,7 @@ function redirectWithError(redirectUri, state, error) {
|
|
|
20308
20864
|
if (state) u.searchParams.set("state", state);
|
|
20309
20865
|
return new Response(null, { status: 302, headers: { Location: u.toString() } });
|
|
20310
20866
|
}
|
|
20311
|
-
var oauthApp = new
|
|
20867
|
+
var oauthApp = new Hono21();
|
|
20312
20868
|
oauthApp.use("*", async (c, next) => {
|
|
20313
20869
|
c.header("Access-Control-Allow-Origin", "*");
|
|
20314
20870
|
c.header("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
|
|
@@ -20562,8 +21118,8 @@ oauthApp.post("/oauth/token", async (c) => {
|
|
|
20562
21118
|
});
|
|
20563
21119
|
|
|
20564
21120
|
// src/api/chat-routes.ts
|
|
20565
|
-
import { Hono as
|
|
20566
|
-
var chatApp = new
|
|
21121
|
+
import { Hono as Hono22 } from "hono";
|
|
21122
|
+
var chatApp = new Hono22();
|
|
20567
21123
|
function tokenOk(token) {
|
|
20568
21124
|
return !!token && token.startsWith("mk_") && token.length > 10;
|
|
20569
21125
|
}
|
|
@@ -21233,7 +21789,7 @@ loadChannels();
|
|
|
21233
21789
|
}
|
|
21234
21790
|
|
|
21235
21791
|
// src/api/vault-routes.ts
|
|
21236
|
-
import { Hono as
|
|
21792
|
+
import { Hono as Hono23 } from "hono";
|
|
21237
21793
|
var VAULTS = ["People", "Organizations", "Deals", "Projects", "Tasks", "Communications", "Calendar"];
|
|
21238
21794
|
var PROJECT_TYPES = ["codebase", "personal", "work", "client"];
|
|
21239
21795
|
var PROJECT_KIND_BY_TYPE = {
|
|
@@ -21252,7 +21808,7 @@ var PROJECT_TYPE_BY_KIND = {
|
|
|
21252
21808
|
var MAX_VAULT_RECORDS = 200;
|
|
21253
21809
|
var MAX_MEMORY_READ_CONCURRENCY = 4;
|
|
21254
21810
|
var MAX_MEMORY_READ_ATTEMPTS = 3;
|
|
21255
|
-
var vaultApp = new
|
|
21811
|
+
var vaultApp = new Hono23();
|
|
21256
21812
|
function validToken(token) {
|
|
21257
21813
|
return !!token && token.startsWith("mk_") && token.length > 20 && token.length < 256;
|
|
21258
21814
|
}
|
|
@@ -21388,7 +21944,7 @@ async function readMemory(tool, args, token) {
|
|
|
21388
21944
|
}
|
|
21389
21945
|
return result;
|
|
21390
21946
|
}
|
|
21391
|
-
async function
|
|
21947
|
+
async function mapWithConcurrency2(items, limit, map) {
|
|
21392
21948
|
const values = new Array(items.length);
|
|
21393
21949
|
let cursor = 0;
|
|
21394
21950
|
const worker = async () => {
|
|
@@ -21405,7 +21961,7 @@ async function readVault(vault, token) {
|
|
|
21405
21961
|
const list = await readMemory("listTool", { vault }, token);
|
|
21406
21962
|
if (!list.ok) throw new Error(list.error ?? `unable to read ${vault}`);
|
|
21407
21963
|
const noteRefs = (list.notes ?? []).slice(0, MAX_VAULT_RECORDS);
|
|
21408
|
-
const notes = await
|
|
21964
|
+
const notes = await mapWithConcurrency2(
|
|
21409
21965
|
noteRefs,
|
|
21410
21966
|
MAX_MEMORY_READ_CONCURRENCY,
|
|
21411
21967
|
({ path: path5 }) => readMemory("getTool", { vault, path: path5 }, token)
|
|
@@ -21601,8 +22157,8 @@ vaultApp.post("/api/activity", async (c) => {
|
|
|
21601
22157
|
});
|
|
21602
22158
|
|
|
21603
22159
|
// src/api/schedule-routes.ts
|
|
21604
|
-
import { Hono as
|
|
21605
|
-
var scheduleApp = new
|
|
22160
|
+
import { Hono as Hono24 } from "hono";
|
|
22161
|
+
var scheduleApp = new Hono24();
|
|
21606
22162
|
function tokenOk2(token) {
|
|
21607
22163
|
return !!token && token.startsWith("mk_") && token.length > 10;
|
|
21608
22164
|
}
|
|
@@ -21933,7 +22489,7 @@ loadActions();
|
|
|
21933
22489
|
}
|
|
21934
22490
|
|
|
21935
22491
|
// src/api/resend-inbound-routes.ts
|
|
21936
|
-
import { Hono as
|
|
22492
|
+
import { Hono as Hono25 } from "hono";
|
|
21937
22493
|
import { Resend } from "resend";
|
|
21938
22494
|
|
|
21939
22495
|
// src/api/resend-support-thread.ts
|
|
@@ -22564,7 +23120,7 @@ async function relayOperatorReply(resend, email, customerEmail, threadAddress, n
|
|
|
22564
23120
|
if (sent.error) throw new Error(`support reply relay failed: ${sent.error.message}`);
|
|
22565
23121
|
return sent.data.id;
|
|
22566
23122
|
}
|
|
22567
|
-
var resendInboundApp = new
|
|
23123
|
+
var resendInboundApp = new Hono25();
|
|
22568
23124
|
resendInboundApp.post("/webhooks", async (c) => {
|
|
22569
23125
|
const apiKey = process.env.RESEND_API_KEY?.trim();
|
|
22570
23126
|
const webhookSecret = process.env.RESEND_WEBHOOK_SECRET?.trim();
|
|
@@ -26005,7 +26561,7 @@ var requirePaidSchedulingTier = createMiddleware3(async (c, next) => {
|
|
|
26005
26561
|
if (!hasPaidSchedulingAccess(user)) return c.json(paidPlanRequiredPayload(), 403);
|
|
26006
26562
|
return next();
|
|
26007
26563
|
});
|
|
26008
|
-
var app = new
|
|
26564
|
+
var app = new Hono26();
|
|
26009
26565
|
var STRIPE_API_VERSION = "2026-02-25.clover";
|
|
26010
26566
|
function requireStripeSecret() {
|
|
26011
26567
|
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
@@ -26054,7 +26610,7 @@ function opForCostPath(p) {
|
|
|
26054
26610
|
if (p.endsWith("/search")) return "google_ads_search";
|
|
26055
26611
|
return "google_ads_intel";
|
|
26056
26612
|
}
|
|
26057
|
-
if (p === "/reddit/thread") return "reddit_thread";
|
|
26613
|
+
if (p === "/reddit/thread" || p === "/reddit/trending") return "reddit_thread";
|
|
26058
26614
|
if (p.startsWith("/trustpilot/")) return "trustpilot_reviews";
|
|
26059
26615
|
if (p.startsWith("/g2/")) return "g2_reviews";
|
|
26060
26616
|
if (p.startsWith("/api/internal/site-architecture-auditor")) return "audit_site";
|
|
@@ -26226,7 +26782,26 @@ app.get("/catalog", (c) => {
|
|
|
26226
26782
|
app.get("/rates", (c) => {
|
|
26227
26783
|
return c.json(getPublicRatesPayload());
|
|
26228
26784
|
});
|
|
26785
|
+
function sessionBootstrapPayload(user) {
|
|
26786
|
+
return {
|
|
26787
|
+
authenticated: true,
|
|
26788
|
+
id: user.id,
|
|
26789
|
+
email: user.email,
|
|
26790
|
+
api_key: user.key_active ? user.api_key : null,
|
|
26791
|
+
created_at: user.created_at,
|
|
26792
|
+
balance_mc: user.balance_mc,
|
|
26793
|
+
balance_credits: user.balance_mc / MC_PER_CREDIT,
|
|
26794
|
+
extra_concurrency_slots: user.extra_concurrency_slots,
|
|
26795
|
+
concurrency_limit: concurrencyLimitForUser(user),
|
|
26796
|
+
has_concurrency_sub: !!user.concurrency_stripe_sub_id,
|
|
26797
|
+
subscription_tier: user.subscription_tier,
|
|
26798
|
+
subscription_concurrency: user.subscription_concurrency,
|
|
26799
|
+
memory_plan: user.memory_plan ?? "free",
|
|
26800
|
+
has_memory_sub: !!user.memory_subscription_id
|
|
26801
|
+
};
|
|
26802
|
+
}
|
|
26229
26803
|
app.get("/me", async (c) => {
|
|
26804
|
+
c.header("Cache-Control", "no-store");
|
|
26230
26805
|
const token = getCookie2(c, "session");
|
|
26231
26806
|
if (!token) return c.json({ authenticated: false });
|
|
26232
26807
|
const userId = verifySession(token);
|
|
@@ -26239,6 +26814,8 @@ app.get("/me", async (c) => {
|
|
|
26239
26814
|
deleteCookie(c, "session", { path: "/", secure: secureCookies2, sameSite: "Strict" });
|
|
26240
26815
|
return c.json({ authenticated: false });
|
|
26241
26816
|
}
|
|
26817
|
+
const bootstrap = sessionBootstrapPayload(foundUser);
|
|
26818
|
+
if (c.req.query("bootstrap") === "1") return c.json(bootstrap);
|
|
26242
26819
|
const refreshed = await applyMonthlyFreeRefresh(foundUser);
|
|
26243
26820
|
const [balanceMc, stats] = await Promise.all([
|
|
26244
26821
|
reconcileBalanceMc(refreshed.id, refreshed.balance_mc),
|
|
@@ -26246,20 +26823,7 @@ app.get("/me", async (c) => {
|
|
|
26246
26823
|
]);
|
|
26247
26824
|
const user = { ...refreshed, balance_mc: balanceMc };
|
|
26248
26825
|
return c.json({
|
|
26249
|
-
|
|
26250
|
-
id: user.id,
|
|
26251
|
-
email: user.email,
|
|
26252
|
-
api_key: user.key_active ? user.api_key : null,
|
|
26253
|
-
created_at: user.created_at,
|
|
26254
|
-
balance_mc: user.balance_mc,
|
|
26255
|
-
balance_credits: user.balance_mc / MC_PER_CREDIT,
|
|
26256
|
-
extra_concurrency_slots: user.extra_concurrency_slots,
|
|
26257
|
-
concurrency_limit: concurrencyLimitForUser(user),
|
|
26258
|
-
has_concurrency_sub: !!user.concurrency_stripe_sub_id,
|
|
26259
|
-
subscription_tier: user.subscription_tier,
|
|
26260
|
-
subscription_concurrency: user.subscription_concurrency,
|
|
26261
|
-
memory_plan: user.memory_plan ?? "free",
|
|
26262
|
-
has_memory_sub: !!user.memory_subscription_id,
|
|
26826
|
+
...sessionBootstrapPayload(user),
|
|
26263
26827
|
...stats
|
|
26264
26828
|
});
|
|
26265
26829
|
});
|
|
@@ -26479,6 +27043,12 @@ app.post("/memory/checkout", auth2, async (c) => {
|
|
|
26479
27043
|
const body = await c.req.json().catch(() => ({}));
|
|
26480
27044
|
const priceId = body.priceId?.trim();
|
|
26481
27045
|
if (!priceId || !MEMORY_PLANS[priceId]) return c.json({ error: "Invalid priceId \u2014 must be a memory plan price." }, 400);
|
|
27046
|
+
if (tierIncludesMemory(user.subscription_tier)) {
|
|
27047
|
+
return c.json({
|
|
27048
|
+
error: "Your current plan already includes Memory Pro \u2014 there is nothing to buy here.",
|
|
27049
|
+
error_code: "memory_already_included"
|
|
27050
|
+
}, 409);
|
|
27051
|
+
}
|
|
26482
27052
|
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
26483
27053
|
if (!secret2) return c.json({ error: "Stripe is not configured." }, 503);
|
|
26484
27054
|
const stripeClient = new Stripe2(secret2, { apiVersion: STRIPE_API_VERSION });
|
|
@@ -27402,6 +27972,30 @@ app.put("/schedule-actions/:id/connections", auth2, requireIntegrationsTier, asy
|
|
|
27402
27972
|
return scheduleConnectionError(c, err, "Unable to update scheduled action service connections.");
|
|
27403
27973
|
}
|
|
27404
27974
|
});
|
|
27975
|
+
app.get("/schedule-actions/:id/runs", auth2, async (c) => {
|
|
27976
|
+
const user = c.get("user");
|
|
27977
|
+
const { key, error } = await getOrCreateUserMemoryKey(user);
|
|
27978
|
+
if (!key) return c.json({ ok: false, error: error ?? "memory unavailable" }, 503);
|
|
27979
|
+
const limitRaw = Number(c.req.query("limit") ?? "50");
|
|
27980
|
+
const limit = Number.isFinite(limitRaw) ? Math.min(Math.max(Math.round(limitRaw), 1), 100) : 50;
|
|
27981
|
+
const r = await memoryCall("listScheduledActionRunsTool", { id: c.req.param("id"), limit }, key);
|
|
27982
|
+
return c.json(r);
|
|
27983
|
+
});
|
|
27984
|
+
app.put("/schedule-actions/:id", auth2, requirePaidSchedulingTier, async (c) => {
|
|
27985
|
+
const user = c.get("user");
|
|
27986
|
+
const { key, error } = await getOrCreateUserMemoryKey(user);
|
|
27987
|
+
if (!key) return c.json({ ok: false, error: error ?? "memory unavailable" }, 503);
|
|
27988
|
+
const body = await c.req.json().catch(() => ({}));
|
|
27989
|
+
const updates = {};
|
|
27990
|
+
for (const field of ["description", "vault", "cadence", "timeOfDay", "timezone"]) {
|
|
27991
|
+
if (Object.prototype.hasOwnProperty.call(body, field)) updates[field] = body[field];
|
|
27992
|
+
}
|
|
27993
|
+
if (Object.keys(updates).length === 0) {
|
|
27994
|
+
return c.json({ ok: false, error: "Provide at least one of description, vault, cadence, timeOfDay, timezone." }, 400);
|
|
27995
|
+
}
|
|
27996
|
+
const r = await memoryCall("updateScheduledActionTool", { id: c.req.param("id"), ...updates }, key);
|
|
27997
|
+
return c.json(r);
|
|
27998
|
+
});
|
|
27405
27999
|
app.post("/schedule-actions/:id/pause", auth2, async (c) => {
|
|
27406
28000
|
const user = c.get("user");
|
|
27407
28001
|
const { key, error } = await getOrCreateUserMemoryKey(user);
|
|
@@ -27587,7 +28181,8 @@ app.post("/harvest/sync", auth2, async (c) => {
|
|
|
27587
28181
|
proxyZip: body.proxyZip,
|
|
27588
28182
|
debug: body.debug ?? false,
|
|
27589
28183
|
serpOnly: body.serpOnly ?? false,
|
|
27590
|
-
pages: Math.min(2, Math.max(1, body.pages ?? 1))
|
|
28184
|
+
pages: Math.min(2, Math.max(1, body.pages ?? 1)),
|
|
28185
|
+
recency: body.recency
|
|
27591
28186
|
};
|
|
27592
28187
|
const syncCost = options.serpOnly ? MC_COSTS.serp_headful : paaCostForQuestionCount(options.maxQuestions);
|
|
27593
28188
|
const { ok: syncOk, balance_mc: syncBal } = await debitMc(user.id, syncCost, options.serpOnly ? LedgerOperation.SERP : LedgerOperation.PAA, options.query);
|
|
@@ -28074,6 +28669,11 @@ async function chargeTierChangeNow(stripeClient, subscriptionId, customerId) {
|
|
|
28074
28669
|
return { ok: false, amountDue: 0, error: err instanceof Error ? err.message : "Unable to charge the plan change immediately." };
|
|
28075
28670
|
}
|
|
28076
28671
|
}
|
|
28672
|
+
app.get("/billing/offer/:slug", async (c) => {
|
|
28673
|
+
const payload = getPrivateOfferPayload(c.req.param("slug").trim().toLowerCase());
|
|
28674
|
+
if (!payload) return c.json({ error: "Unknown offer." }, 404);
|
|
28675
|
+
return c.json(payload);
|
|
28676
|
+
});
|
|
28077
28677
|
app.post("/billing/subscribe", requireAllowedOrigin, sessionAuth, async (c) => {
|
|
28078
28678
|
try {
|
|
28079
28679
|
const user = c.get("sessionUser");
|
|
@@ -28119,7 +28719,7 @@ app.post("/billing/subscribe", requireAllowedOrigin, sessionAuth, async (c) => {
|
|
|
28119
28719
|
billing_address_collection: "required",
|
|
28120
28720
|
customer_update: { address: "auto", name: "auto" },
|
|
28121
28721
|
tax_id_collection: { enabled: true },
|
|
28122
|
-
return_url: `${appOrigin()}/billing?subscribed=${tier.tier}`
|
|
28722
|
+
return_url: tier.private_offer && offerClaimPathForTier(tier.tier) ? `${appOrigin()}${offerClaimPathForTier(tier.tier)}?purchased=${tier.tier}` : `${appOrigin()}/billing?subscribed=${tier.tier}`
|
|
28123
28723
|
});
|
|
28124
28724
|
return c.json({ clientSecret: session.client_secret });
|
|
28125
28725
|
} catch (err) {
|
|
@@ -28205,7 +28805,7 @@ app.post("/billing/subscribe/terminal-checkout", auth2, async (c) => {
|
|
|
28205
28805
|
const user = c.get("user");
|
|
28206
28806
|
const raw = await c.req.json().catch(() => ({}));
|
|
28207
28807
|
const tier = SUBSCRIPTION_TIER_BY_KEY[String(raw.tier ?? "")];
|
|
28208
|
-
if (!tier) return c.json({ error: "Invalid tier. Choose starter, growth, or scale.", tiers: Object.
|
|
28808
|
+
if (!tier) return c.json({ error: "Invalid tier. Choose starter, growth, or scale.", tiers: Object.values(SUBSCRIPTION_TIER_BY_KEY).filter((t) => !t.private_offer).map((t) => t.tier) }, 400);
|
|
28209
28809
|
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
28210
28810
|
if (!secret2) return c.json({ error: "Stripe is not configured." }, 503);
|
|
28211
28811
|
const stripeClient = new Stripe2(secret2, { apiVersion: STRIPE_API_VERSION });
|
|
@@ -28253,7 +28853,12 @@ app.post("/billing/subscribe/terminal-checkout", auth2, async (c) => {
|
|
|
28253
28853
|
tier: tier.tier,
|
|
28254
28854
|
label: tier.label,
|
|
28255
28855
|
monthly_usd: tier.monthly_usd,
|
|
28856
|
+
amount_usd: tierAmountChargedUsd(tier),
|
|
28857
|
+
billing_interval: tierBillingInterval(tier),
|
|
28858
|
+
credits_per_interval: tierCreditsPerInterval(tier),
|
|
28256
28859
|
credits_per_month: tier.credits_mc / MC_PER_CREDIT,
|
|
28860
|
+
credits_never_expire: !!tier.credits_never_expire,
|
|
28861
|
+
includes_memory: !!tier.includes_memory,
|
|
28257
28862
|
concurrency: tier.concurrency,
|
|
28258
28863
|
intro: tier.intro_coupon ? "$1 first month" : null,
|
|
28259
28864
|
next_step: "Open checkout_url in a browser to complete payment."
|
|
@@ -28288,6 +28893,7 @@ app.get("/billing/balance", auth2, async (c) => {
|
|
|
28288
28893
|
ledger
|
|
28289
28894
|
});
|
|
28290
28895
|
});
|
|
28896
|
+
app.get("/billing/summary", auth2, billingSummaryHandler);
|
|
28291
28897
|
app.get("/billing/connected-usage/history", auth2, async (c) => {
|
|
28292
28898
|
return c.json({
|
|
28293
28899
|
ok: true,
|
|
@@ -28331,7 +28937,7 @@ app.get("/cron/tick", async (c) => {
|
|
|
28331
28937
|
if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
|
|
28332
28938
|
return c.json({ error: "Unauthorized" }, 401);
|
|
28333
28939
|
}
|
|
28334
|
-
const { drainQueue } = await import("./worker-
|
|
28940
|
+
const { drainQueue } = await import("./worker-ZZHSYI3Y.js");
|
|
28335
28941
|
const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
|
|
28336
28942
|
const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
|
|
28337
28943
|
const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup] = await Promise.all([
|
|
@@ -28359,15 +28965,15 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
28359
28965
|
return c.json({ error: "Unauthorized" }, 401);
|
|
28360
28966
|
}
|
|
28361
28967
|
const jobId = c.req.param("id");
|
|
28362
|
-
const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-
|
|
28363
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
28968
|
+
const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-JHMVHENZ.js");
|
|
28969
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-GPZRLBVY.js");
|
|
28364
28970
|
const job = await getExtractJob2(jobId);
|
|
28365
28971
|
if (!job) return c.json({ error: "job not found" }, 404);
|
|
28366
28972
|
const stored = await assembleExtractArtifacts(job, {});
|
|
28367
28973
|
await completeExtractJob2(jobId, stored);
|
|
28368
28974
|
let settlement = "already_settled_or_refunded";
|
|
28369
28975
|
if (job.billedMc == null && job.userId != null) {
|
|
28370
|
-
const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-
|
|
28976
|
+
const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-JHMVHENZ.js");
|
|
28371
28977
|
const heldMc = Number(job.options.heldMc ?? 0);
|
|
28372
28978
|
const successful = await countSuccessfulPages2(jobId);
|
|
28373
28979
|
const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
|
|
@@ -28386,6 +28992,8 @@ app.on(["GET", "POST", "PUT"], "/api/inngest", serveInngest({ client: inngest, f
|
|
|
28386
28992
|
app.route("/api/internal/site-architecture-auditor", siteAuditApp);
|
|
28387
28993
|
app.route("/api/internal/memory", internalMemoryApp);
|
|
28388
28994
|
app.route("/api/internal/billing", internalBillingApp);
|
|
28995
|
+
app.route("/api/internal/inbox", internalInboxApp);
|
|
28996
|
+
app.route("/inbox", inboxApp);
|
|
28389
28997
|
app.route("/youtube", youtubeApp);
|
|
28390
28998
|
app.route("/screenshot", screenshotApp);
|
|
28391
28999
|
app.route("/facebook", facebookAdApp);
|
|
@@ -28534,4 +29142,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
28534
29142
|
export {
|
|
28535
29143
|
app
|
|
28536
29144
|
};
|
|
28537
|
-
//# sourceMappingURL=server-
|
|
29145
|
+
//# sourceMappingURL=server-LKVPEQFE.js.map
|