mcp-scraper 0.44.4 → 0.46.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -2
- package/dist/bin/api-server.cjs +4795 -3771
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +3 -2
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +2722 -2509
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +4 -4
- package/dist/{chunk-ZVOVPXDL.js → chunk-BP2IUIAH.js} +2954 -2742
- package/dist/chunk-BP2IUIAH.js.map +1 -0
- package/dist/{chunk-WWIJ2NID.js → chunk-JFOT5JEL.js} +2 -2
- package/dist/{chunk-GRODXRUY.js → chunk-QSUGPYGQ.js} +3 -2
- package/dist/chunk-QSUGPYGQ.js.map +1 -0
- package/dist/{chunk-X5WDJ7D7.js → chunk-SOEYDWJU.js} +46 -3
- package/dist/chunk-SOEYDWJU.js.map +1 -0
- package/dist/chunk-TQKQK7OV.js +7 -0
- package/dist/chunk-TQKQK7OV.js.map +1 -0
- package/dist/{extract-bundle-U3MNYDKC.js → extract-bundle-DYMQRQWG.js} +3 -3
- package/dist/{server-W4JLSNKW.js → server-SMEHT3OU.js} +924 -168
- package/dist/server-SMEHT3OU.js.map +1 -0
- package/dist/{site-extract-repository-CKCJDP7N.js → site-extract-repository-G723PZIL.js} +3 -3
- package/dist/{worker-73AYU4IK.js → worker-4LZGRHDA.js} +2 -2
- package/package.json +1 -1
- package/dist/chunk-GRODXRUY.js.map +0 -1
- package/dist/chunk-HFCIHZHA.js +0 -7
- package/dist/chunk-HFCIHZHA.js.map +0 -1
- package/dist/chunk-X5WDJ7D7.js.map +0 -1
- package/dist/chunk-ZVOVPXDL.js.map +0 -1
- package/dist/server-W4JLSNKW.js.map +0 -1
- /package/dist/{chunk-WWIJ2NID.js.map → chunk-JFOT5JEL.js.map} +0 -0
- /package/dist/{extract-bundle-U3MNYDKC.js.map → extract-bundle-DYMQRQWG.js.map} +0 -0
- /package/dist/{site-extract-repository-CKCJDP7N.js.map → site-extract-repository-G723PZIL.js.map} +0 -0
- /package/dist/{worker-73AYU4IK.js.map → worker-4LZGRHDA.js.map} +0 -0
|
@@ -48,6 +48,10 @@ import {
|
|
|
48
48
|
import {
|
|
49
49
|
ApiErrorResponseSchema,
|
|
50
50
|
ArtifactTemplateConfigSchema,
|
|
51
|
+
CommonsClaimPublicationInputSchema,
|
|
52
|
+
CommonsPreparePublicationInputSchema,
|
|
53
|
+
CommonsPublishEditorialInputSchema,
|
|
54
|
+
CommonsValidatePublicationInputSchema,
|
|
51
55
|
CreateArtifactTemplateInputSchema,
|
|
52
56
|
CreateEditorialReadingRoomInputSchema,
|
|
53
57
|
CreateScheduledRunViewLinkInputSchema,
|
|
@@ -56,6 +60,7 @@ import {
|
|
|
56
60
|
HttpMcpToolExecutor,
|
|
57
61
|
InternalScheduledArtifactRenderRequestSchema,
|
|
58
62
|
ListScheduledRunsQuerySchema,
|
|
63
|
+
LocalSourcebookSchemaTypeSchema,
|
|
59
64
|
PublicScheduledRunTokenSchema,
|
|
60
65
|
RenewEditorialReadingRoomDownloadInputSchema,
|
|
61
66
|
SCHEDULED_ARTIFACT_TEMPLATE_PRESET,
|
|
@@ -72,11 +77,13 @@ import {
|
|
|
72
77
|
configureReportSaving,
|
|
73
78
|
createConnectedDataArtifact,
|
|
74
79
|
createDirectoryCsvArtifact,
|
|
80
|
+
defaultLocalSourcebookSchemaType,
|
|
75
81
|
exportFanout,
|
|
76
82
|
handleMcpTasksHttpRequest,
|
|
77
83
|
harvestTimeoutBudget,
|
|
78
84
|
hashOwnerId,
|
|
79
85
|
isLoopbackBaseUrl,
|
|
86
|
+
localSourcebookSchemaContract,
|
|
80
87
|
outputBaseDir,
|
|
81
88
|
registerBrowserAgentMcpTools,
|
|
82
89
|
registerMemoryMcpTools,
|
|
@@ -85,8 +92,9 @@ import {
|
|
|
85
92
|
renewConnectedDataArtifactDownload,
|
|
86
93
|
renewDirectoryArtifactDownload,
|
|
87
94
|
resolveDeploymentProfile,
|
|
95
|
+
resolveLocalSourcebookSchemaType,
|
|
88
96
|
transcribeMediaUrl
|
|
89
|
-
} from "./chunk-
|
|
97
|
+
} from "./chunk-BP2IUIAH.js";
|
|
90
98
|
import {
|
|
91
99
|
auditImageUrls,
|
|
92
100
|
auditImages,
|
|
@@ -142,7 +150,7 @@ import {
|
|
|
142
150
|
} from "./chunk-3ZUBQQPQ.js";
|
|
143
151
|
import {
|
|
144
152
|
PACKAGE_VERSION
|
|
145
|
-
} from "./chunk-
|
|
153
|
+
} from "./chunk-TQKQK7OV.js";
|
|
146
154
|
import {
|
|
147
155
|
abandonExtractSettlement,
|
|
148
156
|
countSuccessfulPages,
|
|
@@ -165,7 +173,7 @@ import {
|
|
|
165
173
|
setExtractJobTotal,
|
|
166
174
|
settleExtractJob,
|
|
167
175
|
terminalExtractJobStatus
|
|
168
|
-
} from "./chunk-
|
|
176
|
+
} from "./chunk-JFOT5JEL.js";
|
|
169
177
|
import {
|
|
170
178
|
BROWSER_OPEN_MIN_BALANCE_MC,
|
|
171
179
|
CONCURRENCY_PRICE_ID,
|
|
@@ -178,6 +186,7 @@ import {
|
|
|
178
186
|
CONNECTED_USAGE_RATE_POLICY,
|
|
179
187
|
CONNECTED_USAGE_RATE_POLICY_VERSION,
|
|
180
188
|
CREDIT_COST_CATALOG,
|
|
189
|
+
LISTING_WRITE_RATE_POLICY,
|
|
181
190
|
LedgerOperation,
|
|
182
191
|
MC_COSTS,
|
|
183
192
|
MC_PER_CREDIT,
|
|
@@ -213,7 +222,7 @@ import {
|
|
|
213
222
|
tierChangeLeavesPrivateOffer,
|
|
214
223
|
tierCreditsPerInterval,
|
|
215
224
|
vendorCostUsdToMc
|
|
216
|
-
} from "./chunk-
|
|
225
|
+
} from "./chunk-SOEYDWJU.js";
|
|
217
226
|
import {
|
|
218
227
|
CaptchaError,
|
|
219
228
|
MapsSearchIncompletePageError,
|
|
@@ -3638,7 +3647,7 @@ async function generateOgImage(post) {
|
|
|
3638
3647
|
|
|
3639
3648
|
// src/api/server.ts
|
|
3640
3649
|
import { Resend as Resend3 } from "resend";
|
|
3641
|
-
import { randomUUID as
|
|
3650
|
+
import { randomUUID as randomUUID23 } from "crypto";
|
|
3642
3651
|
|
|
3643
3652
|
// src/api/kpo-extractor.ts
|
|
3644
3653
|
import TurndownService from "turndown";
|
|
@@ -8667,7 +8676,7 @@ var siteExtractFn = inngest.createFunction(
|
|
|
8667
8676
|
});
|
|
8668
8677
|
}) : null;
|
|
8669
8678
|
const finalized = await step.run("finalize", async () => {
|
|
8670
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
8679
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-DYMQRQWG.js");
|
|
8671
8680
|
const current = await getExtractJob(jobId2);
|
|
8672
8681
|
if (!current) throw new Error("extract job disappeared before finalization");
|
|
8673
8682
|
const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
|
|
@@ -12033,16 +12042,18 @@ async function createLocalSourcebookSubmission(input) {
|
|
|
12033
12042
|
const existing = await db.execute({ sql: `SELECT * FROM local_sourcebook_submissions WHERE owner_user_id = ? AND idempotency_key = ? LIMIT 1`, args: [input.ownerUserId, input.idempotencyKey] });
|
|
12034
12043
|
if (existing.rows[0]) {
|
|
12035
12044
|
const submission = mapSubmission(existing.rows[0]);
|
|
12036
|
-
if (submission.category !== input.category || submission.state !== input.state || submission.slug !== input.slug || submission.businessName !== input.businessName || submission.websiteUrl !== input.websiteUrl) throw new Error("Local Sourcebook idempotency key conflicts with a different listing identity.");
|
|
12045
|
+
if (submission.category !== input.category || submission.state !== input.state || submission.slug !== input.slug || submission.businessName !== input.businessName || submission.websiteUrl !== input.websiteUrl || input.schemaOrgType != null && submission.coverage.schemaOrgType !== input.schemaOrgType) throw new Error("Local Sourcebook idempotency key conflicts with a different listing identity.");
|
|
12037
12046
|
return submission;
|
|
12038
12047
|
}
|
|
12039
12048
|
}
|
|
12040
12049
|
const id = `lsb_${randomUUID3().replace(/-/g, "")}`;
|
|
12041
12050
|
const coverage = {
|
|
12042
|
-
requested: ["website_crawl", "services_products", "service_areas", "genuine_images", "staff_team", "review_sources"],
|
|
12051
|
+
requested: ["website_crawl", "structured_data", "services_products", "service_areas", "genuine_images", "staff_team", "review_sources"],
|
|
12043
12052
|
crawl: { state: "queued", pagesDiscovered: 0, pagesCaptured: 0 },
|
|
12044
12053
|
media: { state: "queued", genuineImagesCaptured: 0 },
|
|
12045
|
-
reviews: { state: "queued", captured: 0, availableReported: null, stopReason: null }
|
|
12054
|
+
reviews: { state: "queued", captured: 0, availableReported: null, stopReason: null },
|
|
12055
|
+
...input.schemaOrgType ? { schemaOrgType: input.schemaOrgType } : {},
|
|
12056
|
+
...input.billing ? { billing: { mode: "fixed_listing_write", ...input.billing } } : {}
|
|
12046
12057
|
};
|
|
12047
12058
|
await db.batch([
|
|
12048
12059
|
{ sql: `INSERT INTO local_sourcebook_submissions (id, owner_user_id, category, state, slug, business_name, website_url, coverage_json, idempotency_key) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, args: [id, input.ownerUserId, input.category, input.state, input.slug, input.businessName, input.websiteUrl, JSON.stringify(coverage), input.idempotencyKey ?? null] },
|
|
@@ -12062,13 +12073,17 @@ async function getLocalSourcebookDraft(id, ownerUserId) {
|
|
|
12062
12073
|
const result = await getDb().execute({ sql: `SELECT payload_json FROM local_sourcebook_revisions WHERE submission_id = ? AND revision = ?`, args: [id, submission.draftRevision] });
|
|
12063
12074
|
return result.rows[0] ? parseJson2(result.rows[0].payload_json) : null;
|
|
12064
12075
|
}
|
|
12065
|
-
async function queueLocalSourcebookRefresh(id, ownerUserId) {
|
|
12076
|
+
async function queueLocalSourcebookRefresh(id, ownerUserId, billing) {
|
|
12066
12077
|
const existing = await getLocalSourcebookSubmission(id, ownerUserId);
|
|
12067
12078
|
if (!existing) return null;
|
|
12068
12079
|
if (existing.status === "rejected" || existing.status === "unpublished") {
|
|
12069
12080
|
throw new Error(`A ${existing.status} Local Sourcebook listing cannot be refreshed by its owner.`);
|
|
12070
12081
|
}
|
|
12071
|
-
const coverage = {
|
|
12082
|
+
const coverage = {
|
|
12083
|
+
...existing.coverage,
|
|
12084
|
+
refreshRequestedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
12085
|
+
...billing ? { billing: { mode: "fixed_listing_write", ...billing } } : {}
|
|
12086
|
+
};
|
|
12072
12087
|
await getDb().execute({ sql: `UPDATE local_sourcebook_submissions SET status = 'queued', coverage_json = ?, acquisition_claimed_at = NULL, acquisition_error = NULL, updated_at = datetime('now') WHERE id = ? AND owner_user_id = ?`, args: [JSON.stringify(coverage), id, ownerUserId] });
|
|
12073
12088
|
await event(id, "enrichment.queued", "owner", String(ownerUserId));
|
|
12074
12089
|
return getLocalSourcebookSubmission(id, ownerUserId);
|
|
@@ -12774,6 +12789,80 @@ var MapsExtractor = class {
|
|
|
12774
12789
|
}
|
|
12775
12790
|
};
|
|
12776
12791
|
|
|
12792
|
+
// src/api/listing-write-billing.ts
|
|
12793
|
+
function operationFor(surface, kind) {
|
|
12794
|
+
if (surface === "local_sourcebook") {
|
|
12795
|
+
return kind === "create" ? LedgerOperation.LOCAL_SOURCEBOOK_CREATE : LedgerOperation.LOCAL_SOURCEBOOK_EDIT;
|
|
12796
|
+
}
|
|
12797
|
+
return kind === "create" ? LedgerOperation.COMMONS_CREATE : LedgerOperation.COMMONS_EDIT;
|
|
12798
|
+
}
|
|
12799
|
+
function refundOperationFor(surface, kind) {
|
|
12800
|
+
if (surface === "local_sourcebook") {
|
|
12801
|
+
return kind === "create" ? LedgerOperation.LOCAL_SOURCEBOOK_CREATE_REFUND : LedgerOperation.LOCAL_SOURCEBOOK_EDIT_REFUND;
|
|
12802
|
+
}
|
|
12803
|
+
return kind === "create" ? LedgerOperation.COMMONS_CREATE_REFUND : LedgerOperation.COMMONS_EDIT_REFUND;
|
|
12804
|
+
}
|
|
12805
|
+
function listingWritePrice(kind) {
|
|
12806
|
+
return LISTING_WRITE_RATE_POLICY[kind];
|
|
12807
|
+
}
|
|
12808
|
+
function listingWriteDebitKey(surface, userId, idempotencyKey3) {
|
|
12809
|
+
return `listing-write:${surface}:${userId}:${idempotencyKey3}`;
|
|
12810
|
+
}
|
|
12811
|
+
function publicListingWriteBilling(receipt) {
|
|
12812
|
+
return {
|
|
12813
|
+
surface: receipt.surface,
|
|
12814
|
+
operation: receipt.kind,
|
|
12815
|
+
credits: receipt.credits,
|
|
12816
|
+
charged: !receipt.duplicate,
|
|
12817
|
+
idempotentReplay: receipt.duplicate,
|
|
12818
|
+
balanceCredits: receipt.balanceMc / MC_PER_CREDIT,
|
|
12819
|
+
acquisitionIncluded: receipt.surface === "local_sourcebook"
|
|
12820
|
+
};
|
|
12821
|
+
}
|
|
12822
|
+
async function chargeListingWrite(input) {
|
|
12823
|
+
const price = listingWritePrice(input.kind);
|
|
12824
|
+
const debitKey2 = listingWriteDebitKey(input.surface, input.userId, input.idempotencyKey);
|
|
12825
|
+
const debit = await debitMcIdempotent(
|
|
12826
|
+
input.userId,
|
|
12827
|
+
price.mc,
|
|
12828
|
+
operationFor(input.surface, input.kind),
|
|
12829
|
+
`${input.surface}:${input.kind}`,
|
|
12830
|
+
debitKey2
|
|
12831
|
+
);
|
|
12832
|
+
if (!debit.ok) return { insufficient: insufficientBalanceResponse(debit.balance_mc, price.mc) };
|
|
12833
|
+
return {
|
|
12834
|
+
surface: input.surface,
|
|
12835
|
+
kind: input.kind,
|
|
12836
|
+
userId: input.userId,
|
|
12837
|
+
credits: price.credits,
|
|
12838
|
+
amountMc: price.mc,
|
|
12839
|
+
debitKey: debitKey2,
|
|
12840
|
+
duplicate: debit.duplicate,
|
|
12841
|
+
balanceMc: debit.balance_mc
|
|
12842
|
+
};
|
|
12843
|
+
}
|
|
12844
|
+
async function refundListingWrite(receipt, reason) {
|
|
12845
|
+
if (receipt.duplicate) return null;
|
|
12846
|
+
return settleDebitMcIdempotent(
|
|
12847
|
+
receipt.userId,
|
|
12848
|
+
receipt.debitKey,
|
|
12849
|
+
0,
|
|
12850
|
+
refundOperationFor(receipt.surface, receipt.kind),
|
|
12851
|
+
reason,
|
|
12852
|
+
receipt.surface
|
|
12853
|
+
);
|
|
12854
|
+
}
|
|
12855
|
+
async function refundPersistedListingWrite(input) {
|
|
12856
|
+
return settleDebitMcIdempotent(
|
|
12857
|
+
input.userId,
|
|
12858
|
+
input.debitKey,
|
|
12859
|
+
0,
|
|
12860
|
+
refundOperationFor(input.surface, input.kind),
|
|
12861
|
+
input.reason,
|
|
12862
|
+
input.surface
|
|
12863
|
+
);
|
|
12864
|
+
}
|
|
12865
|
+
|
|
12777
12866
|
// src/api/local-sourcebook-compiler.ts
|
|
12778
12867
|
import { createHash as createHash3 } from "crypto";
|
|
12779
12868
|
|
|
@@ -13100,6 +13189,65 @@ function reviewStopReason(input, captured, exposed) {
|
|
|
13100
13189
|
if (exposed && captured < exposed) return `Google Maps stopped exposing additional complete review cards after ${captured} of ${exposed} reported reviews.`;
|
|
13101
13190
|
return `All ${captured} complete, unique review cards exposed during this acquisition were retained.`;
|
|
13102
13191
|
}
|
|
13192
|
+
function buildLocalBusinessJsonLd(input) {
|
|
13193
|
+
const businessId = `${input.profileUrl}#business`;
|
|
13194
|
+
const business = {
|
|
13195
|
+
"@type": input.schemaOrgType === "LocalBusiness" ? "LocalBusiness" : ["LocalBusiness", input.schemaOrgType],
|
|
13196
|
+
"@id": businessId,
|
|
13197
|
+
url: input.profileUrl,
|
|
13198
|
+
name: input.name,
|
|
13199
|
+
description: input.description,
|
|
13200
|
+
...input.phone ? { telephone: input.phone } : {},
|
|
13201
|
+
...input.address || input.locality || input.region || input.postalCode ? {
|
|
13202
|
+
address: {
|
|
13203
|
+
"@type": "PostalAddress",
|
|
13204
|
+
...input.address ? { streetAddress: input.address } : {},
|
|
13205
|
+
...input.locality ? { addressLocality: input.locality } : {},
|
|
13206
|
+
...input.region ? { addressRegion: input.region } : {},
|
|
13207
|
+
...input.postalCode ? { postalCode: input.postalCode } : {},
|
|
13208
|
+
addressCountry: "US"
|
|
13209
|
+
}
|
|
13210
|
+
} : {},
|
|
13211
|
+
...Number.isFinite(input.latitude) && Number.isFinite(input.longitude) ? {
|
|
13212
|
+
geo: { "@type": "GeoCoordinates", latitude: input.latitude, longitude: input.longitude }
|
|
13213
|
+
} : {},
|
|
13214
|
+
...input.images.length ? { image: input.images } : {},
|
|
13215
|
+
...input.serviceAreas.length ? { areaServed: input.serviceAreas } : {},
|
|
13216
|
+
...input.services.length ? {
|
|
13217
|
+
hasOfferCatalog: {
|
|
13218
|
+
"@type": "OfferCatalog",
|
|
13219
|
+
name: "Services",
|
|
13220
|
+
itemListElement: input.services.map((name) => ({ "@type": "Offer", itemOffered: { "@type": "Service", name } }))
|
|
13221
|
+
}
|
|
13222
|
+
} : {},
|
|
13223
|
+
...Number.isFinite(input.rating) && Number.isFinite(input.ratingCount) && Number(input.ratingCount) > 0 ? {
|
|
13224
|
+
aggregateRating: { "@type": "AggregateRating", ratingValue: input.rating, reviewCount: input.ratingCount, bestRating: 5, worstRating: 1 }
|
|
13225
|
+
} : {},
|
|
13226
|
+
sameAs: unique([input.website, input.mapsUrl], 10)
|
|
13227
|
+
};
|
|
13228
|
+
const graph = [business, {
|
|
13229
|
+
"@type": "BreadcrumbList",
|
|
13230
|
+
"@id": `${input.profileUrl}#breadcrumbs`,
|
|
13231
|
+
itemListElement: [
|
|
13232
|
+
{ "@type": "ListItem", position: 1, name: "Local Sourcebook", item: "https://localsourcebook.com/" },
|
|
13233
|
+
{ "@type": "ListItem", position: 2, name: input.region || "State directory", item: input.stateUrl },
|
|
13234
|
+
{ "@type": "ListItem", position: 3, name: input.name, item: input.profileUrl }
|
|
13235
|
+
]
|
|
13236
|
+
}];
|
|
13237
|
+
if (input.faq.length) {
|
|
13238
|
+
graph.push({
|
|
13239
|
+
"@type": "FAQPage",
|
|
13240
|
+
"@id": `${input.profileUrl}#faq`,
|
|
13241
|
+
about: { "@id": businessId },
|
|
13242
|
+
mainEntity: input.faq.map((item) => ({
|
|
13243
|
+
"@type": "Question",
|
|
13244
|
+
name: item.question,
|
|
13245
|
+
acceptedAnswer: { "@type": "Answer", text: item.answer }
|
|
13246
|
+
}))
|
|
13247
|
+
});
|
|
13248
|
+
}
|
|
13249
|
+
return { "@context": "https://schema.org", "@graph": graph };
|
|
13250
|
+
}
|
|
13103
13251
|
function compileLocalSourcebookListing(input) {
|
|
13104
13252
|
const collectedAt = input.collectedAt ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
13105
13253
|
const pages = successfulPages(input.site);
|
|
@@ -13169,6 +13317,33 @@ function compileLocalSourcebookListing(input) {
|
|
|
13169
13317
|
}
|
|
13170
13318
|
];
|
|
13171
13319
|
const publicUrls = localSourcebookPublicUrls(input.submission.category, input.submission.state, input.submission.slug);
|
|
13320
|
+
const schemaOrgType = resolveLocalSourcebookSchemaType(input.submission.category, input.submission.coverage.schemaOrgType);
|
|
13321
|
+
const listingName = maps?.name || input.submission.businessName;
|
|
13322
|
+
const listingWebsite = maps?.website || input.submission.websiteUrl;
|
|
13323
|
+
const listingMapsUrl = maps?.placeUrl || maps?.cidUrl || void 0;
|
|
13324
|
+
const stateUrl = new URL(`/${input.submission.state.toLowerCase()}`, publicUrls.profileUrl).href;
|
|
13325
|
+
const jsonLd = buildLocalBusinessJsonLd({
|
|
13326
|
+
schemaOrgType,
|
|
13327
|
+
profileUrl: publicUrls.profileUrl,
|
|
13328
|
+
stateUrl,
|
|
13329
|
+
name: listingName,
|
|
13330
|
+
description: summary,
|
|
13331
|
+
address: location2.address,
|
|
13332
|
+
locality: location2.locality,
|
|
13333
|
+
region: location2.region,
|
|
13334
|
+
postalCode: location2.postalCode,
|
|
13335
|
+
phone: maps?.phoneDisplay || maps?.phone || void 0,
|
|
13336
|
+
website: listingWebsite,
|
|
13337
|
+
mapsUrl: listingMapsUrl,
|
|
13338
|
+
latitude: maps?.lat ?? void 0,
|
|
13339
|
+
longitude: maps?.lng ?? void 0,
|
|
13340
|
+
services,
|
|
13341
|
+
serviceAreas,
|
|
13342
|
+
images: media.map((item) => String(item.src)),
|
|
13343
|
+
rating: Number.isFinite(rating) ? rating : void 0,
|
|
13344
|
+
ratingCount: Number.isFinite(exposedCount) ? exposedCount : void 0,
|
|
13345
|
+
faq
|
|
13346
|
+
});
|
|
13172
13347
|
const listing = {
|
|
13173
13348
|
id: input.submission.id,
|
|
13174
13349
|
publicationRevision: input.submission.draftRevision + 1,
|
|
@@ -13185,7 +13360,9 @@ function compileLocalSourcebookListing(input) {
|
|
|
13185
13360
|
profile: publicUrls.profileUrl,
|
|
13186
13361
|
reviews: publicUrls.reviewsUrl
|
|
13187
13362
|
},
|
|
13188
|
-
name:
|
|
13363
|
+
name: listingName,
|
|
13364
|
+
schemaOrgType,
|
|
13365
|
+
jsonLd,
|
|
13189
13366
|
summary,
|
|
13190
13367
|
description,
|
|
13191
13368
|
address: location2.address,
|
|
@@ -13193,8 +13370,8 @@ function compileLocalSourcebookListing(input) {
|
|
|
13193
13370
|
region: location2.region,
|
|
13194
13371
|
postalCode: location2.postalCode,
|
|
13195
13372
|
phone: maps?.phoneDisplay || maps?.phone || void 0,
|
|
13196
|
-
website:
|
|
13197
|
-
mapsUrl:
|
|
13373
|
+
website: listingWebsite,
|
|
13374
|
+
mapsUrl: listingMapsUrl,
|
|
13198
13375
|
latitude: maps?.lat ?? void 0,
|
|
13199
13376
|
longitude: maps?.lng ?? void 0,
|
|
13200
13377
|
hours,
|
|
@@ -13234,11 +13411,12 @@ function compileLocalSourcebookListing(input) {
|
|
|
13234
13411
|
return {
|
|
13235
13412
|
listing,
|
|
13236
13413
|
coverage: {
|
|
13237
|
-
requested: ["website_crawl", "services_products", "service_areas", "genuine_images", "staff_team", "review_sources"],
|
|
13414
|
+
requested: ["website_crawl", "structured_data", "services_products", "service_areas", "genuine_images", "staff_team", "review_sources"],
|
|
13238
13415
|
crawl: { state: input.siteError ? "failed" : input.site?.truncated ? "partial" : "complete", pagesDiscovered: input.site?.spider.totalFound ?? 0, pagesCaptured: pages.length, requestedMaximum: input.requestedMaxPages, error: input.siteError ?? null },
|
|
13239
13416
|
media: { state: input.siteError ? "failed" : "complete", genuineImagesCaptured: media.length, teamImagesCaptured: media.filter((item) => item.kind === "team").length },
|
|
13240
13417
|
reviews: { state: input.mapsError || input.reviewBillingLimited ? "partial" : "complete", captured: reviews.length, availableReported: Number.isFinite(exposedCount) ? exposedCount : null, requestedMaximum: input.requestedMaxReviews, stopReason: reviewStopReason(input, reviews.length, Number.isFinite(exposedCount) ? exposedCount : void 0) },
|
|
13241
13418
|
maps: { state: maps ? "complete" : "failed", exactPlaceUrl: maps?.placeUrl ?? null, cid: maps?.cidDecimal ?? maps?.cid ?? null, error: input.mapsError ?? null },
|
|
13419
|
+
structuredData: { state: "complete", rootType: "LocalBusiness", schemaOrgType, graphNodes: jsonLd["@graph"].length },
|
|
13242
13420
|
compiledAt: collectedAt
|
|
13243
13421
|
}
|
|
13244
13422
|
};
|
|
@@ -13323,110 +13501,83 @@ async function runLocalSourcebookAcquisition(submission, provider = createDefaul
|
|
|
13323
13501
|
const user = await getUserById(submission.ownerUserId);
|
|
13324
13502
|
if (!user) throw new Error("The submission owner is no longer active.");
|
|
13325
13503
|
if (!user.subscription_tier) throw new Error("An active MCP Scraper subscription is required for acquisition.");
|
|
13326
|
-
const
|
|
13327
|
-
const
|
|
13328
|
-
const
|
|
13329
|
-
|
|
13330
|
-
|
|
13331
|
-
|
|
13332
|
-
|
|
13333
|
-
|
|
13334
|
-
|
|
13335
|
-
|
|
13336
|
-
|
|
13337
|
-
|
|
13504
|
+
const storedBilling = submission.coverage.billing;
|
|
13505
|
+
const inferredKind = submission.publishedRevision == null ? "create" : "edit";
|
|
13506
|
+
const kind = storedBilling?.kind === "edit" ? "edit" : storedBilling?.kind === "create" ? "create" : inferredKind;
|
|
13507
|
+
const price = listingWritePrice(kind);
|
|
13508
|
+
let debitKey2 = typeof storedBilling?.debitKey === "string" ? storedBilling.debitKey : "";
|
|
13509
|
+
if (!debitKey2) {
|
|
13510
|
+
const fallback = await chargeListingWrite({
|
|
13511
|
+
surface: "local_sourcebook",
|
|
13512
|
+
kind,
|
|
13513
|
+
userId: submission.ownerUserId,
|
|
13514
|
+
idempotencyKey: `worker:${submission.id}:revision:${submission.draftRevision}`
|
|
13515
|
+
});
|
|
13516
|
+
if ("insufficient" in fallback) throw new Error(fallback.insufficient.message);
|
|
13517
|
+
debitKey2 = fallback.debitKey;
|
|
13518
|
+
}
|
|
13519
|
+
const maxPages = LOCAL_SOURCEBOOK_MAX_PAGES;
|
|
13338
13520
|
let site = null;
|
|
13339
13521
|
let maps = null;
|
|
13340
13522
|
let siteError = null;
|
|
13341
13523
|
let mapsError = null;
|
|
13342
|
-
let reviewBillingLimited = false;
|
|
13343
|
-
let siteHoldMc = 0;
|
|
13344
|
-
let siteActualMc = 0;
|
|
13345
|
-
let mapsRefunded = false;
|
|
13346
|
-
let reviewsBilledMc = 0;
|
|
13347
|
-
if (maxPages < 1) {
|
|
13348
|
-
siteError = "The subscriber balance covered exact-place acquisition but not a website page.";
|
|
13349
|
-
} else {
|
|
13350
|
-
siteHoldMc = maxPages * MC_COSTS.page_scrape;
|
|
13351
|
-
const siteDescription = `Local Sourcebook: ${new URL(submission.websiteUrl).hostname}`;
|
|
13352
|
-
const siteHold = await debitMcIdempotent(
|
|
13353
|
-
submission.ownerUserId,
|
|
13354
|
-
siteHoldMc,
|
|
13355
|
-
LedgerOperation.EXTRACT_SITE_HOLD,
|
|
13356
|
-
siteDescription,
|
|
13357
|
-
`${billingCycle}:website-hold`
|
|
13358
|
-
);
|
|
13359
|
-
if (!siteHold.ok) {
|
|
13360
|
-
siteError = "The subscriber balance changed before the website acquisition hold could be placed.";
|
|
13361
|
-
siteHoldMc = 0;
|
|
13362
|
-
} else {
|
|
13363
|
-
try {
|
|
13364
|
-
site = await provider.extractWebsite(submission, maxPages);
|
|
13365
|
-
const successfulPages2 = site.pages.filter(isPageExtractionSuccessful).length;
|
|
13366
|
-
siteActualMc = successfulPages2 * MC_COSTS.page_scrape;
|
|
13367
|
-
const refund = siteHoldMc - siteActualMc;
|
|
13368
|
-
if (refund > 0) await creditMcIdempotent(submission.ownerUserId, refund, LedgerOperation.EXTRACT_SITE_REFUND, `Local Sourcebook unused website-page hold: ${submission.id}`, `${billingCycle}:website-unused-refund`);
|
|
13369
|
-
await logRequestEvent({ userId: submission.ownerUserId, source: "local_sourcebook_website", status: "done", query: submission.websiteUrl, resultCount: successfulPages2, result: { submissionId: submission.id, discovered: site.spider.totalFound, captured: successfulPages2, truncated: site.truncated } });
|
|
13370
|
-
} catch (error) {
|
|
13371
|
-
siteError = errorMessage2(error);
|
|
13372
|
-
await creditMcIdempotent(submission.ownerUserId, siteHoldMc, LedgerOperation.EXTRACT_SITE_REFUND, `Local Sourcebook failed website acquisition: ${submission.id}`, `${billingCycle}:website-failure-refund`);
|
|
13373
|
-
siteHoldMc = 0;
|
|
13374
|
-
await logRequestEvent({ userId: submission.ownerUserId, source: "local_sourcebook_website", status: "failed", query: submission.websiteUrl, error: siteError });
|
|
13375
|
-
}
|
|
13376
|
-
}
|
|
13377
|
-
}
|
|
13378
13524
|
try {
|
|
13379
|
-
|
|
13380
|
-
|
|
13381
|
-
|
|
13382
|
-
|
|
13383
|
-
|
|
13384
|
-
|
|
13385
|
-
|
|
13386
|
-
reviewCost,
|
|
13387
|
-
LedgerOperation.MAPS_REVIEW,
|
|
13388
|
-
`Local Sourcebook reviews: ${submission.businessName}`,
|
|
13389
|
-
`${billingCycle}:maps-reviews`
|
|
13390
|
-
);
|
|
13391
|
-
if (debit.ok) {
|
|
13392
|
-
reviewsBilledMc = reviewCost;
|
|
13393
|
-
} else {
|
|
13394
|
-
reviewBillingLimited = true;
|
|
13395
|
-
maps.reviews = [];
|
|
13396
|
-
}
|
|
13525
|
+
try {
|
|
13526
|
+
site = await provider.extractWebsite(submission, maxPages);
|
|
13527
|
+
const successfulPages2 = site.pages.filter(isPageExtractionSuccessful).length;
|
|
13528
|
+
await logRequestEvent({ userId: submission.ownerUserId, source: "local_sourcebook_website", status: "done", query: submission.websiteUrl, resultCount: successfulPages2, result: { submissionId: submission.id, discovered: site.spider.totalFound, captured: successfulPages2, truncated: site.truncated } });
|
|
13529
|
+
} catch (error) {
|
|
13530
|
+
siteError = errorMessage2(error);
|
|
13531
|
+
await logRequestEvent({ userId: submission.ownerUserId, source: "local_sourcebook_website", status: "failed", query: submission.websiteUrl, error: siteError });
|
|
13397
13532
|
}
|
|
13398
|
-
|
|
13533
|
+
try {
|
|
13534
|
+
maps = await provider.extractMaps(submission, LOCAL_SOURCEBOOK_MAX_REVIEWS);
|
|
13535
|
+
validateLocalSourcebookMapsIdentity(submission, maps);
|
|
13536
|
+
await logRequestEvent({ userId: submission.ownerUserId, source: "local_sourcebook_maps", status: "done", query: submission.businessName, location: submission.state.toUpperCase(), resultCount: maps.reviews.length, result: { submissionId: submission.id, placeUrl: maps.placeUrl, cid: maps.cidDecimal ?? maps.cid, reviewsStatus: maps.reviewsStatus, reviewsRetained: maps.reviews.length, services: maps.services.length, areasServed: maps.areasServed.length } });
|
|
13537
|
+
} catch (error) {
|
|
13538
|
+
mapsError = errorMessage2(error);
|
|
13539
|
+
maps = null;
|
|
13540
|
+
await logRequestEvent({ userId: submission.ownerUserId, source: "local_sourcebook_maps", status: "failed", query: submission.businessName, location: submission.state.toUpperCase(), error: mapsError });
|
|
13541
|
+
}
|
|
13542
|
+
if (!maps) throw new Error(`Exact-place acquisition is required before publication. Maps: ${mapsError || "not available"}. Website: ${siteError || (site ? "captured" : "not available")}.`);
|
|
13543
|
+
const compiled = compileLocalSourcebookListing({
|
|
13544
|
+
submission,
|
|
13545
|
+
site,
|
|
13546
|
+
maps,
|
|
13547
|
+
requestedMaxPages: maxPages,
|
|
13548
|
+
requestedMaxReviews: LOCAL_SOURCEBOOK_MAX_REVIEWS,
|
|
13549
|
+
siteError,
|
|
13550
|
+
mapsError,
|
|
13551
|
+
reviewBillingLimited: false
|
|
13552
|
+
});
|
|
13553
|
+
const coverage = {
|
|
13554
|
+
...compiled.coverage,
|
|
13555
|
+
billing: {
|
|
13556
|
+
mode: "fixed_listing_write",
|
|
13557
|
+
kind,
|
|
13558
|
+
credits: price.credits,
|
|
13559
|
+
amountMc: price.mc,
|
|
13560
|
+
debitKey: debitKey2,
|
|
13561
|
+
acquisitionIncluded: true
|
|
13562
|
+
}
|
|
13563
|
+
};
|
|
13564
|
+
const completed = await completeLocalSourcebookAcquisition(submission.id, compiled.listing, coverage, submission.draftRevision);
|
|
13565
|
+
if (!completed) throw new Error("The Local Sourcebook draft could not be saved after acquisition.");
|
|
13566
|
+
return completed;
|
|
13399
13567
|
} catch (error) {
|
|
13400
|
-
|
|
13401
|
-
|
|
13402
|
-
|
|
13403
|
-
|
|
13404
|
-
|
|
13405
|
-
|
|
13406
|
-
|
|
13407
|
-
|
|
13408
|
-
|
|
13409
|
-
|
|
13410
|
-
maps,
|
|
13411
|
-
requestedMaxPages: maxPages,
|
|
13412
|
-
requestedMaxReviews: LOCAL_SOURCEBOOK_MAX_REVIEWS,
|
|
13413
|
-
siteError,
|
|
13414
|
-
mapsError,
|
|
13415
|
-
reviewBillingLimited
|
|
13416
|
-
});
|
|
13417
|
-
const coverage = {
|
|
13418
|
-
...compiled.coverage,
|
|
13419
|
-
billing: {
|
|
13420
|
-
websiteHeldMc: siteHoldMc,
|
|
13421
|
-
websiteUsedMc: siteActualMc,
|
|
13422
|
-
mapsUsedMc: mapsRefunded ? 0 : MC_COSTS.maps_place,
|
|
13423
|
-
reviewsUsedMc: reviewsBilledMc,
|
|
13424
|
-
reviewBillingLimited
|
|
13568
|
+
const current = await getLocalSourcebookSubmission(submission.id).catch(() => null);
|
|
13569
|
+
const revisionAlreadyPublished = current?.status === "published" && current.publishedRevision != null && current.draftRevision > submission.draftRevision;
|
|
13570
|
+
if (!revisionAlreadyPublished) {
|
|
13571
|
+
await refundPersistedListingWrite({
|
|
13572
|
+
surface: "local_sourcebook",
|
|
13573
|
+
kind,
|
|
13574
|
+
userId: submission.ownerUserId,
|
|
13575
|
+
debitKey: debitKey2,
|
|
13576
|
+
reason: `Local Sourcebook acquisition failed: ${errorMessage2(error)}`
|
|
13577
|
+
}).catch(() => null);
|
|
13425
13578
|
}
|
|
13426
|
-
|
|
13427
|
-
|
|
13428
|
-
if (!completed) throw new Error("The Local Sourcebook draft could not be saved after acquisition.");
|
|
13429
|
-
return completed;
|
|
13579
|
+
throw error;
|
|
13580
|
+
}
|
|
13430
13581
|
}
|
|
13431
13582
|
async function processLocalSourcebookSubmission(submissionId) {
|
|
13432
13583
|
const submission = await claimLocalSourcebookAcquisition(submissionId);
|
|
@@ -21322,6 +21473,7 @@ var DirectoryIdentitySchema = z25.object({
|
|
|
21322
21473
|
state: z25.string().trim().length(2).transform((value) => value.toLowerCase()),
|
|
21323
21474
|
businessName: z25.string().trim().min(2).max(160),
|
|
21324
21475
|
websiteUrl: z25.url().refine((value) => /^https?:\/\//.test(value), "websiteUrl must use http or https"),
|
|
21476
|
+
schemaOrgType: LocalSourcebookSchemaTypeSchema.optional(),
|
|
21325
21477
|
slug: z25.string().trim().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/).max(100).optional(),
|
|
21326
21478
|
tags: z25.array(z25.string().trim().min(1).max(60)).max(20).default([]),
|
|
21327
21479
|
idempotencyKey: z25.string().trim().min(8).max(200)
|
|
@@ -21406,7 +21558,7 @@ async function resolveLocalSourcebookTags(candidates) {
|
|
|
21406
21558
|
function getLocalSourcebookContract(category) {
|
|
21407
21559
|
return {
|
|
21408
21560
|
name: "Local Sourcebook listing",
|
|
21409
|
-
version: "
|
|
21561
|
+
version: "local_sourcebook_listing_v3",
|
|
21410
21562
|
purpose: "An owner-scoped, evidence-backed local business record with a separate public profile and complete captured-review archive.",
|
|
21411
21563
|
categories: LOCAL_SOURCEBOOK_CATEGORIES,
|
|
21412
21564
|
selectedCategory: category ?? null,
|
|
@@ -21417,11 +21569,14 @@ function getLocalSourcebookContract(category) {
|
|
|
21417
21569
|
reviews: "https://{category}.localsourcebook.com/{state}/{business-slug}/reviews"
|
|
21418
21570
|
},
|
|
21419
21571
|
requiredCreateFields: ["category", "state", "businessName", "websiteUrl", "tags", "idempotencyKey"],
|
|
21420
|
-
|
|
21572
|
+
optionalCreateFields: ["schemaOrgType"],
|
|
21573
|
+
systemCompiledSections: ["identity", "LocalBusiness JSON-LD", "services/products", "service areas/locations", "genuine media", "reviews and theme counts", "evidence", "answered FAQ", "collection receipts"],
|
|
21421
21574
|
workflow: ["list-local-sourcebook-tags", "get-local-sourcebook-contract", "prepare-local-sourcebook-write", "validate-local-sourcebook-write", "local-sourcebook-capture", "local_sourcebook_submission_status"],
|
|
21422
21575
|
ownership: "Only the MCP Scraper account that captures a listing can read it or request a paid evidence refresh.",
|
|
21576
|
+
billing: { validationCredits: 0, createCredits: 10, editCredits: 2, acquisitionIncluded: true, chargeEvent: "accepted_write" },
|
|
21423
21577
|
acquisition: { websitePagesMaximum: 100, reviewsMaximum: 500, exactMapsIdentityRequired: true, partialResultsRequireStoppingReasons: true },
|
|
21424
21578
|
publication: { automatic: true, evidenceCompiledOnly: true, ownerAuthoredClaimsAccepted: false, adminReviewRequired: false, lastPublishedRevisionRemainsPublicDuringRefresh: true },
|
|
21579
|
+
structuredData: localSourcebookSchemaContract(category),
|
|
21425
21580
|
tagPolicy: { inspectVocabularyFirst: true, minimum: 1, maximum: 20, newTagsRequire: ["central=true", "reusable=true", "description"], newTagsStartAs: "pending" }
|
|
21426
21581
|
};
|
|
21427
21582
|
}
|
|
@@ -21436,17 +21591,20 @@ async function prepareLocalSourcebookWrite(input) {
|
|
|
21436
21591
|
];
|
|
21437
21592
|
const tagResolutions = await resolveLocalSourcebookTags(candidates);
|
|
21438
21593
|
const publicUrls = localSourcebookPublicUrls(parsed.category, parsed.state, slug2);
|
|
21594
|
+
const schemaOrgType = parsed.schemaOrgType ?? defaultLocalSourcebookSchemaType(parsed.category);
|
|
21439
21595
|
return {
|
|
21440
21596
|
ok: true,
|
|
21441
21597
|
route: { category: parsed.category, state: parsed.state, slug: slug2, ...publicUrls },
|
|
21442
|
-
proposedWrite: { ...parsed, slug: slug2, tags: tagResolutions.filter((item) => item.action === "reuse").map((item) => item.tag) },
|
|
21598
|
+
proposedWrite: { ...parsed, schemaOrgType, slug: slug2, tags: tagResolutions.filter((item) => item.action === "reuse").map((item) => item.tag) },
|
|
21443
21599
|
contract: getLocalSourcebookContract(parsed.category),
|
|
21444
21600
|
tagResolutions,
|
|
21445
21601
|
instructions: [
|
|
21446
21602
|
"Resolve every review candidate before capture; do not silently create a near-duplicate tag.",
|
|
21447
21603
|
"Call validate-local-sourcebook-write with the proposed identity, resolved tags, and any required tag decisions.",
|
|
21604
|
+
"Keep the proposed schemaOrgType or choose a more specific accepted LocalBusiness subtype from the contract. Arbitrary JSON-LD is not accepted.",
|
|
21448
21605
|
"Call local-sourcebook-capture only after validation returns valid=true. Capture queues paid acquisition and automatically publishes the resulting system-compiled evidence revision.",
|
|
21449
|
-
"
|
|
21606
|
+
"Creating the listing costs 10 Credits total, including acquisition. Validation is free.",
|
|
21607
|
+
"Owners cannot supply public listing claims directly. A source-backed local_sourcebook_refresh costs 2 Credits."
|
|
21450
21608
|
]
|
|
21451
21609
|
};
|
|
21452
21610
|
}
|
|
@@ -21456,7 +21614,10 @@ async function validateLocalSourcebookWrite(input) {
|
|
|
21456
21614
|
let normalizedIdentity2;
|
|
21457
21615
|
const parsedIdentity = DirectoryIdentitySchema.safeParse(input.identity);
|
|
21458
21616
|
if (!parsedIdentity.success) errors.push(...parsedIdentity.error.issues.map((issue) => `identity.${issue.path.join(".")}: ${issue.message}`));
|
|
21459
|
-
else normalizedIdentity2 =
|
|
21617
|
+
else normalizedIdentity2 = {
|
|
21618
|
+
...parsedIdentity.data,
|
|
21619
|
+
schemaOrgType: parsedIdentity.data.schemaOrgType ?? defaultLocalSourcebookSchemaType(parsedIdentity.data.category)
|
|
21620
|
+
};
|
|
21460
21621
|
const candidates = input.tagCandidates ?? (normalizedIdentity2?.tags.map((tag) => ({ tag })) ?? []);
|
|
21461
21622
|
const resolutions = await resolveLocalSourcebookTags(candidates);
|
|
21462
21623
|
const decisions = new Map((input.tagDecisions ?? []).map((decision) => [normalizeTag(decision.tag), decision]));
|
|
@@ -21505,6 +21666,7 @@ var SubmitSchema = z26.object({
|
|
|
21505
21666
|
state: z26.string().trim().length(2).transform((value) => value.toLowerCase()),
|
|
21506
21667
|
businessName: z26.string().trim().min(2).max(160),
|
|
21507
21668
|
websiteUrl: z26.url().refine((value) => /^https?:\/\//.test(value), "websiteUrl must use http or https"),
|
|
21669
|
+
schemaOrgType: LocalSourcebookSchemaTypeSchema.optional(),
|
|
21508
21670
|
slug: z26.string().trim().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/).max(100).optional(),
|
|
21509
21671
|
idempotencyKey: z26.string().trim().min(8).max(200).optional()
|
|
21510
21672
|
}).strict();
|
|
@@ -21561,13 +21723,19 @@ localSourcebookApp.post("/capture", createApiKeyAuth(), async (c) => {
|
|
|
21561
21723
|
if (!parsed.success) return c.json({ error: "invalid_capture_request", issues: parsed.error.issues }, 400);
|
|
21562
21724
|
const validation = await validateLocalSourcebookWrite(parsed.data);
|
|
21563
21725
|
if (!validation.valid) return c.json({ error: "invalid_directory_write", ...validation }, 400);
|
|
21726
|
+
let billing = null;
|
|
21727
|
+
let persisted = false;
|
|
21564
21728
|
try {
|
|
21565
21729
|
const identity = validation.normalizedIdentity;
|
|
21566
|
-
|
|
21730
|
+
const idempotencyKey3 = parsed.data.idempotencyKey ?? identity.idempotencyKey;
|
|
21731
|
+
if (!idempotencyKey3) {
|
|
21567
21732
|
return c.json({ error: "idempotency_key_required", message: "A stable idempotencyKey is required for a new Local Sourcebook capture." }, 400);
|
|
21568
21733
|
}
|
|
21569
21734
|
const listingSlug = identity.slug ?? slug(identity.businessName);
|
|
21570
21735
|
const publicUrls = localSourcebookPublicUrls(identity.category, identity.state, listingSlug);
|
|
21736
|
+
const charged = await chargeListingWrite({ surface: "local_sourcebook", kind: "create", userId: Number(user.id), idempotencyKey: idempotencyKey3 });
|
|
21737
|
+
if ("insufficient" in charged) return c.json(charged.insufficient, 402);
|
|
21738
|
+
billing = charged;
|
|
21571
21739
|
const submission = await createLocalSourcebookSubmission({
|
|
21572
21740
|
ownerUserId: Number(user.id),
|
|
21573
21741
|
category: identity.category,
|
|
@@ -21575,7 +21743,9 @@ localSourcebookApp.post("/capture", createApiKeyAuth(), async (c) => {
|
|
|
21575
21743
|
slug: listingSlug,
|
|
21576
21744
|
businessName: identity.businessName,
|
|
21577
21745
|
websiteUrl: identity.websiteUrl,
|
|
21578
|
-
|
|
21746
|
+
schemaOrgType: identity.schemaOrgType,
|
|
21747
|
+
idempotencyKey: idempotencyKey3,
|
|
21748
|
+
billing: { kind: billing.kind, credits: billing.credits, amountMc: billing.amountMc, debitKey: billing.debitKey },
|
|
21579
21749
|
payload: {
|
|
21580
21750
|
id: "",
|
|
21581
21751
|
category: identity.category,
|
|
@@ -21583,6 +21753,7 @@ localSourcebookApp.post("/capture", createApiKeyAuth(), async (c) => {
|
|
|
21583
21753
|
slug: listingSlug,
|
|
21584
21754
|
name: identity.businessName,
|
|
21585
21755
|
websiteUrl: identity.websiteUrl,
|
|
21756
|
+
schemaOrgType: identity.schemaOrgType,
|
|
21586
21757
|
tags: validation.normalizedTags,
|
|
21587
21758
|
canonicalPaths: { profile: publicUrls.profilePath, reviews: publicUrls.reviewsPath },
|
|
21588
21759
|
canonicalUrls: { profile: publicUrls.profileUrl, reviews: publicUrls.reviewsUrl },
|
|
@@ -21593,10 +21764,12 @@ localSourcebookApp.post("/capture", createApiKeyAuth(), async (c) => {
|
|
|
21593
21764
|
reviews: []
|
|
21594
21765
|
}
|
|
21595
21766
|
});
|
|
21767
|
+
persisted = true;
|
|
21596
21768
|
await persistLocalSourcebookTags(submission.id, Number(user.id), validation.normalizedTags, parsed.data.tagDecisions);
|
|
21597
21769
|
const dispatched = await dispatchLocalSourcebookAcquisition(submission.id).catch(() => false);
|
|
21598
|
-
return c.json({ ok: true, valid: true, submission, publicUrls: localSourcebookPublicationLinks(submission), normalizedTags: validation.normalizedTags, acquisition: { queued: true, dispatched }, acquisitionPlan: acquisitionPlan(identity.websiteUrl) }, 202);
|
|
21770
|
+
return c.json({ ok: true, valid: true, submission, publicUrls: localSourcebookPublicationLinks(submission), normalizedTags: validation.normalizedTags, acquisition: { queued: true, dispatched }, acquisitionPlan: acquisitionPlan(identity.websiteUrl), billing: publicListingWriteBilling(billing) }, 202);
|
|
21599
21771
|
} catch (error) {
|
|
21772
|
+
if (billing && !persisted) await refundListingWrite(billing, "Local Sourcebook creation failed before persistence.").catch(() => null);
|
|
21600
21773
|
const message = error instanceof Error ? error.message : String(error);
|
|
21601
21774
|
const conflict = message.includes("revision conflict") ? "revision_conflict" : message.includes("idempotency key conflicts") ? "idempotency_conflict" : null;
|
|
21602
21775
|
return c.json({ error: conflict ?? "capture_failed", message }, conflict ? 409 : 400);
|
|
@@ -21608,13 +21781,22 @@ localSourcebookApp.post("/submissions", createApiKeyAuth(), async (c) => {
|
|
|
21608
21781
|
const parsed = SubmitSchema.safeParse(await c.req.json().catch(() => null));
|
|
21609
21782
|
if (!parsed.success) return c.json({ error: "invalid_submission", issues: parsed.error.issues }, 400);
|
|
21610
21783
|
const input = parsed.data;
|
|
21784
|
+
if (!input.idempotencyKey) return c.json({ error: "idempotency_key_required", message: "A stable idempotencyKey is required for a new Local Sourcebook listing." }, 400);
|
|
21611
21785
|
const listingSlug = input.slug ?? slug(input.businessName);
|
|
21786
|
+
const schemaOrgType = input.schemaOrgType ?? defaultLocalSourcebookSchemaType(input.category);
|
|
21612
21787
|
const publicUrls = localSourcebookPublicUrls(input.category, input.state, listingSlug);
|
|
21788
|
+
let billing = null;
|
|
21789
|
+
let persisted = false;
|
|
21613
21790
|
try {
|
|
21791
|
+
const charged = await chargeListingWrite({ surface: "local_sourcebook", kind: "create", userId: Number(user.id), idempotencyKey: input.idempotencyKey });
|
|
21792
|
+
if ("insufficient" in charged) return c.json(charged.insufficient, 402);
|
|
21793
|
+
billing = charged;
|
|
21614
21794
|
const submission = await createLocalSourcebookSubmission({
|
|
21615
21795
|
ownerUserId: Number(user.id),
|
|
21616
21796
|
...input,
|
|
21797
|
+
schemaOrgType,
|
|
21617
21798
|
slug: listingSlug,
|
|
21799
|
+
billing: { kind: billing.kind, credits: billing.credits, amountMc: billing.amountMc, debitKey: billing.debitKey },
|
|
21618
21800
|
payload: {
|
|
21619
21801
|
id: "",
|
|
21620
21802
|
category: input.category,
|
|
@@ -21622,6 +21804,7 @@ localSourcebookApp.post("/submissions", createApiKeyAuth(), async (c) => {
|
|
|
21622
21804
|
slug: listingSlug,
|
|
21623
21805
|
name: input.businessName,
|
|
21624
21806
|
websiteUrl: input.websiteUrl,
|
|
21807
|
+
schemaOrgType,
|
|
21625
21808
|
canonicalPaths: { profile: publicUrls.profilePath, reviews: publicUrls.reviewsPath },
|
|
21626
21809
|
canonicalUrls: { profile: publicUrls.profileUrl, reviews: publicUrls.reviewsUrl },
|
|
21627
21810
|
verificationState: "submitted",
|
|
@@ -21631,9 +21814,11 @@ localSourcebookApp.post("/submissions", createApiKeyAuth(), async (c) => {
|
|
|
21631
21814
|
reviews: []
|
|
21632
21815
|
}
|
|
21633
21816
|
});
|
|
21817
|
+
persisted = true;
|
|
21634
21818
|
const dispatched = await dispatchLocalSourcebookAcquisition(submission.id).catch(() => false);
|
|
21635
|
-
return c.json({ ok: true, submission, publicUrls: localSourcebookPublicationLinks(submission), acquisition: { queued: true, dispatched }, acquisitionPlan: acquisitionPlan(input.websiteUrl) }, 202);
|
|
21819
|
+
return c.json({ ok: true, submission, publicUrls: localSourcebookPublicationLinks(submission), acquisition: { queued: true, dispatched }, acquisitionPlan: acquisitionPlan(input.websiteUrl), billing: publicListingWriteBilling(billing) }, 202);
|
|
21636
21820
|
} catch (error) {
|
|
21821
|
+
if (billing && !persisted) await refundListingWrite(billing, "Local Sourcebook creation failed before persistence.").catch(() => null);
|
|
21637
21822
|
return c.json({ error: "submission_conflict", message: error instanceof Error ? error.message : String(error) }, 409);
|
|
21638
21823
|
}
|
|
21639
21824
|
});
|
|
@@ -21650,12 +21835,35 @@ localSourcebookApp.on(["PATCH", "POST"], "/submissions/:id/draft", createApiKeyA
|
|
|
21650
21835
|
localSourcebookApp.post("/submissions/:id/refresh", createApiKeyAuth(), async (c) => {
|
|
21651
21836
|
const user = c.get("user");
|
|
21652
21837
|
if (!user.subscription_tier) return c.json({ error: "subscription_required" }, 402);
|
|
21838
|
+
const userId = Number(user.id);
|
|
21839
|
+
const existing = await getLocalSourcebookSubmission(c.req.param("id"), userId);
|
|
21840
|
+
if (!existing) return c.json({ error: "not_found" }, 404);
|
|
21841
|
+
if (existing.status === "rejected" || existing.status === "unpublished") {
|
|
21842
|
+
return c.json({ error: "moderated_listing", message: `A ${existing.status} Local Sourcebook listing cannot be refreshed by its owner.` }, 409);
|
|
21843
|
+
}
|
|
21844
|
+
if (existing.status === "queued" || existing.status === "enriching") {
|
|
21845
|
+
return c.json({ error: "acquisition_in_progress", message: "This listing already has an acquisition in progress." }, 409);
|
|
21846
|
+
}
|
|
21847
|
+
const idempotencyKey3 = `refresh:${existing.id}:revision:${existing.draftRevision}`;
|
|
21848
|
+
let billing = null;
|
|
21653
21849
|
try {
|
|
21654
|
-
const
|
|
21655
|
-
if (
|
|
21850
|
+
const charged = await chargeListingWrite({ surface: "local_sourcebook", kind: "edit", userId, idempotencyKey: idempotencyKey3 });
|
|
21851
|
+
if ("insufficient" in charged) return c.json(charged.insufficient, 402);
|
|
21852
|
+
billing = charged;
|
|
21853
|
+
const submission = await queueLocalSourcebookRefresh(c.req.param("id"), userId, {
|
|
21854
|
+
kind: billing.kind,
|
|
21855
|
+
credits: billing.credits,
|
|
21856
|
+
amountMc: billing.amountMc,
|
|
21857
|
+
debitKey: billing.debitKey
|
|
21858
|
+
});
|
|
21859
|
+
if (!submission) {
|
|
21860
|
+
await refundListingWrite(billing, "Local Sourcebook listing was not found before refresh persistence.").catch(() => null);
|
|
21861
|
+
return c.json({ error: "not_found" }, 404);
|
|
21862
|
+
}
|
|
21656
21863
|
const dispatched = await dispatchLocalSourcebookAcquisition(submission.id).catch(() => false);
|
|
21657
|
-
return c.json({ ok: true, submission, publicUrls: localSourcebookPublicationLinks(submission), acquisition: { queued: true, dispatched }, acquisitionPlan: acquisitionPlan(submission.websiteUrl) }, 202);
|
|
21864
|
+
return c.json({ ok: true, submission, publicUrls: localSourcebookPublicationLinks(submission), acquisition: { queued: true, dispatched }, acquisitionPlan: acquisitionPlan(submission.websiteUrl), billing: publicListingWriteBilling(billing) }, 202);
|
|
21658
21865
|
} catch (error) {
|
|
21866
|
+
if (billing) await refundListingWrite(billing, "Local Sourcebook refresh failed before persistence.").catch(() => null);
|
|
21659
21867
|
return c.json({ error: "moderated_listing", message: error instanceof Error ? error.message : String(error) }, 409);
|
|
21660
21868
|
}
|
|
21661
21869
|
});
|
|
@@ -30641,7 +30849,7 @@ import { z as z32 } from "zod";
|
|
|
30641
30849
|
|
|
30642
30850
|
// src/api/commons-repository.ts
|
|
30643
30851
|
import { createHash as createHash12, randomUUID as randomUUID15 } from "crypto";
|
|
30644
|
-
var COMMONS_SCHEMA_VERSION = "2026-08-04.
|
|
30852
|
+
var COMMONS_SCHEMA_VERSION = "2026-08-04.3";
|
|
30645
30853
|
var DEFAULT_COMMONS_BASE_URL = "https://transparent-commons.cc";
|
|
30646
30854
|
var DEFAULT_ENTITY_TYPE = "PublicArticle";
|
|
30647
30855
|
var COMMONS_ENTITY_PROFILES = {
|
|
@@ -30923,6 +31131,54 @@ async function ensureCommonsSchema() {
|
|
|
30923
31131
|
},
|
|
30924
31132
|
{ sql: `CREATE INDEX IF NOT EXISTS commons_index_documents_status_updated ON commons_index_documents(embedding_status, updated_at DESC)`, args: [] },
|
|
30925
31133
|
{ sql: `CREATE INDEX IF NOT EXISTS commons_index_documents_entity_type ON commons_index_documents(entity_id, document_type)`, args: [] },
|
|
31134
|
+
{
|
|
31135
|
+
sql: `
|
|
31136
|
+
CREATE TABLE IF NOT EXISTS commons_publications (
|
|
31137
|
+
id TEXT PRIMARY KEY,
|
|
31138
|
+
owner_user_id INTEGER NOT NULL REFERENCES users(id),
|
|
31139
|
+
subdomain TEXT NOT NULL UNIQUE,
|
|
31140
|
+
title TEXT NOT NULL,
|
|
31141
|
+
description TEXT NOT NULL DEFAULT '',
|
|
31142
|
+
claim_idempotency_key TEXT NOT NULL,
|
|
31143
|
+
latest_edition_id TEXT,
|
|
31144
|
+
created_at TEXT NOT NULL,
|
|
31145
|
+
updated_at TEXT NOT NULL,
|
|
31146
|
+
UNIQUE(owner_user_id),
|
|
31147
|
+
UNIQUE(owner_user_id, claim_idempotency_key)
|
|
31148
|
+
)
|
|
31149
|
+
`,
|
|
31150
|
+
args: []
|
|
31151
|
+
},
|
|
31152
|
+
{ sql: `CREATE INDEX IF NOT EXISTS commons_publications_updated ON commons_publications(updated_at DESC)`, args: [] },
|
|
31153
|
+
{
|
|
31154
|
+
sql: `
|
|
31155
|
+
CREATE TABLE IF NOT EXISTS commons_publication_editions (
|
|
31156
|
+
id TEXT PRIMARY KEY,
|
|
31157
|
+
publication_id TEXT NOT NULL REFERENCES commons_publications(id),
|
|
31158
|
+
owner_user_id INTEGER NOT NULL REFERENCES users(id),
|
|
31159
|
+
edition_slug TEXT NOT NULL,
|
|
31160
|
+
revision INTEGER NOT NULL,
|
|
31161
|
+
title TEXT NOT NULL,
|
|
31162
|
+
site_json TEXT NOT NULL,
|
|
31163
|
+
deck TEXT NOT NULL,
|
|
31164
|
+
articles_json TEXT NOT NULL,
|
|
31165
|
+
html TEXT NOT NULL,
|
|
31166
|
+
filename TEXT NOT NULL,
|
|
31167
|
+
sha256 TEXT NOT NULL,
|
|
31168
|
+
article_count INTEGER NOT NULL,
|
|
31169
|
+
word_count INTEGER NOT NULL,
|
|
31170
|
+
bytes INTEGER NOT NULL,
|
|
31171
|
+
warnings_json TEXT NOT NULL DEFAULT '[]',
|
|
31172
|
+
idempotency_key TEXT NOT NULL,
|
|
31173
|
+
created_at TEXT NOT NULL,
|
|
31174
|
+
published_at TEXT NOT NULL,
|
|
31175
|
+
UNIQUE(publication_id, edition_slug, revision),
|
|
31176
|
+
UNIQUE(owner_user_id, idempotency_key)
|
|
31177
|
+
)
|
|
31178
|
+
`,
|
|
31179
|
+
args: []
|
|
31180
|
+
},
|
|
31181
|
+
{ sql: `CREATE INDEX IF NOT EXISTS commons_publication_editions_publication_published ON commons_publication_editions(publication_id, published_at DESC)`, args: [] },
|
|
30926
31182
|
{
|
|
30927
31183
|
sql: "INSERT OR IGNORE INTO schema_migrations (version) VALUES (?)",
|
|
30928
31184
|
args: [COMMONS_SCHEMA_VERSION]
|
|
@@ -31027,6 +31283,12 @@ async function prepareCommonsEntity(input, user) {
|
|
|
31027
31283
|
writePath: "commons_prepare_entity -> commons_validate_entity -> commons_submit_entity"
|
|
31028
31284
|
},
|
|
31029
31285
|
contract,
|
|
31286
|
+
billing: {
|
|
31287
|
+
validationCredits: LISTING_WRITE_RATE_POLICY.validationCredits,
|
|
31288
|
+
createCredits: LISTING_WRITE_RATE_POLICY.create.credits,
|
|
31289
|
+
editCredits: LISTING_WRITE_RATE_POLICY.edit.credits,
|
|
31290
|
+
chargeEvent: LISTING_WRITE_RATE_POLICY.chargeEvent
|
|
31291
|
+
},
|
|
31030
31292
|
duplicateCandidates,
|
|
31031
31293
|
tagResolutions,
|
|
31032
31294
|
headingPlan: {
|
|
@@ -31041,7 +31303,8 @@ async function prepareCommonsEntity(input, user) {
|
|
|
31041
31303
|
"Use the returned profile as a contract, not a rigid empty template: do not include History, Reception, Pricing, Timeline, or similar sections unless source evidence supports them.",
|
|
31042
31304
|
"Write neutral encyclopedia prose. Do not paste raw scrape Markdown, marketing layout fragments, or empty/filler headings.",
|
|
31043
31305
|
"Attach a featured image for auto-publish eligibility, preserve original URL/source byline/relCanonical when republishing, and put media assets in the media array.",
|
|
31044
|
-
"Call commons_validate_entity with the composed payload before commons_submit_entity."
|
|
31306
|
+
"Call commons_validate_entity with the composed payload before commons_submit_entity.",
|
|
31307
|
+
"Validation is free. Submitting a new entity costs 10 Credits; submitting an edit to an existing entity costs 2 Credits."
|
|
31045
31308
|
]
|
|
31046
31309
|
};
|
|
31047
31310
|
}
|
|
@@ -31130,6 +31393,14 @@ async function getCommonsEntity(idOrSlug) {
|
|
|
31130
31393
|
});
|
|
31131
31394
|
return result.rows[0] ? rowToEntity(result.rows[0]) : null;
|
|
31132
31395
|
}
|
|
31396
|
+
async function classifyCommonsWrite(input, user) {
|
|
31397
|
+
await ensureCommonsSchema();
|
|
31398
|
+
const normalized = normalizeSubmitInput(input);
|
|
31399
|
+
const existingProposal = await findProposalByIdempotency(user.id, normalized.idempotencyKey);
|
|
31400
|
+
if (existingProposal) return existingProposal.proposalType === "edit_entity" ? "edit" : "create";
|
|
31401
|
+
const existing = await findEntityBySlugOrIdAnyStatus(normalized.slug, normalized.entityId);
|
|
31402
|
+
return existing ? "edit" : "create";
|
|
31403
|
+
}
|
|
31133
31404
|
async function submitCommonsEntity(input, user) {
|
|
31134
31405
|
await ensureCommonsSchema();
|
|
31135
31406
|
const normalized = normalizeSubmitInput(input);
|
|
@@ -31394,7 +31665,8 @@ function commonsDatabaseReport() {
|
|
|
31394
31665
|
normalWorkflow: ["commons_prepare_entity", "commons_validate_entity", "commons_submit_entity"],
|
|
31395
31666
|
writesCreate: ["commons_proposals", "commons_ledger", "commons_entities when accepted"],
|
|
31396
31667
|
safeAutoPublish: "New non-conflicting entities with title, description, featured image, and body/source evidence.",
|
|
31397
|
-
conflictBehavior: "Existing entity edits require baseRevision. Otherwise the proposal is pending_review."
|
|
31668
|
+
conflictBehavior: "Existing entity edits require baseRevision. Otherwise the proposal is pending_review.",
|
|
31669
|
+
billing: { validationCredits: 0, createCredits: LISTING_WRITE_RATE_POLICY.create.credits, editCredits: LISTING_WRITE_RATE_POLICY.edit.credits }
|
|
31398
31670
|
},
|
|
31399
31671
|
searchCorpus: ["title", "disambiguationName", "description", "tags", "keywords", "relatedEntities", "bodyMarkdown", "sections", "media", "source metadata", "jsonLd"],
|
|
31400
31672
|
indexing: commonsIndexingPolicy(),
|
|
@@ -32484,6 +32756,392 @@ var CommonsRepositoryError = class extends Error {
|
|
|
32484
32756
|
httpStatus;
|
|
32485
32757
|
};
|
|
32486
32758
|
|
|
32759
|
+
// src/api/commons-publication-repository.ts
|
|
32760
|
+
import { createHash as createHash13, randomUUID as randomUUID16 } from "crypto";
|
|
32761
|
+
var PUBLICATION_ROOT_DOMAIN = process.env.COMMONS_PUBLICATION_ROOT_DOMAIN || "transparent-commons.cc";
|
|
32762
|
+
var RESERVED_SUBDOMAINS = /* @__PURE__ */ new Set([
|
|
32763
|
+
"admin",
|
|
32764
|
+
"api",
|
|
32765
|
+
"app",
|
|
32766
|
+
"assets",
|
|
32767
|
+
"auth",
|
|
32768
|
+
"blog",
|
|
32769
|
+
"cdn",
|
|
32770
|
+
"docs",
|
|
32771
|
+
"help",
|
|
32772
|
+
"login",
|
|
32773
|
+
"mail",
|
|
32774
|
+
"mcp",
|
|
32775
|
+
"signup",
|
|
32776
|
+
"sitemap",
|
|
32777
|
+
"static",
|
|
32778
|
+
"support",
|
|
32779
|
+
"transparent-commons",
|
|
32780
|
+
"transparentcommons",
|
|
32781
|
+
"www",
|
|
32782
|
+
"wiki"
|
|
32783
|
+
]);
|
|
32784
|
+
var CommonsPublicationError = class extends Error {
|
|
32785
|
+
constructor(code, message, httpStatus = 400) {
|
|
32786
|
+
super(message);
|
|
32787
|
+
this.code = code;
|
|
32788
|
+
this.httpStatus = httpStatus;
|
|
32789
|
+
this.name = "CommonsPublicationError";
|
|
32790
|
+
}
|
|
32791
|
+
code;
|
|
32792
|
+
httpStatus;
|
|
32793
|
+
};
|
|
32794
|
+
async function prepareCommonsPublication(input, user) {
|
|
32795
|
+
await ensureCommonsSchema();
|
|
32796
|
+
const subdomain = normalizePublicationSubdomain(input.requestedSubdomain);
|
|
32797
|
+
const [owned, claimed] = await Promise.all([
|
|
32798
|
+
getCommonsPublicationForOwner(Number(user.id)),
|
|
32799
|
+
getCommonsPublicationBySubdomain(subdomain)
|
|
32800
|
+
]);
|
|
32801
|
+
const availability = claimed ? claimed.ownerUserId === Number(user.id) ? "owned_by_caller" : "unavailable" : "available";
|
|
32802
|
+
return {
|
|
32803
|
+
requestedSubdomain: input.requestedSubdomain,
|
|
32804
|
+
normalizedSubdomain: subdomain,
|
|
32805
|
+
availability,
|
|
32806
|
+
requestedPublicUrl: publicationPublicUrl(subdomain),
|
|
32807
|
+
currentPublication: owned,
|
|
32808
|
+
contract: {
|
|
32809
|
+
ownership: "One publication per authenticated MCP Scraper account.",
|
|
32810
|
+
permanence: "Claimed names are stable and globally unique.",
|
|
32811
|
+
renderer: "Published editions use the MCP Scraper editorial reading-room renderer.",
|
|
32812
|
+
revisionRule: "Editing an existing edition requires its current baseRevision.",
|
|
32813
|
+
publicRoutes: ["/", "/archive", "/editions/{editionSlug}"],
|
|
32814
|
+
workflow: [
|
|
32815
|
+
"commons_prepare_publication",
|
|
32816
|
+
"commons_validate_publication",
|
|
32817
|
+
"commons_claim_publication",
|
|
32818
|
+
"commons_publish_editorial"
|
|
32819
|
+
]
|
|
32820
|
+
},
|
|
32821
|
+
proposed: {
|
|
32822
|
+
title: cleanText(input.title, 140) || publicationTitleFromSubdomain(subdomain),
|
|
32823
|
+
description: cleanText(input.description, 500)
|
|
32824
|
+
}
|
|
32825
|
+
};
|
|
32826
|
+
}
|
|
32827
|
+
async function validateCommonsPublication(input, user) {
|
|
32828
|
+
await ensureCommonsSchema();
|
|
32829
|
+
const errors = [];
|
|
32830
|
+
const warnings = [];
|
|
32831
|
+
let subdomain = "";
|
|
32832
|
+
try {
|
|
32833
|
+
subdomain = normalizePublicationSubdomain(input.requestedSubdomain || input.publicationSubdomain || "");
|
|
32834
|
+
} catch (error) {
|
|
32835
|
+
errors.push(error instanceof Error ? error.message : String(error));
|
|
32836
|
+
}
|
|
32837
|
+
const publication = subdomain ? await getCommonsPublicationBySubdomain(subdomain) : null;
|
|
32838
|
+
if (input.operation === "claim") {
|
|
32839
|
+
const owned = await getCommonsPublicationForOwner(Number(user.id));
|
|
32840
|
+
if (publication && publication.ownerUserId !== Number(user.id)) errors.push("That publication name is already claimed by another account.");
|
|
32841
|
+
if (owned && owned.subdomain !== subdomain) errors.push(`This account already owns ${owned.publicUrl}.`);
|
|
32842
|
+
if (!cleanText(input.title, 140)) warnings.push("No display title was supplied; the publication name will be title-cased.");
|
|
32843
|
+
} else {
|
|
32844
|
+
if (!publication) errors.push("Claim this publication name before publishing an edition.");
|
|
32845
|
+
else if (publication.ownerUserId !== Number(user.id)) errors.push("Only the account that claimed this publication can publish to it.");
|
|
32846
|
+
if (!input.edition) errors.push("A complete editorial reading-room edition is required for publish validation.");
|
|
32847
|
+
if (input.edition) {
|
|
32848
|
+
try {
|
|
32849
|
+
renderEditorialReadingRoom(input.edition);
|
|
32850
|
+
} catch (error) {
|
|
32851
|
+
errors.push(error instanceof Error ? error.message : String(error));
|
|
32852
|
+
}
|
|
32853
|
+
const slug2 = normalizeEditionSlug(input.editionSlug || input.edition.site.slug);
|
|
32854
|
+
const latest = publication ? await getLatestEdition(publication.id, slug2) : null;
|
|
32855
|
+
if (latest && input.baseRevision === void 0) errors.push(`Edition ${slug2} already exists at revision ${latest.revision}; supply baseRevision to edit it.`);
|
|
32856
|
+
if (latest && input.baseRevision !== void 0 && latest.revision !== input.baseRevision) {
|
|
32857
|
+
errors.push(`Edition ${slug2} is revision ${latest.revision}; the proposed edit targeted ${input.baseRevision}.`);
|
|
32858
|
+
}
|
|
32859
|
+
}
|
|
32860
|
+
}
|
|
32861
|
+
return {
|
|
32862
|
+
valid: errors.length === 0,
|
|
32863
|
+
operation: input.operation,
|
|
32864
|
+
normalizedSubdomain: subdomain || null,
|
|
32865
|
+
publicUrl: subdomain ? publicationPublicUrl(subdomain) : null,
|
|
32866
|
+
errors,
|
|
32867
|
+
warnings,
|
|
32868
|
+
publication
|
|
32869
|
+
};
|
|
32870
|
+
}
|
|
32871
|
+
async function claimCommonsPublication(input, user) {
|
|
32872
|
+
await ensureCommonsSchema();
|
|
32873
|
+
const subdomain = normalizePublicationSubdomain(input.requestedSubdomain);
|
|
32874
|
+
const userId = Number(user.id);
|
|
32875
|
+
const existingOwned = await getCommonsPublicationForOwner(userId);
|
|
32876
|
+
if (existingOwned) {
|
|
32877
|
+
if (existingOwned.subdomain === subdomain) return { publication: existingOwned, idempotentReplay: true };
|
|
32878
|
+
throw new CommonsPublicationError("publication_already_claimed", `This account already owns ${existingOwned.publicUrl}.`, 409);
|
|
32879
|
+
}
|
|
32880
|
+
const existingName = await getCommonsPublicationBySubdomain(subdomain);
|
|
32881
|
+
if (existingName) throw new CommonsPublicationError("publication_name_unavailable", "That publication name is already claimed.", 409);
|
|
32882
|
+
const id = `tcpub_${randomUUID16()}`;
|
|
32883
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
32884
|
+
try {
|
|
32885
|
+
await getDb().execute({
|
|
32886
|
+
sql: `
|
|
32887
|
+
INSERT INTO commons_publications (
|
|
32888
|
+
id, owner_user_id, subdomain, title, description, claim_idempotency_key, created_at, updated_at
|
|
32889
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
32890
|
+
`,
|
|
32891
|
+
args: [
|
|
32892
|
+
id,
|
|
32893
|
+
userId,
|
|
32894
|
+
subdomain,
|
|
32895
|
+
cleanText(input.title, 140) || publicationTitleFromSubdomain(subdomain),
|
|
32896
|
+
cleanText(input.description, 500),
|
|
32897
|
+
requiredIdempotencyKey(input.idempotencyKey),
|
|
32898
|
+
now,
|
|
32899
|
+
now
|
|
32900
|
+
]
|
|
32901
|
+
});
|
|
32902
|
+
} catch (error) {
|
|
32903
|
+
const replay = await getCommonsPublicationForOwner(userId);
|
|
32904
|
+
if (replay?.subdomain === subdomain) return { publication: replay, idempotentReplay: true };
|
|
32905
|
+
throw error;
|
|
32906
|
+
}
|
|
32907
|
+
const publication = await getCommonsPublicationBySubdomain(subdomain);
|
|
32908
|
+
if (!publication) throw new CommonsPublicationError("publication_claim_failed", "The publication claim was not persisted.");
|
|
32909
|
+
return { publication, idempotentReplay: false };
|
|
32910
|
+
}
|
|
32911
|
+
async function publishCommonsEditorial(input, user) {
|
|
32912
|
+
await ensureCommonsSchema();
|
|
32913
|
+
const subdomain = normalizePublicationSubdomain(input.publicationSubdomain);
|
|
32914
|
+
const publication = await getCommonsPublicationBySubdomain(subdomain);
|
|
32915
|
+
if (!publication) throw new CommonsPublicationError("publication_not_found", "Claim this publication name before publishing an edition.", 404);
|
|
32916
|
+
if (publication.ownerUserId !== Number(user.id)) {
|
|
32917
|
+
throw new CommonsPublicationError("publication_not_owned", "Only the account that claimed this publication can publish to it.", 404);
|
|
32918
|
+
}
|
|
32919
|
+
const idempotencyKey3 = requiredIdempotencyKey(input.idempotencyKey);
|
|
32920
|
+
const replay = await getEditionByIdempotency(Number(user.id), idempotencyKey3);
|
|
32921
|
+
if (replay) return publicationResult(publication, replay, true);
|
|
32922
|
+
const editionSlug = normalizeEditionSlug(input.editionSlug || input.site.slug);
|
|
32923
|
+
const latest = await getLatestEdition(publication.id, editionSlug);
|
|
32924
|
+
if (latest && input.baseRevision === void 0) {
|
|
32925
|
+
throw new CommonsPublicationError("publication_base_revision_required", `Edition ${editionSlug} already exists at revision ${latest.revision}; supply baseRevision to edit it.`, 409);
|
|
32926
|
+
}
|
|
32927
|
+
if (latest && input.baseRevision !== latest.revision) {
|
|
32928
|
+
throw new CommonsPublicationError("publication_revision_conflict", `Edition ${editionSlug} is revision ${latest.revision}; refresh it before publishing an edit.`, 409);
|
|
32929
|
+
}
|
|
32930
|
+
const { publicationSubdomain: _publicationSubdomain, editionSlug: _editionSlug, idempotencyKey: _idempotencyKey, baseRevision: _baseRevision, ...editionInput } = input;
|
|
32931
|
+
const canonicalUrl = editionPublicUrl(subdomain, editionSlug);
|
|
32932
|
+
const rendered = renderEditorialReadingRoom(editionInput);
|
|
32933
|
+
const html = addPublicMetadata(rendered.html, canonicalUrl, publication.title);
|
|
32934
|
+
const editionId = `tced_${randomUUID16()}`;
|
|
32935
|
+
const revision = (latest?.revision ?? 0) + 1;
|
|
32936
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
32937
|
+
await getDb().batch([
|
|
32938
|
+
{
|
|
32939
|
+
sql: `
|
|
32940
|
+
INSERT INTO commons_publication_editions (
|
|
32941
|
+
id, publication_id, owner_user_id, edition_slug, revision, title, site_json, deck,
|
|
32942
|
+
articles_json, html, filename, sha256, article_count, word_count, bytes, warnings_json,
|
|
32943
|
+
idempotency_key, created_at, published_at
|
|
32944
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
32945
|
+
`,
|
|
32946
|
+
args: [
|
|
32947
|
+
editionId,
|
|
32948
|
+
publication.id,
|
|
32949
|
+
Number(user.id),
|
|
32950
|
+
editionSlug,
|
|
32951
|
+
revision,
|
|
32952
|
+
editionInput.site.title,
|
|
32953
|
+
JSON.stringify(editionInput.site),
|
|
32954
|
+
editionInput.deck,
|
|
32955
|
+
JSON.stringify(editionInput.articles),
|
|
32956
|
+
html,
|
|
32957
|
+
rendered.filename,
|
|
32958
|
+
createHash13("sha256").update(html).digest("hex"),
|
|
32959
|
+
rendered.articleCount,
|
|
32960
|
+
rendered.wordCount,
|
|
32961
|
+
Buffer.byteLength(html),
|
|
32962
|
+
JSON.stringify(rendered.warnings),
|
|
32963
|
+
idempotencyKey3,
|
|
32964
|
+
now,
|
|
32965
|
+
now
|
|
32966
|
+
]
|
|
32967
|
+
},
|
|
32968
|
+
{
|
|
32969
|
+
sql: "UPDATE commons_publications SET latest_edition_id = ?, updated_at = ? WHERE id = ? AND owner_user_id = ?",
|
|
32970
|
+
args: [editionId, now, publication.id, Number(user.id)]
|
|
32971
|
+
}
|
|
32972
|
+
], "write");
|
|
32973
|
+
const edition = await getPublicationEditionById(editionId);
|
|
32974
|
+
if (!edition) throw new CommonsPublicationError("publication_publish_failed", "The published edition was not persisted.");
|
|
32975
|
+
return publicationResult({ ...publication, latestEditionId: editionId, updatedAt: now }, edition, false);
|
|
32976
|
+
}
|
|
32977
|
+
async function getCommonsPublicationBySubdomain(subdomainInput) {
|
|
32978
|
+
await ensureCommonsSchema();
|
|
32979
|
+
const subdomain = normalizePublicationSubdomain(subdomainInput);
|
|
32980
|
+
const result = await getDb().execute({ sql: "SELECT * FROM commons_publications WHERE subdomain = ? LIMIT 1", args: [subdomain] });
|
|
32981
|
+
return result.rows[0] ? rowToPublication(result.rows[0]) : null;
|
|
32982
|
+
}
|
|
32983
|
+
async function getCommonsPublicationForOwner(userId) {
|
|
32984
|
+
await ensureCommonsSchema();
|
|
32985
|
+
const result = await getDb().execute({ sql: "SELECT * FROM commons_publications WHERE owner_user_id = ? LIMIT 1", args: [userId] });
|
|
32986
|
+
return result.rows[0] ? rowToPublication(result.rows[0]) : null;
|
|
32987
|
+
}
|
|
32988
|
+
async function listCommonsPublicationEditions(publicationId) {
|
|
32989
|
+
await ensureCommonsSchema();
|
|
32990
|
+
const result = await getDb().execute({
|
|
32991
|
+
sql: `
|
|
32992
|
+
SELECT edition.* FROM commons_publication_editions edition
|
|
32993
|
+
JOIN (
|
|
32994
|
+
SELECT edition_slug, MAX(revision) AS revision
|
|
32995
|
+
FROM commons_publication_editions WHERE publication_id = ? GROUP BY edition_slug
|
|
32996
|
+
) latest ON latest.edition_slug = edition.edition_slug AND latest.revision = edition.revision
|
|
32997
|
+
WHERE edition.publication_id = ? ORDER BY edition.published_at DESC
|
|
32998
|
+
`,
|
|
32999
|
+
args: [publicationId, publicationId]
|
|
33000
|
+
});
|
|
33001
|
+
const publication = await getPublicationById(publicationId);
|
|
33002
|
+
if (!publication) return [];
|
|
33003
|
+
return result.rows.map((row) => rowToEdition(row, publication.subdomain));
|
|
33004
|
+
}
|
|
33005
|
+
async function getCommonsPublicationEditionHtml(subdomainInput, editionSlugInput) {
|
|
33006
|
+
const publication = await getCommonsPublicationBySubdomain(subdomainInput);
|
|
33007
|
+
if (!publication) return null;
|
|
33008
|
+
const result = editionSlugInput ? await getDb().execute({
|
|
33009
|
+
sql: "SELECT * FROM commons_publication_editions WHERE publication_id = ? AND edition_slug = ? ORDER BY revision DESC LIMIT 1",
|
|
33010
|
+
args: [publication.id, normalizeEditionSlug(editionSlugInput)]
|
|
33011
|
+
}) : await getDb().execute({
|
|
33012
|
+
sql: "SELECT * FROM commons_publication_editions WHERE id = ? AND publication_id = ? LIMIT 1",
|
|
33013
|
+
args: [publication.latestEditionId || "", publication.id]
|
|
33014
|
+
});
|
|
33015
|
+
if (!result.rows[0]) return null;
|
|
33016
|
+
const record = result.rows[0];
|
|
33017
|
+
return { publication, edition: rowToEdition(record, publication.subdomain), html: String(record.html || "") };
|
|
33018
|
+
}
|
|
33019
|
+
async function getPublicationById(id) {
|
|
33020
|
+
const result = await getDb().execute({ sql: "SELECT * FROM commons_publications WHERE id = ? LIMIT 1", args: [id] });
|
|
33021
|
+
return result.rows[0] ? rowToPublication(result.rows[0]) : null;
|
|
33022
|
+
}
|
|
33023
|
+
async function getLatestEdition(publicationId, editionSlug) {
|
|
33024
|
+
const publication = await getPublicationById(publicationId);
|
|
33025
|
+
if (!publication) return null;
|
|
33026
|
+
const result = await getDb().execute({
|
|
33027
|
+
sql: "SELECT * FROM commons_publication_editions WHERE publication_id = ? AND edition_slug = ? ORDER BY revision DESC LIMIT 1",
|
|
33028
|
+
args: [publicationId, editionSlug]
|
|
33029
|
+
});
|
|
33030
|
+
return result.rows[0] ? rowToEdition(result.rows[0], publication.subdomain) : null;
|
|
33031
|
+
}
|
|
33032
|
+
async function getEditionByIdempotency(userId, idempotencyKey3) {
|
|
33033
|
+
const result = await getDb().execute({
|
|
33034
|
+
sql: `SELECT edition.*, publication.subdomain FROM commons_publication_editions edition JOIN commons_publications publication ON publication.id = edition.publication_id WHERE edition.owner_user_id = ? AND edition.idempotency_key = ? LIMIT 1`,
|
|
33035
|
+
args: [userId, idempotencyKey3]
|
|
33036
|
+
});
|
|
33037
|
+
if (!result.rows[0]) return null;
|
|
33038
|
+
const record = result.rows[0];
|
|
33039
|
+
return rowToEdition(record, String(record.subdomain));
|
|
33040
|
+
}
|
|
33041
|
+
async function getPublicationEditionById(id) {
|
|
33042
|
+
const result = await getDb().execute({
|
|
33043
|
+
sql: `SELECT edition.*, publication.subdomain FROM commons_publication_editions edition JOIN commons_publications publication ON publication.id = edition.publication_id WHERE edition.id = ? LIMIT 1`,
|
|
33044
|
+
args: [id]
|
|
33045
|
+
});
|
|
33046
|
+
if (!result.rows[0]) return null;
|
|
33047
|
+
const record = result.rows[0];
|
|
33048
|
+
return rowToEdition(record, String(record.subdomain));
|
|
33049
|
+
}
|
|
33050
|
+
function rowToPublication(row) {
|
|
33051
|
+
const subdomain = String(row.subdomain);
|
|
33052
|
+
return {
|
|
33053
|
+
id: String(row.id),
|
|
33054
|
+
ownerUserId: Number(row.owner_user_id),
|
|
33055
|
+
subdomain,
|
|
33056
|
+
title: String(row.title),
|
|
33057
|
+
description: String(row.description || ""),
|
|
33058
|
+
latestEditionId: row.latest_edition_id ? String(row.latest_edition_id) : null,
|
|
33059
|
+
createdAt: String(row.created_at),
|
|
33060
|
+
updatedAt: String(row.updated_at),
|
|
33061
|
+
publicUrl: publicationPublicUrl(subdomain),
|
|
33062
|
+
archiveUrl: `${publicationPublicUrl(subdomain)}/archive`
|
|
33063
|
+
};
|
|
33064
|
+
}
|
|
33065
|
+
function rowToEdition(row, subdomain) {
|
|
33066
|
+
const editionSlug = String(row.edition_slug);
|
|
33067
|
+
return {
|
|
33068
|
+
id: String(row.id),
|
|
33069
|
+
publicationId: String(row.publication_id),
|
|
33070
|
+
editionSlug,
|
|
33071
|
+
revision: Number(row.revision),
|
|
33072
|
+
title: String(row.title),
|
|
33073
|
+
site: parseJson4(row.site_json, {}),
|
|
33074
|
+
deck: String(row.deck),
|
|
33075
|
+
filename: String(row.filename),
|
|
33076
|
+
sha256: String(row.sha256),
|
|
33077
|
+
articleCount: Number(row.article_count),
|
|
33078
|
+
wordCount: Number(row.word_count),
|
|
33079
|
+
bytes: Number(row.bytes),
|
|
33080
|
+
warnings: parseJson4(row.warnings_json, []),
|
|
33081
|
+
createdAt: String(row.created_at),
|
|
33082
|
+
publishedAt: String(row.published_at),
|
|
33083
|
+
publicUrl: editionPublicUrl(subdomain, editionSlug)
|
|
33084
|
+
};
|
|
33085
|
+
}
|
|
33086
|
+
function publicationResult(publication, edition, idempotentReplay) {
|
|
33087
|
+
return {
|
|
33088
|
+
publication,
|
|
33089
|
+
edition,
|
|
33090
|
+
publicUrl: publication.publicUrl,
|
|
33091
|
+
archiveUrl: publication.archiveUrl,
|
|
33092
|
+
editionUrl: edition.publicUrl,
|
|
33093
|
+
idempotentReplay
|
|
33094
|
+
};
|
|
33095
|
+
}
|
|
33096
|
+
function normalizePublicationSubdomain(value) {
|
|
33097
|
+
const normalized = String(value || "").trim().toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "").replace(/-+/g, "-").replace(/^-|-$/g, "");
|
|
33098
|
+
if (!/^[a-z0-9](?:[a-z0-9-]{1,48}[a-z0-9])$/.test(normalized)) {
|
|
33099
|
+
throw new CommonsPublicationError("publication_name_invalid", "Choose a publication name between 3 and 50 characters using letters, numbers, and interior hyphens.");
|
|
33100
|
+
}
|
|
33101
|
+
if (RESERVED_SUBDOMAINS.has(normalized)) throw new CommonsPublicationError("publication_name_reserved", "That publication name is reserved; choose a more specific name.", 409);
|
|
33102
|
+
return normalized;
|
|
33103
|
+
}
|
|
33104
|
+
function normalizeEditionSlug(value) {
|
|
33105
|
+
const normalized = String(value || "").trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 80);
|
|
33106
|
+
if (!normalized) throw new CommonsPublicationError("edition_slug_invalid", "A published edition needs a stable slug.");
|
|
33107
|
+
return normalized;
|
|
33108
|
+
}
|
|
33109
|
+
function requiredIdempotencyKey(value) {
|
|
33110
|
+
const normalized = cleanText(value, 200);
|
|
33111
|
+
if (normalized.length < 8) throw new CommonsPublicationError("idempotency_key_invalid", "Use an idempotency key of at least 8 characters and reuse it only when retrying the same write.");
|
|
33112
|
+
return normalized;
|
|
33113
|
+
}
|
|
33114
|
+
function publicationPublicUrl(subdomain) {
|
|
33115
|
+
return `https://${subdomain}.${PUBLICATION_ROOT_DOMAIN}`;
|
|
33116
|
+
}
|
|
33117
|
+
function editionPublicUrl(subdomain, editionSlug) {
|
|
33118
|
+
return `${publicationPublicUrl(subdomain)}/editions/${encodeURIComponent(editionSlug)}`;
|
|
33119
|
+
}
|
|
33120
|
+
function publicationTitleFromSubdomain(subdomain) {
|
|
33121
|
+
return subdomain.split("-").map((part) => part ? `${part[0]?.toUpperCase()}${part.slice(1)}` : "").join(" ");
|
|
33122
|
+
}
|
|
33123
|
+
function cleanText(value, max) {
|
|
33124
|
+
return String(value || "").replace(/\s+/g, " ").trim().slice(0, max);
|
|
33125
|
+
}
|
|
33126
|
+
function parseJson4(value, fallback) {
|
|
33127
|
+
try {
|
|
33128
|
+
return JSON.parse(String(value || ""));
|
|
33129
|
+
} catch {
|
|
33130
|
+
return fallback;
|
|
33131
|
+
}
|
|
33132
|
+
}
|
|
33133
|
+
function addPublicMetadata(html, canonicalUrl, publicationTitle) {
|
|
33134
|
+
const escapedUrl = canonicalUrl.replace(/&/g, "&").replace(/"/g, """);
|
|
33135
|
+
const escapedTitle = publicationTitle.replace(/&/g, "&").replace(/"/g, """);
|
|
33136
|
+
return html.replace("</head>", [
|
|
33137
|
+
` <link rel="canonical" href="${escapedUrl}">`,
|
|
33138
|
+
` <meta property="og:url" content="${escapedUrl}">`,
|
|
33139
|
+
` <meta property="og:site_name" content="${escapedTitle}">`,
|
|
33140
|
+
' <meta name="robots" content="index,follow">',
|
|
33141
|
+
"</head>"
|
|
33142
|
+
].join("\n"));
|
|
33143
|
+
}
|
|
33144
|
+
|
|
32487
33145
|
// src/api/commons-routes.ts
|
|
32488
33146
|
var commonsApp = new Hono31();
|
|
32489
33147
|
var auth2 = createApiKeyAuth();
|
|
@@ -32645,10 +33303,83 @@ var NeedsLinkBodySchema = z32.object({
|
|
|
32645
33303
|
limit: z32.number().int().min(1).max(100).optional(),
|
|
32646
33304
|
offset: z32.number().int().min(0).max(1e4).optional()
|
|
32647
33305
|
}).strict();
|
|
33306
|
+
var PreparePublicationSchema = z32.object(CommonsPreparePublicationInputSchema).strict();
|
|
33307
|
+
var ValidatePublicationSchema = z32.object(CommonsValidatePublicationInputSchema).strict();
|
|
33308
|
+
var ClaimPublicationSchema = z32.object(CommonsClaimPublicationInputSchema).strict();
|
|
33309
|
+
var PublishEditorialSchema = z32.object(CommonsPublishEditorialInputSchema).strict();
|
|
32648
33310
|
commonsApp.get("/health", async (c) => {
|
|
32649
33311
|
await ensureCommonsSchema();
|
|
32650
33312
|
return c.json({ ok: true, data: commonsDatabaseReport() });
|
|
32651
33313
|
});
|
|
33314
|
+
commonsApp.post("/publications/prepare", auth2, async (c) => {
|
|
33315
|
+
const parsed = PreparePublicationSchema.safeParse(await c.req.json().catch(() => ({})));
|
|
33316
|
+
if (!parsed.success) return validationError(c, parsed.error);
|
|
33317
|
+
return publicationOperation(c, () => prepareCommonsPublication(parsed.data, c.get("user")));
|
|
33318
|
+
});
|
|
33319
|
+
commonsApp.post("/publications/validate", auth2, async (c) => {
|
|
33320
|
+
const parsed = ValidatePublicationSchema.safeParse(await c.req.json().catch(() => ({})));
|
|
33321
|
+
if (!parsed.success) return validationError(c, parsed.error);
|
|
33322
|
+
return publicationOperation(c, () => validateCommonsPublication(parsed.data, c.get("user")));
|
|
33323
|
+
});
|
|
33324
|
+
commonsApp.post("/publications/claim", auth2, async (c) => {
|
|
33325
|
+
const parsed = ClaimPublicationSchema.safeParse(await c.req.json().catch(() => ({})));
|
|
33326
|
+
if (!parsed.success) return validationError(c, parsed.error);
|
|
33327
|
+
return publicationOperation(c, () => claimCommonsPublication(parsed.data, c.get("user")), 201);
|
|
33328
|
+
});
|
|
33329
|
+
commonsApp.post("/publications/publish", auth2, async (c) => {
|
|
33330
|
+
const parsed = PublishEditorialSchema.safeParse(await c.req.json().catch(() => ({})));
|
|
33331
|
+
if (!parsed.success) return validationError(c, parsed.error);
|
|
33332
|
+
return publicationOperation(c, () => publishCommonsEditorial(parsed.data, c.get("user")), 201);
|
|
33333
|
+
});
|
|
33334
|
+
commonsApp.get("/publications/me", auth2, async (c) => {
|
|
33335
|
+
const publication = await getCommonsPublicationForOwner(Number(c.get("user").id));
|
|
33336
|
+
if (!publication) return c.json({ ok: false, error: "publication_not_found", message: "This account has not claimed a Commons publication yet." }, 404);
|
|
33337
|
+
const editions = c.req.query("includeEditions") === "false" ? [] : await listCommonsPublicationEditions(publication.id);
|
|
33338
|
+
return c.json({ ok: true, data: { publication, editions } });
|
|
33339
|
+
});
|
|
33340
|
+
commonsApp.get("/publications/:subdomain/site", async (c) => {
|
|
33341
|
+
try {
|
|
33342
|
+
const result = await getCommonsPublicationEditionHtml(c.req.param("subdomain"));
|
|
33343
|
+
if (!result) return c.json({ ok: false, error: "publication_not_published", message: "No published edition exists for this publication." }, 404);
|
|
33344
|
+
return c.html(result.html, 200, {
|
|
33345
|
+
"cache-control": "public, max-age=60, s-maxage=300",
|
|
33346
|
+
"x-robots-tag": "index, follow"
|
|
33347
|
+
});
|
|
33348
|
+
} catch (error) {
|
|
33349
|
+
return publicationError(c, error);
|
|
33350
|
+
}
|
|
33351
|
+
});
|
|
33352
|
+
commonsApp.get("/publications/:subdomain/editions/:editionSlug/site", async (c) => {
|
|
33353
|
+
try {
|
|
33354
|
+
const result = await getCommonsPublicationEditionHtml(c.req.param("subdomain"), c.req.param("editionSlug"));
|
|
33355
|
+
if (!result) return c.json({ ok: false, error: "edition_not_found", message: "No published edition matched that publication and slug." }, 404);
|
|
33356
|
+
return c.html(result.html, 200, {
|
|
33357
|
+
"cache-control": "public, max-age=60, s-maxage=300",
|
|
33358
|
+
"x-robots-tag": "index, follow"
|
|
33359
|
+
});
|
|
33360
|
+
} catch (error) {
|
|
33361
|
+
return publicationError(c, error);
|
|
33362
|
+
}
|
|
33363
|
+
});
|
|
33364
|
+
commonsApp.get("/publications/:subdomain/editions", async (c) => {
|
|
33365
|
+
try {
|
|
33366
|
+
const publication = await getCommonsPublicationBySubdomain(c.req.param("subdomain"));
|
|
33367
|
+
if (!publication) return c.json({ ok: false, error: "publication_not_found", message: "No Commons publication matched that name." }, 404);
|
|
33368
|
+
return c.json({ ok: true, data: { publication, editions: await listCommonsPublicationEditions(publication.id) } });
|
|
33369
|
+
} catch (error) {
|
|
33370
|
+
return publicationError(c, error);
|
|
33371
|
+
}
|
|
33372
|
+
});
|
|
33373
|
+
commonsApp.get("/publications/:subdomain", async (c) => {
|
|
33374
|
+
try {
|
|
33375
|
+
const publication = await getCommonsPublicationBySubdomain(c.req.param("subdomain"));
|
|
33376
|
+
if (!publication) return c.json({ ok: false, error: "publication_not_found", message: "No Commons publication matched that name." }, 404);
|
|
33377
|
+
const editions = c.req.query("includeEditions") === "false" ? [] : await listCommonsPublicationEditions(publication.id);
|
|
33378
|
+
return c.json({ ok: true, data: { publication, editions } });
|
|
33379
|
+
} catch (error) {
|
|
33380
|
+
return publicationError(c, error);
|
|
33381
|
+
}
|
|
33382
|
+
});
|
|
32652
33383
|
commonsApp.get("/entities", async (c) => {
|
|
32653
33384
|
const result = await searchCommonsEntities(filtersFromQuery(c.req.query()));
|
|
32654
33385
|
return c.json({ ok: true, data: result });
|
|
@@ -32685,10 +33416,22 @@ commonsApp.post("/entities/validate", auth2, async (c) => {
|
|
|
32685
33416
|
commonsApp.post("/entities/propose", auth2, async (c) => {
|
|
32686
33417
|
const parsed = SubmitEntitySchema.safeParse(await c.req.json().catch(() => ({})));
|
|
32687
33418
|
if (!parsed.success) return validationError(c, parsed.error);
|
|
33419
|
+
let billing = null;
|
|
32688
33420
|
try {
|
|
33421
|
+
const user = c.get("user");
|
|
33422
|
+
const kind = await classifyCommonsWrite(parsed.data, user);
|
|
33423
|
+
const charged = await chargeListingWrite({
|
|
33424
|
+
surface: "transparent_commons",
|
|
33425
|
+
kind,
|
|
33426
|
+
userId: Number(user.id),
|
|
33427
|
+
idempotencyKey: parsed.data.idempotencyKey
|
|
33428
|
+
});
|
|
33429
|
+
if ("insufficient" in charged) return c.json(charged.insufficient, 402);
|
|
33430
|
+
billing = charged;
|
|
32689
33431
|
const result = await submitCommonsEntity(parsed.data, c.get("user"));
|
|
32690
|
-
return c.json({ ok: true, data: result }, result.applied ? 201 : 202);
|
|
33432
|
+
return c.json({ ok: true, data: result, billing: publicListingWriteBilling(billing) }, result.applied ? 201 : 202);
|
|
32691
33433
|
} catch (error) {
|
|
33434
|
+
if (billing) await refundListingWrite(billing, "Transparent Commons write failed before persistence.").catch(() => null);
|
|
32692
33435
|
if (error instanceof CommonsRepositoryError) {
|
|
32693
33436
|
return c.json({ ok: false, error: error.code, message: error.message }, repositoryStatus(error));
|
|
32694
33437
|
}
|
|
@@ -32798,6 +33541,19 @@ function repositoryStatus(error) {
|
|
|
32798
33541
|
if (error.httpStatus === 409) return 409;
|
|
32799
33542
|
return 400;
|
|
32800
33543
|
}
|
|
33544
|
+
async function publicationOperation(c, operation, successStatus = 200) {
|
|
33545
|
+
try {
|
|
33546
|
+
return c.json({ ok: true, data: await operation() }, successStatus);
|
|
33547
|
+
} catch (error) {
|
|
33548
|
+
return publicationError(c, error);
|
|
33549
|
+
}
|
|
33550
|
+
}
|
|
33551
|
+
function publicationError(c, error) {
|
|
33552
|
+
if (error instanceof CommonsPublicationError) {
|
|
33553
|
+
return c.json({ ok: false, error: error.code, message: error.message }, error.httpStatus);
|
|
33554
|
+
}
|
|
33555
|
+
throw error;
|
|
33556
|
+
}
|
|
32801
33557
|
function xmlEscape(value) {
|
|
32802
33558
|
return value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll('"', """).replaceAll("'", "'");
|
|
32803
33559
|
}
|
|
@@ -32807,13 +33563,13 @@ import { Hono as Hono32 } from "hono";
|
|
|
32807
33563
|
import { z as z33 } from "zod";
|
|
32808
33564
|
|
|
32809
33565
|
// src/api/scheduled-artifact-owner.ts
|
|
32810
|
-
import { createHash as
|
|
33566
|
+
import { createHash as createHash14 } from "crypto";
|
|
32811
33567
|
function scheduledArtifactOwnerIdForApiKey(apiKey) {
|
|
32812
|
-
return
|
|
33568
|
+
return createHash14("sha256").update(apiKey).digest("hex").slice(0, 24);
|
|
32813
33569
|
}
|
|
32814
33570
|
|
|
32815
33571
|
// src/api/scheduled-run-view-links.ts
|
|
32816
|
-
import { createHash as
|
|
33572
|
+
import { createHash as createHash15, randomBytes as randomBytes3, randomUUID as randomUUID17 } from "crypto";
|
|
32817
33573
|
var schemaReady3 = false;
|
|
32818
33574
|
async function ensureScheduledRunViewLinksSchema() {
|
|
32819
33575
|
if (schemaReady3) return;
|
|
@@ -32844,7 +33600,7 @@ async function ensureScheduledRunViewLinksSchema() {
|
|
|
32844
33600
|
schemaReady3 = true;
|
|
32845
33601
|
}
|
|
32846
33602
|
function tokenHash2(token4) {
|
|
32847
|
-
return
|
|
33603
|
+
return createHash15("sha256").update(token4).digest("hex");
|
|
32848
33604
|
}
|
|
32849
33605
|
function mapRow(row) {
|
|
32850
33606
|
return {
|
|
@@ -32866,7 +33622,7 @@ async function createScheduledRunViewLink(input) {
|
|
|
32866
33622
|
const now = input.now ?? /* @__PURE__ */ new Date();
|
|
32867
33623
|
const token4 = randomBytes3(32).toString("base64url");
|
|
32868
33624
|
const record = {
|
|
32869
|
-
shareId:
|
|
33625
|
+
shareId: randomUUID17(),
|
|
32870
33626
|
ownerId: input.ownerId,
|
|
32871
33627
|
runId: input.runId,
|
|
32872
33628
|
artifactId: input.artifactId,
|
|
@@ -33103,7 +33859,7 @@ scheduledResultApp.delete("/schedule-runs/:runId/view-links/:shareId", async (c)
|
|
|
33103
33859
|
import { Hono as Hono33 } from "hono";
|
|
33104
33860
|
|
|
33105
33861
|
// src/scheduled-artifacts/scheduled-run-artifact-store.ts
|
|
33106
|
-
import { createHash as
|
|
33862
|
+
import { createHash as createHash16 } from "crypto";
|
|
33107
33863
|
var SCHEDULED_RUN_ARTIFACT_PREFIX = "scheduled-run-artifacts/";
|
|
33108
33864
|
var SCHEDULED_RUN_ARTIFACT_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
|
|
33109
33865
|
var SCHEDULED_RUN_ARTIFACT_MAX_BYTES = 2e6;
|
|
@@ -33119,7 +33875,7 @@ function policy3() {
|
|
|
33119
33875
|
};
|
|
33120
33876
|
}
|
|
33121
33877
|
function runStorageSegment(runId) {
|
|
33122
|
-
return /^[a-zA-Z0-9_-]{1,160}$/.test(runId) ? runId : `run-${
|
|
33878
|
+
return /^[a-zA-Z0-9_-]{1,160}$/.test(runId) ? runId : `run-${createHash16("sha256").update(runId).digest("hex").slice(0, 32)}`;
|
|
33123
33879
|
}
|
|
33124
33880
|
async function createScheduledRunArtifact(args) {
|
|
33125
33881
|
if (args.rendered.bytes > SCHEDULED_RUN_ARTIFACT_MAX_BYTES) {
|
|
@@ -33226,10 +33982,10 @@ publicScheduledResultApp.get("/scheduled-run-view-links/:token/content", async (
|
|
|
33226
33982
|
import { Hono as Hono34 } from "hono";
|
|
33227
33983
|
|
|
33228
33984
|
// src/api/scheduler-integration-auth.ts
|
|
33229
|
-
import { createHash as
|
|
33985
|
+
import { createHash as createHash17, createHmac as createHmac4, timingSafeEqual as timingSafeEqual3 } from "crypto";
|
|
33230
33986
|
|
|
33231
33987
|
// src/api/service-connections.ts
|
|
33232
|
-
import { randomUUID as
|
|
33988
|
+
import { randomUUID as randomUUID18 } from "crypto";
|
|
33233
33989
|
var schemaReady4 = null;
|
|
33234
33990
|
var schemaDb3 = null;
|
|
33235
33991
|
function ensureServiceConnectionsSchema() {
|
|
@@ -33371,7 +34127,7 @@ async function reconcileDiscoveredNangoConnections(identity, discovered) {
|
|
|
33371
34127
|
updated_at = excluded.updated_at
|
|
33372
34128
|
`,
|
|
33373
34129
|
args: [
|
|
33374
|
-
|
|
34130
|
+
randomUUID18(),
|
|
33375
34131
|
userId,
|
|
33376
34132
|
connection.providerConfigKey,
|
|
33377
34133
|
connection.provider,
|
|
@@ -33442,7 +34198,7 @@ async function recordServiceConnectionHealth(args) {
|
|
|
33442
34198
|
});
|
|
33443
34199
|
await getDb().execute({
|
|
33444
34200
|
sql: `INSERT INTO service_connection_health_events (id, connection_id, operational_status, failure_code, retryable, evidence_source) VALUES (?, ?, ?, ?, ?, ?)`,
|
|
33445
|
-
args: [
|
|
34201
|
+
args: [randomUUID18(), args.connectionId, args.operationalStatus, args.failureCode ?? null, args.retryable == null ? null : args.retryable ? 1 : 0, args.evidenceSource]
|
|
33446
34202
|
});
|
|
33447
34203
|
}
|
|
33448
34204
|
async function setServiceConnectionActions(identity, connectionId, enabled) {
|
|
@@ -33482,7 +34238,7 @@ async function claimServiceConnectionAction(args) {
|
|
|
33482
34238
|
if (!connection) throw new Error("service_connection_not_found");
|
|
33483
34239
|
const inserted = await getDb().execute({
|
|
33484
34240
|
sql: `INSERT OR IGNORE INTO service_connection_action_audit (id, connection_id, user_id, tool, request_id, status, request_digest) VALUES (?, ?, ?, ?, ?, 'started', ?)`,
|
|
33485
|
-
args: [
|
|
34241
|
+
args: [randomUUID18(), connection.id, connection.userId, args.tool, args.requestId, args.requestDigest]
|
|
33486
34242
|
});
|
|
33487
34243
|
if (Number(inserted.rowsAffected ?? 0) === 1) return { claimed: true };
|
|
33488
34244
|
const existing = await getDb().execute({
|
|
@@ -33524,7 +34280,7 @@ function signingSecret() {
|
|
|
33524
34280
|
return secret2;
|
|
33525
34281
|
}
|
|
33526
34282
|
function schedulerIntegrationSignature(args) {
|
|
33527
|
-
const bodyHash =
|
|
34283
|
+
const bodyHash = createHash17("sha256").update(args.body).digest("hex");
|
|
33528
34284
|
return createHmac4("sha256", args.secret).update(`${args.method.toUpperCase()}
|
|
33529
34285
|
${args.path}
|
|
33530
34286
|
${args.timestamp}
|
|
@@ -33808,13 +34564,13 @@ async function reconcileSiteExtractSettlements(limit = 25) {
|
|
|
33808
34564
|
}
|
|
33809
34565
|
|
|
33810
34566
|
// src/api/page-diff.ts
|
|
33811
|
-
import { createHash as
|
|
34567
|
+
import { createHash as createHash18 } from "crypto";
|
|
33812
34568
|
import { diffLines } from "diff";
|
|
33813
34569
|
var MAX_SNAPSHOT_CONTENT_CHARS = 25e4;
|
|
33814
34570
|
var MAX_DIFF_HUNKS = 200;
|
|
33815
34571
|
var MAX_DIFF_LINES_PER_RESPONSE = 2e3;
|
|
33816
34572
|
function sha256Hex(value) {
|
|
33817
|
-
return
|
|
34573
|
+
return createHash18("sha256").update(value).digest("hex");
|
|
33818
34574
|
}
|
|
33819
34575
|
function truncateForStorage(value, maxChars = MAX_SNAPSHOT_CONTENT_CHARS) {
|
|
33820
34576
|
if (value.length <= maxChars) return { value, truncated: false };
|
|
@@ -33928,10 +34684,10 @@ async function persistScrapeBody(user, opts) {
|
|
|
33928
34684
|
}
|
|
33929
34685
|
|
|
33930
34686
|
// src/api/scrape-image-sink.ts
|
|
33931
|
-
import { createHash as
|
|
34687
|
+
import { createHash as createHash19 } from "crypto";
|
|
33932
34688
|
var MAX_IMAGES_PER_SCRAPE = 25;
|
|
33933
34689
|
function idempotencyKey2(userId, vault, input) {
|
|
33934
|
-
return `scrape-image-${
|
|
34690
|
+
return `scrape-image-${createHash19("sha256").update(`${userId}\0${vault}\0${input.sourceKind}\0${input.sourceUrl}\0${input.imageUrl ?? ""}\0${input.imageBase64 ?? ""}`).digest("hex")}`;
|
|
33935
34691
|
}
|
|
33936
34692
|
async function persistScrapeImagesToMemory(user, inputs, vault) {
|
|
33937
34693
|
const selected = inputs.slice(0, MAX_IMAGES_PER_SCRAPE);
|
|
@@ -34403,7 +35159,7 @@ function inferredWaybackCapturedAt(content) {
|
|
|
34403
35159
|
}
|
|
34404
35160
|
|
|
34405
35161
|
// src/api/connection-memory-import.ts
|
|
34406
|
-
import { createHash as
|
|
35162
|
+
import { createHash as createHash20 } from "crypto";
|
|
34407
35163
|
var CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES = 64 * 1024;
|
|
34408
35164
|
var CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES = 1e6;
|
|
34409
35165
|
var CONNECTION_MEMORY_IMPORT_MAX_STRING_CHARS = 256e3;
|
|
@@ -34433,7 +35189,7 @@ function canonicalJson(value) {
|
|
|
34433
35189
|
return JSON.stringify(value);
|
|
34434
35190
|
}
|
|
34435
35191
|
function sha2562(value) {
|
|
34436
|
-
return
|
|
35192
|
+
return createHash20("sha256").update(value).digest("hex");
|
|
34437
35193
|
}
|
|
34438
35194
|
function sensitiveKey(key) {
|
|
34439
35195
|
const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
@@ -34702,7 +35458,7 @@ async function cleanupExpiredScrapeBlobs(maxAgeMs = SCRAPE_BLOB_TTL_MS) {
|
|
|
34702
35458
|
}
|
|
34703
35459
|
|
|
34704
35460
|
// src/api/connected-data-export.ts
|
|
34705
|
-
import { randomUUID as
|
|
35461
|
+
import { randomUUID as randomUUID19 } from "crypto";
|
|
34706
35462
|
var CONNECTED_DATA_INLINE_BUDGET_BYTES = Number(
|
|
34707
35463
|
process.env.MCP_SCRAPER_CONNECTED_DATA_INLINE_BUDGET_BYTES ?? 5e4
|
|
34708
35464
|
);
|
|
@@ -34873,7 +35629,7 @@ function finitePositive(value, fallback) {
|
|
|
34873
35629
|
return Number.isFinite(value) && value > 0 ? value : fallback;
|
|
34874
35630
|
}
|
|
34875
35631
|
async function collectConnectedDataExport(args) {
|
|
34876
|
-
const exportId =
|
|
35632
|
+
const exportId = randomUUID19();
|
|
34877
35633
|
const now = args.now ?? Date.now;
|
|
34878
35634
|
const startedAt = now();
|
|
34879
35635
|
const budgetMs = finitePositive(CONNECTED_DATA_EXPORT_BUDGET_MS, 24e4);
|
|
@@ -34989,7 +35745,7 @@ ${lines.length ? `${lines.join("\n")}
|
|
|
34989
35745
|
}
|
|
34990
35746
|
|
|
34991
35747
|
// src/api/search-console-table-export.ts
|
|
34992
|
-
import { randomUUID as
|
|
35748
|
+
import { randomUUID as randomUUID20 } from "crypto";
|
|
34993
35749
|
var SEARCH_CONSOLE_TABLE_EXPORT_MAX_ROWS = 5e4;
|
|
34994
35750
|
var SEARCH_CONSOLE_TABLE_EXPORT_PAGE_SIZE = 2e3;
|
|
34995
35751
|
var SEARCH_CONSOLE_TABLE_EXPORT_MAX_BYTES = 50 * 1024 * 1024;
|
|
@@ -35089,7 +35845,7 @@ async function exportSearchConsoleTableData(args) {
|
|
|
35089
35845
|
offset += rows.length;
|
|
35090
35846
|
if (stoppedForBytes || rows.length < limit || offset >= matchedRows) break;
|
|
35091
35847
|
}
|
|
35092
|
-
const exportId =
|
|
35848
|
+
const exportId = randomUUID20();
|
|
35093
35849
|
const artifact = await args.writeArtifact({
|
|
35094
35850
|
ownerId: args.ownerId,
|
|
35095
35851
|
exportId,
|
|
@@ -35444,7 +36200,7 @@ async function dbUsage(identity, plan) {
|
|
|
35444
36200
|
}
|
|
35445
36201
|
|
|
35446
36202
|
// src/api/nango-control.ts
|
|
35447
|
-
import { createHash as
|
|
36203
|
+
import { createHash as createHash22, randomUUID as randomUUID22 } from "crypto";
|
|
35448
36204
|
|
|
35449
36205
|
// src/api/slack-archive-analysis.ts
|
|
35450
36206
|
function isRecord2(value) {
|
|
@@ -35818,7 +36574,7 @@ async function exportSlackChannelPage(input, dependencies) {
|
|
|
35818
36574
|
}
|
|
35819
36575
|
|
|
35820
36576
|
// src/api/main-nango-transport.ts
|
|
35821
|
-
import { createHash as
|
|
36577
|
+
import { createHash as createHash21, randomUUID as randomUUID21 } from "crypto";
|
|
35822
36578
|
import { Client, StreamableHTTPClientTransport } from "@modelcontextprotocol/client";
|
|
35823
36579
|
|
|
35824
36580
|
// src/api/connected-tool-policy.ts
|
|
@@ -36211,7 +36967,7 @@ async function listNangoToolsDirect(identity, connectionId) {
|
|
|
36211
36967
|
});
|
|
36212
36968
|
const readTools = [...policies.values()].filter((policy4) => policy4.classification === "read").map((policy4) => policy4.name);
|
|
36213
36969
|
const actionTools = [...policies.values()].filter((policy4) => policy4.classification === "action").map((policy4) => policy4.name);
|
|
36214
|
-
const revision =
|
|
36970
|
+
const revision = createHash21("sha256").update(JSON.stringify(tools.map((tool) => ({ name: tool.name, inputSchema: tool.inputSchema })))).digest("hex");
|
|
36215
36971
|
await updateServiceConnectionTools(connection.id, readTools, actionTools, revision);
|
|
36216
36972
|
const refreshed = await getOwnedServiceConnection(identity, connection.id);
|
|
36217
36973
|
return { connection: refreshed ?? { ...connection, readTools, actionTools, toolRevision: revision }, tools };
|
|
@@ -36238,8 +36994,8 @@ async function callNangoToolDirect(args) {
|
|
|
36238
36994
|
identity: args.identity,
|
|
36239
36995
|
ratePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION
|
|
36240
36996
|
});
|
|
36241
|
-
const requestId = args.requestId?.trim() ||
|
|
36242
|
-
const idempotencyKey3 = `main-nango:${
|
|
36997
|
+
const requestId = args.requestId?.trim() || randomUUID21();
|
|
36998
|
+
const idempotencyKey3 = `main-nango:${createHash21("sha256").update(args.identity.toLowerCase()).update("\0").update(connection.id).update("\0").update(args.tool).update("\0").update(requestId).digest("hex")}`;
|
|
36243
36999
|
const startedAt = /* @__PURE__ */ new Date();
|
|
36244
37000
|
const started = performance.now();
|
|
36245
37001
|
let result;
|
|
@@ -36264,7 +37020,7 @@ async function callNangoToolDirect(args) {
|
|
|
36264
37020
|
toolName: args.tool,
|
|
36265
37021
|
operationKind: args.operationKind ?? args.classification,
|
|
36266
37022
|
outcome: providerError ? "error" : "partial",
|
|
36267
|
-
requestId: requestId.length <= 200 ? requestId :
|
|
37023
|
+
requestId: requestId.length <= 200 ? requestId : createHash21("sha256").update(requestId).digest("hex"),
|
|
36268
37024
|
startedAt: startedAt.toISOString(),
|
|
36269
37025
|
completedAt: completedAt.toISOString()
|
|
36270
37026
|
}
|
|
@@ -36306,7 +37062,7 @@ async function describeNangoToolDirect(identity, connectionId, toolName) {
|
|
|
36306
37062
|
providerContractHash: MAIN_INTEGRATION_CONTRACT_HASH,
|
|
36307
37063
|
protocolVersion: null,
|
|
36308
37064
|
schemaSource: "live_tools_list",
|
|
36309
|
-
schemaHash:
|
|
37065
|
+
schemaHash: createHash21("sha256").update(JSON.stringify(projected)).digest("hex"),
|
|
36310
37066
|
fetchedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
36311
37067
|
};
|
|
36312
37068
|
}
|
|
@@ -37030,8 +37786,8 @@ async function setScheduleConnectionActionsEnabled(identity, connectionId, enabl
|
|
|
37030
37786
|
return data.connection.actionsEnabled === true;
|
|
37031
37787
|
}
|
|
37032
37788
|
async function callScheduleConnectionAction(identity, connectionId, input, tool, idempotencyKey3) {
|
|
37033
|
-
const requestId = `main-connected-action:${
|
|
37034
|
-
const requestDigest =
|
|
37789
|
+
const requestId = `main-connected-action:${createHash22("sha256").update(identity).update("\0").update(idempotencyKey3?.trim() || randomUUID22()).digest("hex")}`;
|
|
37790
|
+
const requestDigest = createHash22("sha256").update(connectionId).update("\0").update(tool?.trim() ?? "").update("\0").update(canonicalJson2(input)).digest("hex");
|
|
37035
37791
|
if (mainOwnsIntegrations()) {
|
|
37036
37792
|
const selectedTool = tool?.trim();
|
|
37037
37793
|
if (!selectedTool) throw new NangoControlError("An action tool is required.", 400, "invalid_request", false);
|
|
@@ -37182,7 +37938,7 @@ function canonicalJson2(value) {
|
|
|
37182
37938
|
return JSON.stringify(value);
|
|
37183
37939
|
}
|
|
37184
37940
|
function projectedToolSchemaHash(tool) {
|
|
37185
|
-
return
|
|
37941
|
+
return createHash22("sha256").update(canonicalJson2(tool)).digest("hex");
|
|
37186
37942
|
}
|
|
37187
37943
|
async function describeNangoTool(identity, connectionId, tool, fresh) {
|
|
37188
37944
|
if (mainOwnsIntegrations()) {
|
|
@@ -37461,7 +38217,7 @@ async function callMainOwnedExportPage(identity, input) {
|
|
|
37461
38217
|
}
|
|
37462
38218
|
|
|
37463
38219
|
// src/api/resend-control.ts
|
|
37464
|
-
import { createHash as
|
|
38220
|
+
import { createHash as createHash23 } from "crypto";
|
|
37465
38221
|
var DEFAULT_CONNECTION_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
|
|
37466
38222
|
var RESEND_PROVIDER_CONFIG_KEY = "resend";
|
|
37467
38223
|
var RESEND_LOGO_URL = "https://cdn.resend.com/brand/resend-icon-black.svg";
|
|
@@ -37720,7 +38476,7 @@ async function callResendRead(identity, connectionId, tool, args) {
|
|
|
37720
38476
|
return isRecord5(data) ? data.result ?? data : data;
|
|
37721
38477
|
}
|
|
37722
38478
|
async function callResendAction(identity, connectionId, tool, input, idempotencyKey3) {
|
|
37723
|
-
const requestId = `main-resend-action:${
|
|
38479
|
+
const requestId = `main-resend-action:${createHash23("sha256").update(identity).update("\0").update(idempotencyKey3.trim()).digest("hex")}`;
|
|
37724
38480
|
const body = await controlRequest2("/api/internal/resend/actions/call", {
|
|
37725
38481
|
method: "POST",
|
|
37726
38482
|
headers: { "x-request-id": requestId },
|
|
@@ -39554,7 +40310,7 @@ app.post("/harvest", auth3, async (c) => {
|
|
|
39554
40310
|
if (!harvestOk) return c.json(insufficientBalanceResponse(harvestBal, harvestCost), 402);
|
|
39555
40311
|
jobId2 = await createJob(user.id, options.query, { ...options, billingHoldMc: harvestCost }, body.callback_url);
|
|
39556
40312
|
} else {
|
|
39557
|
-
jobId2 =
|
|
40313
|
+
jobId2 = randomUUID23();
|
|
39558
40314
|
const billingDebitKey = `paa-harvest:${jobId2}:hold`;
|
|
39559
40315
|
const description = `PAA harvest: ${options.query}`.slice(0, 500);
|
|
39560
40316
|
const hold = await debitMcIdempotent(
|
|
@@ -40772,7 +41528,7 @@ app.get("/cron/tick", async (c) => {
|
|
|
40772
41528
|
if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
|
|
40773
41529
|
return c.json({ error: "Unauthorized" }, 401);
|
|
40774
41530
|
}
|
|
40775
|
-
const { drainQueue } = await import("./worker-
|
|
41531
|
+
const { drainQueue } = await import("./worker-4LZGRHDA.js");
|
|
40776
41532
|
const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
|
|
40777
41533
|
const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
|
|
40778
41534
|
const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup, connectedAccountBilling, directoryWorkflowDispatch, directoryWorkflowReconciliation, directoryArtifactCleanup, siteExtractArtifactCleanup, siteExtractRedispatch, siteExtractReconciliation, paaHarvestRedispatch, paaHarvestReconciliation, localSourcebookRedispatch] = await Promise.all([
|
|
@@ -40827,8 +41583,8 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
40827
41583
|
finishExtractJob: finishExtractJob2,
|
|
40828
41584
|
terminalExtractJobStatus: terminalExtractJobStatus2,
|
|
40829
41585
|
extractJobLimitInfo: extractJobLimitInfo2
|
|
40830
|
-
} = await import("./site-extract-repository-
|
|
40831
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
41586
|
+
} = await import("./site-extract-repository-G723PZIL.js");
|
|
41587
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-DYMQRQWG.js");
|
|
40832
41588
|
const existing = await getExtractJob2(jobId2);
|
|
40833
41589
|
if (!existing) return c.json({ error: "job not found" }, 404);
|
|
40834
41590
|
if (existing.status !== "failed") {
|
|
@@ -40855,7 +41611,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
40855
41611
|
if (!updated) return c.json({ error: "job state changed during re-finalization" }, 409);
|
|
40856
41612
|
let settlement = "already_settled_or_refunded";
|
|
40857
41613
|
if (job.billedMc == null && job.userId != null) {
|
|
40858
|
-
const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-
|
|
41614
|
+
const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-G723PZIL.js");
|
|
40859
41615
|
const heldMc = Number(job.options.heldMc ?? 0);
|
|
40860
41616
|
const successful = await countSuccessfulPages2(jobId2);
|
|
40861
41617
|
const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
|
|
@@ -41039,4 +41795,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
41039
41795
|
export {
|
|
41040
41796
|
app
|
|
41041
41797
|
};
|
|
41042
|
-
//# sourceMappingURL=server-
|
|
41798
|
+
//# sourceMappingURL=server-SMEHT3OU.js.map
|