mcp-scraper 0.44.4 → 0.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/bin/api-server.cjs +4103 -3710
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +2619 -2507
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +3 -3
- package/dist/chunk-CEZ3NFBP.js +7 -0
- package/dist/chunk-CEZ3NFBP.js.map +1 -0
- package/dist/{chunk-ZVOVPXDL.js → chunk-FQIICRR4.js} +2849 -2741
- package/dist/chunk-FQIICRR4.js.map +1 -0
- package/dist/{chunk-WWIJ2NID.js → chunk-JFOT5JEL.js} +2 -2
- package/dist/{chunk-X5WDJ7D7.js → chunk-SOEYDWJU.js} +46 -3
- package/dist/chunk-SOEYDWJU.js.map +1 -0
- package/dist/{extract-bundle-U3MNYDKC.js → extract-bundle-DYMQRQWG.js} +3 -3
- package/dist/{server-W4JLSNKW.js → server-7B75SDNG.js} +362 -130
- package/dist/server-7B75SDNG.js.map +1 -0
- package/dist/{site-extract-repository-CKCJDP7N.js → site-extract-repository-G723PZIL.js} +3 -3
- package/dist/{worker-73AYU4IK.js → worker-4LZGRHDA.js} +2 -2
- package/package.json +1 -1
- package/dist/chunk-HFCIHZHA.js +0 -7
- package/dist/chunk-HFCIHZHA.js.map +0 -1
- package/dist/chunk-X5WDJ7D7.js.map +0 -1
- package/dist/chunk-ZVOVPXDL.js.map +0 -1
- package/dist/server-W4JLSNKW.js.map +0 -1
- /package/dist/{chunk-WWIJ2NID.js.map → chunk-JFOT5JEL.js.map} +0 -0
- /package/dist/{extract-bundle-U3MNYDKC.js.map → extract-bundle-DYMQRQWG.js.map} +0 -0
- /package/dist/{site-extract-repository-CKCJDP7N.js.map → site-extract-repository-G723PZIL.js.map} +0 -0
- /package/dist/{worker-73AYU4IK.js.map → worker-4LZGRHDA.js.map} +0 -0
|
@@ -56,6 +56,7 @@ import {
|
|
|
56
56
|
HttpMcpToolExecutor,
|
|
57
57
|
InternalScheduledArtifactRenderRequestSchema,
|
|
58
58
|
ListScheduledRunsQuerySchema,
|
|
59
|
+
LocalSourcebookSchemaTypeSchema,
|
|
59
60
|
PublicScheduledRunTokenSchema,
|
|
60
61
|
RenewEditorialReadingRoomDownloadInputSchema,
|
|
61
62
|
SCHEDULED_ARTIFACT_TEMPLATE_PRESET,
|
|
@@ -72,11 +73,13 @@ import {
|
|
|
72
73
|
configureReportSaving,
|
|
73
74
|
createConnectedDataArtifact,
|
|
74
75
|
createDirectoryCsvArtifact,
|
|
76
|
+
defaultLocalSourcebookSchemaType,
|
|
75
77
|
exportFanout,
|
|
76
78
|
handleMcpTasksHttpRequest,
|
|
77
79
|
harvestTimeoutBudget,
|
|
78
80
|
hashOwnerId,
|
|
79
81
|
isLoopbackBaseUrl,
|
|
82
|
+
localSourcebookSchemaContract,
|
|
80
83
|
outputBaseDir,
|
|
81
84
|
registerBrowserAgentMcpTools,
|
|
82
85
|
registerMemoryMcpTools,
|
|
@@ -85,8 +88,9 @@ import {
|
|
|
85
88
|
renewConnectedDataArtifactDownload,
|
|
86
89
|
renewDirectoryArtifactDownload,
|
|
87
90
|
resolveDeploymentProfile,
|
|
91
|
+
resolveLocalSourcebookSchemaType,
|
|
88
92
|
transcribeMediaUrl
|
|
89
|
-
} from "./chunk-
|
|
93
|
+
} from "./chunk-FQIICRR4.js";
|
|
90
94
|
import {
|
|
91
95
|
auditImageUrls,
|
|
92
96
|
auditImages,
|
|
@@ -142,7 +146,7 @@ import {
|
|
|
142
146
|
} from "./chunk-3ZUBQQPQ.js";
|
|
143
147
|
import {
|
|
144
148
|
PACKAGE_VERSION
|
|
145
|
-
} from "./chunk-
|
|
149
|
+
} from "./chunk-CEZ3NFBP.js";
|
|
146
150
|
import {
|
|
147
151
|
abandonExtractSettlement,
|
|
148
152
|
countSuccessfulPages,
|
|
@@ -165,7 +169,7 @@ import {
|
|
|
165
169
|
setExtractJobTotal,
|
|
166
170
|
settleExtractJob,
|
|
167
171
|
terminalExtractJobStatus
|
|
168
|
-
} from "./chunk-
|
|
172
|
+
} from "./chunk-JFOT5JEL.js";
|
|
169
173
|
import {
|
|
170
174
|
BROWSER_OPEN_MIN_BALANCE_MC,
|
|
171
175
|
CONCURRENCY_PRICE_ID,
|
|
@@ -178,6 +182,7 @@ import {
|
|
|
178
182
|
CONNECTED_USAGE_RATE_POLICY,
|
|
179
183
|
CONNECTED_USAGE_RATE_POLICY_VERSION,
|
|
180
184
|
CREDIT_COST_CATALOG,
|
|
185
|
+
LISTING_WRITE_RATE_POLICY,
|
|
181
186
|
LedgerOperation,
|
|
182
187
|
MC_COSTS,
|
|
183
188
|
MC_PER_CREDIT,
|
|
@@ -213,7 +218,7 @@ import {
|
|
|
213
218
|
tierChangeLeavesPrivateOffer,
|
|
214
219
|
tierCreditsPerInterval,
|
|
215
220
|
vendorCostUsdToMc
|
|
216
|
-
} from "./chunk-
|
|
221
|
+
} from "./chunk-SOEYDWJU.js";
|
|
217
222
|
import {
|
|
218
223
|
CaptchaError,
|
|
219
224
|
MapsSearchIncompletePageError,
|
|
@@ -8667,7 +8672,7 @@ var siteExtractFn = inngest.createFunction(
|
|
|
8667
8672
|
});
|
|
8668
8673
|
}) : null;
|
|
8669
8674
|
const finalized = await step.run("finalize", async () => {
|
|
8670
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
8675
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-DYMQRQWG.js");
|
|
8671
8676
|
const current = await getExtractJob(jobId2);
|
|
8672
8677
|
if (!current) throw new Error("extract job disappeared before finalization");
|
|
8673
8678
|
const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
|
|
@@ -12033,16 +12038,18 @@ async function createLocalSourcebookSubmission(input) {
|
|
|
12033
12038
|
const existing = await db.execute({ sql: `SELECT * FROM local_sourcebook_submissions WHERE owner_user_id = ? AND idempotency_key = ? LIMIT 1`, args: [input.ownerUserId, input.idempotencyKey] });
|
|
12034
12039
|
if (existing.rows[0]) {
|
|
12035
12040
|
const submission = mapSubmission(existing.rows[0]);
|
|
12036
|
-
if (submission.category !== input.category || submission.state !== input.state || submission.slug !== input.slug || submission.businessName !== input.businessName || submission.websiteUrl !== input.websiteUrl) throw new Error("Local Sourcebook idempotency key conflicts with a different listing identity.");
|
|
12041
|
+
if (submission.category !== input.category || submission.state !== input.state || submission.slug !== input.slug || submission.businessName !== input.businessName || submission.websiteUrl !== input.websiteUrl || input.schemaOrgType != null && submission.coverage.schemaOrgType !== input.schemaOrgType) throw new Error("Local Sourcebook idempotency key conflicts with a different listing identity.");
|
|
12037
12042
|
return submission;
|
|
12038
12043
|
}
|
|
12039
12044
|
}
|
|
12040
12045
|
const id = `lsb_${randomUUID3().replace(/-/g, "")}`;
|
|
12041
12046
|
const coverage = {
|
|
12042
|
-
requested: ["website_crawl", "services_products", "service_areas", "genuine_images", "staff_team", "review_sources"],
|
|
12047
|
+
requested: ["website_crawl", "structured_data", "services_products", "service_areas", "genuine_images", "staff_team", "review_sources"],
|
|
12043
12048
|
crawl: { state: "queued", pagesDiscovered: 0, pagesCaptured: 0 },
|
|
12044
12049
|
media: { state: "queued", genuineImagesCaptured: 0 },
|
|
12045
|
-
reviews: { state: "queued", captured: 0, availableReported: null, stopReason: null }
|
|
12050
|
+
reviews: { state: "queued", captured: 0, availableReported: null, stopReason: null },
|
|
12051
|
+
...input.schemaOrgType ? { schemaOrgType: input.schemaOrgType } : {},
|
|
12052
|
+
...input.billing ? { billing: { mode: "fixed_listing_write", ...input.billing } } : {}
|
|
12046
12053
|
};
|
|
12047
12054
|
await db.batch([
|
|
12048
12055
|
{ sql: `INSERT INTO local_sourcebook_submissions (id, owner_user_id, category, state, slug, business_name, website_url, coverage_json, idempotency_key) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, args: [id, input.ownerUserId, input.category, input.state, input.slug, input.businessName, input.websiteUrl, JSON.stringify(coverage), input.idempotencyKey ?? null] },
|
|
@@ -12062,13 +12069,17 @@ async function getLocalSourcebookDraft(id, ownerUserId) {
|
|
|
12062
12069
|
const result = await getDb().execute({ sql: `SELECT payload_json FROM local_sourcebook_revisions WHERE submission_id = ? AND revision = ?`, args: [id, submission.draftRevision] });
|
|
12063
12070
|
return result.rows[0] ? parseJson2(result.rows[0].payload_json) : null;
|
|
12064
12071
|
}
|
|
12065
|
-
async function queueLocalSourcebookRefresh(id, ownerUserId) {
|
|
12072
|
+
async function queueLocalSourcebookRefresh(id, ownerUserId, billing) {
|
|
12066
12073
|
const existing = await getLocalSourcebookSubmission(id, ownerUserId);
|
|
12067
12074
|
if (!existing) return null;
|
|
12068
12075
|
if (existing.status === "rejected" || existing.status === "unpublished") {
|
|
12069
12076
|
throw new Error(`A ${existing.status} Local Sourcebook listing cannot be refreshed by its owner.`);
|
|
12070
12077
|
}
|
|
12071
|
-
const coverage = {
|
|
12078
|
+
const coverage = {
|
|
12079
|
+
...existing.coverage,
|
|
12080
|
+
refreshRequestedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
12081
|
+
...billing ? { billing: { mode: "fixed_listing_write", ...billing } } : {}
|
|
12082
|
+
};
|
|
12072
12083
|
await getDb().execute({ sql: `UPDATE local_sourcebook_submissions SET status = 'queued', coverage_json = ?, acquisition_claimed_at = NULL, acquisition_error = NULL, updated_at = datetime('now') WHERE id = ? AND owner_user_id = ?`, args: [JSON.stringify(coverage), id, ownerUserId] });
|
|
12073
12084
|
await event(id, "enrichment.queued", "owner", String(ownerUserId));
|
|
12074
12085
|
return getLocalSourcebookSubmission(id, ownerUserId);
|
|
@@ -12774,6 +12785,80 @@ var MapsExtractor = class {
|
|
|
12774
12785
|
}
|
|
12775
12786
|
};
|
|
12776
12787
|
|
|
12788
|
+
// src/api/listing-write-billing.ts
|
|
12789
|
+
function operationFor(surface, kind) {
|
|
12790
|
+
if (surface === "local_sourcebook") {
|
|
12791
|
+
return kind === "create" ? LedgerOperation.LOCAL_SOURCEBOOK_CREATE : LedgerOperation.LOCAL_SOURCEBOOK_EDIT;
|
|
12792
|
+
}
|
|
12793
|
+
return kind === "create" ? LedgerOperation.COMMONS_CREATE : LedgerOperation.COMMONS_EDIT;
|
|
12794
|
+
}
|
|
12795
|
+
function refundOperationFor(surface, kind) {
|
|
12796
|
+
if (surface === "local_sourcebook") {
|
|
12797
|
+
return kind === "create" ? LedgerOperation.LOCAL_SOURCEBOOK_CREATE_REFUND : LedgerOperation.LOCAL_SOURCEBOOK_EDIT_REFUND;
|
|
12798
|
+
}
|
|
12799
|
+
return kind === "create" ? LedgerOperation.COMMONS_CREATE_REFUND : LedgerOperation.COMMONS_EDIT_REFUND;
|
|
12800
|
+
}
|
|
12801
|
+
function listingWritePrice(kind) {
|
|
12802
|
+
return LISTING_WRITE_RATE_POLICY[kind];
|
|
12803
|
+
}
|
|
12804
|
+
function listingWriteDebitKey(surface, userId, idempotencyKey3) {
|
|
12805
|
+
return `listing-write:${surface}:${userId}:${idempotencyKey3}`;
|
|
12806
|
+
}
|
|
12807
|
+
function publicListingWriteBilling(receipt) {
|
|
12808
|
+
return {
|
|
12809
|
+
surface: receipt.surface,
|
|
12810
|
+
operation: receipt.kind,
|
|
12811
|
+
credits: receipt.credits,
|
|
12812
|
+
charged: !receipt.duplicate,
|
|
12813
|
+
idempotentReplay: receipt.duplicate,
|
|
12814
|
+
balanceCredits: receipt.balanceMc / MC_PER_CREDIT,
|
|
12815
|
+
acquisitionIncluded: receipt.surface === "local_sourcebook"
|
|
12816
|
+
};
|
|
12817
|
+
}
|
|
12818
|
+
async function chargeListingWrite(input) {
|
|
12819
|
+
const price = listingWritePrice(input.kind);
|
|
12820
|
+
const debitKey2 = listingWriteDebitKey(input.surface, input.userId, input.idempotencyKey);
|
|
12821
|
+
const debit = await debitMcIdempotent(
|
|
12822
|
+
input.userId,
|
|
12823
|
+
price.mc,
|
|
12824
|
+
operationFor(input.surface, input.kind),
|
|
12825
|
+
`${input.surface}:${input.kind}`,
|
|
12826
|
+
debitKey2
|
|
12827
|
+
);
|
|
12828
|
+
if (!debit.ok) return { insufficient: insufficientBalanceResponse(debit.balance_mc, price.mc) };
|
|
12829
|
+
return {
|
|
12830
|
+
surface: input.surface,
|
|
12831
|
+
kind: input.kind,
|
|
12832
|
+
userId: input.userId,
|
|
12833
|
+
credits: price.credits,
|
|
12834
|
+
amountMc: price.mc,
|
|
12835
|
+
debitKey: debitKey2,
|
|
12836
|
+
duplicate: debit.duplicate,
|
|
12837
|
+
balanceMc: debit.balance_mc
|
|
12838
|
+
};
|
|
12839
|
+
}
|
|
12840
|
+
async function refundListingWrite(receipt, reason) {
|
|
12841
|
+
if (receipt.duplicate) return null;
|
|
12842
|
+
return settleDebitMcIdempotent(
|
|
12843
|
+
receipt.userId,
|
|
12844
|
+
receipt.debitKey,
|
|
12845
|
+
0,
|
|
12846
|
+
refundOperationFor(receipt.surface, receipt.kind),
|
|
12847
|
+
reason,
|
|
12848
|
+
receipt.surface
|
|
12849
|
+
);
|
|
12850
|
+
}
|
|
12851
|
+
async function refundPersistedListingWrite(input) {
|
|
12852
|
+
return settleDebitMcIdempotent(
|
|
12853
|
+
input.userId,
|
|
12854
|
+
input.debitKey,
|
|
12855
|
+
0,
|
|
12856
|
+
refundOperationFor(input.surface, input.kind),
|
|
12857
|
+
input.reason,
|
|
12858
|
+
input.surface
|
|
12859
|
+
);
|
|
12860
|
+
}
|
|
12861
|
+
|
|
12777
12862
|
// src/api/local-sourcebook-compiler.ts
|
|
12778
12863
|
import { createHash as createHash3 } from "crypto";
|
|
12779
12864
|
|
|
@@ -13100,6 +13185,65 @@ function reviewStopReason(input, captured, exposed) {
|
|
|
13100
13185
|
if (exposed && captured < exposed) return `Google Maps stopped exposing additional complete review cards after ${captured} of ${exposed} reported reviews.`;
|
|
13101
13186
|
return `All ${captured} complete, unique review cards exposed during this acquisition were retained.`;
|
|
13102
13187
|
}
|
|
13188
|
+
function buildLocalBusinessJsonLd(input) {
|
|
13189
|
+
const businessId = `${input.profileUrl}#business`;
|
|
13190
|
+
const business = {
|
|
13191
|
+
"@type": input.schemaOrgType === "LocalBusiness" ? "LocalBusiness" : ["LocalBusiness", input.schemaOrgType],
|
|
13192
|
+
"@id": businessId,
|
|
13193
|
+
url: input.profileUrl,
|
|
13194
|
+
name: input.name,
|
|
13195
|
+
description: input.description,
|
|
13196
|
+
...input.phone ? { telephone: input.phone } : {},
|
|
13197
|
+
...input.address || input.locality || input.region || input.postalCode ? {
|
|
13198
|
+
address: {
|
|
13199
|
+
"@type": "PostalAddress",
|
|
13200
|
+
...input.address ? { streetAddress: input.address } : {},
|
|
13201
|
+
...input.locality ? { addressLocality: input.locality } : {},
|
|
13202
|
+
...input.region ? { addressRegion: input.region } : {},
|
|
13203
|
+
...input.postalCode ? { postalCode: input.postalCode } : {},
|
|
13204
|
+
addressCountry: "US"
|
|
13205
|
+
}
|
|
13206
|
+
} : {},
|
|
13207
|
+
...Number.isFinite(input.latitude) && Number.isFinite(input.longitude) ? {
|
|
13208
|
+
geo: { "@type": "GeoCoordinates", latitude: input.latitude, longitude: input.longitude }
|
|
13209
|
+
} : {},
|
|
13210
|
+
...input.images.length ? { image: input.images } : {},
|
|
13211
|
+
...input.serviceAreas.length ? { areaServed: input.serviceAreas } : {},
|
|
13212
|
+
...input.services.length ? {
|
|
13213
|
+
hasOfferCatalog: {
|
|
13214
|
+
"@type": "OfferCatalog",
|
|
13215
|
+
name: "Services",
|
|
13216
|
+
itemListElement: input.services.map((name) => ({ "@type": "Offer", itemOffered: { "@type": "Service", name } }))
|
|
13217
|
+
}
|
|
13218
|
+
} : {},
|
|
13219
|
+
...Number.isFinite(input.rating) && Number.isFinite(input.ratingCount) && Number(input.ratingCount) > 0 ? {
|
|
13220
|
+
aggregateRating: { "@type": "AggregateRating", ratingValue: input.rating, reviewCount: input.ratingCount, bestRating: 5, worstRating: 1 }
|
|
13221
|
+
} : {},
|
|
13222
|
+
sameAs: unique([input.website, input.mapsUrl], 10)
|
|
13223
|
+
};
|
|
13224
|
+
const graph = [business, {
|
|
13225
|
+
"@type": "BreadcrumbList",
|
|
13226
|
+
"@id": `${input.profileUrl}#breadcrumbs`,
|
|
13227
|
+
itemListElement: [
|
|
13228
|
+
{ "@type": "ListItem", position: 1, name: "Local Sourcebook", item: "https://localsourcebook.com/" },
|
|
13229
|
+
{ "@type": "ListItem", position: 2, name: input.region || "State directory", item: input.stateUrl },
|
|
13230
|
+
{ "@type": "ListItem", position: 3, name: input.name, item: input.profileUrl }
|
|
13231
|
+
]
|
|
13232
|
+
}];
|
|
13233
|
+
if (input.faq.length) {
|
|
13234
|
+
graph.push({
|
|
13235
|
+
"@type": "FAQPage",
|
|
13236
|
+
"@id": `${input.profileUrl}#faq`,
|
|
13237
|
+
about: { "@id": businessId },
|
|
13238
|
+
mainEntity: input.faq.map((item) => ({
|
|
13239
|
+
"@type": "Question",
|
|
13240
|
+
name: item.question,
|
|
13241
|
+
acceptedAnswer: { "@type": "Answer", text: item.answer }
|
|
13242
|
+
}))
|
|
13243
|
+
});
|
|
13244
|
+
}
|
|
13245
|
+
return { "@context": "https://schema.org", "@graph": graph };
|
|
13246
|
+
}
|
|
13103
13247
|
function compileLocalSourcebookListing(input) {
|
|
13104
13248
|
const collectedAt = input.collectedAt ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
13105
13249
|
const pages = successfulPages(input.site);
|
|
@@ -13169,6 +13313,33 @@ function compileLocalSourcebookListing(input) {
|
|
|
13169
13313
|
}
|
|
13170
13314
|
];
|
|
13171
13315
|
const publicUrls = localSourcebookPublicUrls(input.submission.category, input.submission.state, input.submission.slug);
|
|
13316
|
+
const schemaOrgType = resolveLocalSourcebookSchemaType(input.submission.category, input.submission.coverage.schemaOrgType);
|
|
13317
|
+
const listingName = maps?.name || input.submission.businessName;
|
|
13318
|
+
const listingWebsite = maps?.website || input.submission.websiteUrl;
|
|
13319
|
+
const listingMapsUrl = maps?.placeUrl || maps?.cidUrl || void 0;
|
|
13320
|
+
const stateUrl = new URL(`/${input.submission.state.toLowerCase()}`, publicUrls.profileUrl).href;
|
|
13321
|
+
const jsonLd = buildLocalBusinessJsonLd({
|
|
13322
|
+
schemaOrgType,
|
|
13323
|
+
profileUrl: publicUrls.profileUrl,
|
|
13324
|
+
stateUrl,
|
|
13325
|
+
name: listingName,
|
|
13326
|
+
description: summary,
|
|
13327
|
+
address: location2.address,
|
|
13328
|
+
locality: location2.locality,
|
|
13329
|
+
region: location2.region,
|
|
13330
|
+
postalCode: location2.postalCode,
|
|
13331
|
+
phone: maps?.phoneDisplay || maps?.phone || void 0,
|
|
13332
|
+
website: listingWebsite,
|
|
13333
|
+
mapsUrl: listingMapsUrl,
|
|
13334
|
+
latitude: maps?.lat ?? void 0,
|
|
13335
|
+
longitude: maps?.lng ?? void 0,
|
|
13336
|
+
services,
|
|
13337
|
+
serviceAreas,
|
|
13338
|
+
images: media.map((item) => String(item.src)),
|
|
13339
|
+
rating: Number.isFinite(rating) ? rating : void 0,
|
|
13340
|
+
ratingCount: Number.isFinite(exposedCount) ? exposedCount : void 0,
|
|
13341
|
+
faq
|
|
13342
|
+
});
|
|
13172
13343
|
const listing = {
|
|
13173
13344
|
id: input.submission.id,
|
|
13174
13345
|
publicationRevision: input.submission.draftRevision + 1,
|
|
@@ -13185,7 +13356,9 @@ function compileLocalSourcebookListing(input) {
|
|
|
13185
13356
|
profile: publicUrls.profileUrl,
|
|
13186
13357
|
reviews: publicUrls.reviewsUrl
|
|
13187
13358
|
},
|
|
13188
|
-
name:
|
|
13359
|
+
name: listingName,
|
|
13360
|
+
schemaOrgType,
|
|
13361
|
+
jsonLd,
|
|
13189
13362
|
summary,
|
|
13190
13363
|
description,
|
|
13191
13364
|
address: location2.address,
|
|
@@ -13193,8 +13366,8 @@ function compileLocalSourcebookListing(input) {
|
|
|
13193
13366
|
region: location2.region,
|
|
13194
13367
|
postalCode: location2.postalCode,
|
|
13195
13368
|
phone: maps?.phoneDisplay || maps?.phone || void 0,
|
|
13196
|
-
website:
|
|
13197
|
-
mapsUrl:
|
|
13369
|
+
website: listingWebsite,
|
|
13370
|
+
mapsUrl: listingMapsUrl,
|
|
13198
13371
|
latitude: maps?.lat ?? void 0,
|
|
13199
13372
|
longitude: maps?.lng ?? void 0,
|
|
13200
13373
|
hours,
|
|
@@ -13234,11 +13407,12 @@ function compileLocalSourcebookListing(input) {
|
|
|
13234
13407
|
return {
|
|
13235
13408
|
listing,
|
|
13236
13409
|
coverage: {
|
|
13237
|
-
requested: ["website_crawl", "services_products", "service_areas", "genuine_images", "staff_team", "review_sources"],
|
|
13410
|
+
requested: ["website_crawl", "structured_data", "services_products", "service_areas", "genuine_images", "staff_team", "review_sources"],
|
|
13238
13411
|
crawl: { state: input.siteError ? "failed" : input.site?.truncated ? "partial" : "complete", pagesDiscovered: input.site?.spider.totalFound ?? 0, pagesCaptured: pages.length, requestedMaximum: input.requestedMaxPages, error: input.siteError ?? null },
|
|
13239
13412
|
media: { state: input.siteError ? "failed" : "complete", genuineImagesCaptured: media.length, teamImagesCaptured: media.filter((item) => item.kind === "team").length },
|
|
13240
13413
|
reviews: { state: input.mapsError || input.reviewBillingLimited ? "partial" : "complete", captured: reviews.length, availableReported: Number.isFinite(exposedCount) ? exposedCount : null, requestedMaximum: input.requestedMaxReviews, stopReason: reviewStopReason(input, reviews.length, Number.isFinite(exposedCount) ? exposedCount : void 0) },
|
|
13241
13414
|
maps: { state: maps ? "complete" : "failed", exactPlaceUrl: maps?.placeUrl ?? null, cid: maps?.cidDecimal ?? maps?.cid ?? null, error: input.mapsError ?? null },
|
|
13415
|
+
structuredData: { state: "complete", rootType: "LocalBusiness", schemaOrgType, graphNodes: jsonLd["@graph"].length },
|
|
13242
13416
|
compiledAt: collectedAt
|
|
13243
13417
|
}
|
|
13244
13418
|
};
|
|
@@ -13323,110 +13497,83 @@ async function runLocalSourcebookAcquisition(submission, provider = createDefaul
|
|
|
13323
13497
|
const user = await getUserById(submission.ownerUserId);
|
|
13324
13498
|
if (!user) throw new Error("The submission owner is no longer active.");
|
|
13325
13499
|
if (!user.subscription_tier) throw new Error("An active MCP Scraper subscription is required for acquisition.");
|
|
13326
|
-
const
|
|
13327
|
-
const
|
|
13328
|
-
const
|
|
13329
|
-
|
|
13330
|
-
|
|
13331
|
-
|
|
13332
|
-
|
|
13333
|
-
|
|
13334
|
-
|
|
13335
|
-
|
|
13336
|
-
|
|
13337
|
-
|
|
13500
|
+
const storedBilling = submission.coverage.billing;
|
|
13501
|
+
const inferredKind = submission.publishedRevision == null ? "create" : "edit";
|
|
13502
|
+
const kind = storedBilling?.kind === "edit" ? "edit" : storedBilling?.kind === "create" ? "create" : inferredKind;
|
|
13503
|
+
const price = listingWritePrice(kind);
|
|
13504
|
+
let debitKey2 = typeof storedBilling?.debitKey === "string" ? storedBilling.debitKey : "";
|
|
13505
|
+
if (!debitKey2) {
|
|
13506
|
+
const fallback = await chargeListingWrite({
|
|
13507
|
+
surface: "local_sourcebook",
|
|
13508
|
+
kind,
|
|
13509
|
+
userId: submission.ownerUserId,
|
|
13510
|
+
idempotencyKey: `worker:${submission.id}:revision:${submission.draftRevision}`
|
|
13511
|
+
});
|
|
13512
|
+
if ("insufficient" in fallback) throw new Error(fallback.insufficient.message);
|
|
13513
|
+
debitKey2 = fallback.debitKey;
|
|
13514
|
+
}
|
|
13515
|
+
const maxPages = LOCAL_SOURCEBOOK_MAX_PAGES;
|
|
13338
13516
|
let site = null;
|
|
13339
13517
|
let maps = null;
|
|
13340
13518
|
let siteError = null;
|
|
13341
13519
|
let mapsError = null;
|
|
13342
|
-
let reviewBillingLimited = false;
|
|
13343
|
-
let siteHoldMc = 0;
|
|
13344
|
-
let siteActualMc = 0;
|
|
13345
|
-
let mapsRefunded = false;
|
|
13346
|
-
let reviewsBilledMc = 0;
|
|
13347
|
-
if (maxPages < 1) {
|
|
13348
|
-
siteError = "The subscriber balance covered exact-place acquisition but not a website page.";
|
|
13349
|
-
} else {
|
|
13350
|
-
siteHoldMc = maxPages * MC_COSTS.page_scrape;
|
|
13351
|
-
const siteDescription = `Local Sourcebook: ${new URL(submission.websiteUrl).hostname}`;
|
|
13352
|
-
const siteHold = await debitMcIdempotent(
|
|
13353
|
-
submission.ownerUserId,
|
|
13354
|
-
siteHoldMc,
|
|
13355
|
-
LedgerOperation.EXTRACT_SITE_HOLD,
|
|
13356
|
-
siteDescription,
|
|
13357
|
-
`${billingCycle}:website-hold`
|
|
13358
|
-
);
|
|
13359
|
-
if (!siteHold.ok) {
|
|
13360
|
-
siteError = "The subscriber balance changed before the website acquisition hold could be placed.";
|
|
13361
|
-
siteHoldMc = 0;
|
|
13362
|
-
} else {
|
|
13363
|
-
try {
|
|
13364
|
-
site = await provider.extractWebsite(submission, maxPages);
|
|
13365
|
-
const successfulPages2 = site.pages.filter(isPageExtractionSuccessful).length;
|
|
13366
|
-
siteActualMc = successfulPages2 * MC_COSTS.page_scrape;
|
|
13367
|
-
const refund = siteHoldMc - siteActualMc;
|
|
13368
|
-
if (refund > 0) await creditMcIdempotent(submission.ownerUserId, refund, LedgerOperation.EXTRACT_SITE_REFUND, `Local Sourcebook unused website-page hold: ${submission.id}`, `${billingCycle}:website-unused-refund`);
|
|
13369
|
-
await logRequestEvent({ userId: submission.ownerUserId, source: "local_sourcebook_website", status: "done", query: submission.websiteUrl, resultCount: successfulPages2, result: { submissionId: submission.id, discovered: site.spider.totalFound, captured: successfulPages2, truncated: site.truncated } });
|
|
13370
|
-
} catch (error) {
|
|
13371
|
-
siteError = errorMessage2(error);
|
|
13372
|
-
await creditMcIdempotent(submission.ownerUserId, siteHoldMc, LedgerOperation.EXTRACT_SITE_REFUND, `Local Sourcebook failed website acquisition: ${submission.id}`, `${billingCycle}:website-failure-refund`);
|
|
13373
|
-
siteHoldMc = 0;
|
|
13374
|
-
await logRequestEvent({ userId: submission.ownerUserId, source: "local_sourcebook_website", status: "failed", query: submission.websiteUrl, error: siteError });
|
|
13375
|
-
}
|
|
13376
|
-
}
|
|
13377
|
-
}
|
|
13378
13520
|
try {
|
|
13379
|
-
|
|
13380
|
-
|
|
13381
|
-
|
|
13382
|
-
|
|
13383
|
-
|
|
13384
|
-
|
|
13385
|
-
|
|
13386
|
-
reviewCost,
|
|
13387
|
-
LedgerOperation.MAPS_REVIEW,
|
|
13388
|
-
`Local Sourcebook reviews: ${submission.businessName}`,
|
|
13389
|
-
`${billingCycle}:maps-reviews`
|
|
13390
|
-
);
|
|
13391
|
-
if (debit.ok) {
|
|
13392
|
-
reviewsBilledMc = reviewCost;
|
|
13393
|
-
} else {
|
|
13394
|
-
reviewBillingLimited = true;
|
|
13395
|
-
maps.reviews = [];
|
|
13396
|
-
}
|
|
13521
|
+
try {
|
|
13522
|
+
site = await provider.extractWebsite(submission, maxPages);
|
|
13523
|
+
const successfulPages2 = site.pages.filter(isPageExtractionSuccessful).length;
|
|
13524
|
+
await logRequestEvent({ userId: submission.ownerUserId, source: "local_sourcebook_website", status: "done", query: submission.websiteUrl, resultCount: successfulPages2, result: { submissionId: submission.id, discovered: site.spider.totalFound, captured: successfulPages2, truncated: site.truncated } });
|
|
13525
|
+
} catch (error) {
|
|
13526
|
+
siteError = errorMessage2(error);
|
|
13527
|
+
await logRequestEvent({ userId: submission.ownerUserId, source: "local_sourcebook_website", status: "failed", query: submission.websiteUrl, error: siteError });
|
|
13397
13528
|
}
|
|
13398
|
-
|
|
13529
|
+
try {
|
|
13530
|
+
maps = await provider.extractMaps(submission, LOCAL_SOURCEBOOK_MAX_REVIEWS);
|
|
13531
|
+
validateLocalSourcebookMapsIdentity(submission, maps);
|
|
13532
|
+
await logRequestEvent({ userId: submission.ownerUserId, source: "local_sourcebook_maps", status: "done", query: submission.businessName, location: submission.state.toUpperCase(), resultCount: maps.reviews.length, result: { submissionId: submission.id, placeUrl: maps.placeUrl, cid: maps.cidDecimal ?? maps.cid, reviewsStatus: maps.reviewsStatus, reviewsRetained: maps.reviews.length, services: maps.services.length, areasServed: maps.areasServed.length } });
|
|
13533
|
+
} catch (error) {
|
|
13534
|
+
mapsError = errorMessage2(error);
|
|
13535
|
+
maps = null;
|
|
13536
|
+
await logRequestEvent({ userId: submission.ownerUserId, source: "local_sourcebook_maps", status: "failed", query: submission.businessName, location: submission.state.toUpperCase(), error: mapsError });
|
|
13537
|
+
}
|
|
13538
|
+
if (!maps) throw new Error(`Exact-place acquisition is required before publication. Maps: ${mapsError || "not available"}. Website: ${siteError || (site ? "captured" : "not available")}.`);
|
|
13539
|
+
const compiled = compileLocalSourcebookListing({
|
|
13540
|
+
submission,
|
|
13541
|
+
site,
|
|
13542
|
+
maps,
|
|
13543
|
+
requestedMaxPages: maxPages,
|
|
13544
|
+
requestedMaxReviews: LOCAL_SOURCEBOOK_MAX_REVIEWS,
|
|
13545
|
+
siteError,
|
|
13546
|
+
mapsError,
|
|
13547
|
+
reviewBillingLimited: false
|
|
13548
|
+
});
|
|
13549
|
+
const coverage = {
|
|
13550
|
+
...compiled.coverage,
|
|
13551
|
+
billing: {
|
|
13552
|
+
mode: "fixed_listing_write",
|
|
13553
|
+
kind,
|
|
13554
|
+
credits: price.credits,
|
|
13555
|
+
amountMc: price.mc,
|
|
13556
|
+
debitKey: debitKey2,
|
|
13557
|
+
acquisitionIncluded: true
|
|
13558
|
+
}
|
|
13559
|
+
};
|
|
13560
|
+
const completed = await completeLocalSourcebookAcquisition(submission.id, compiled.listing, coverage, submission.draftRevision);
|
|
13561
|
+
if (!completed) throw new Error("The Local Sourcebook draft could not be saved after acquisition.");
|
|
13562
|
+
return completed;
|
|
13399
13563
|
} catch (error) {
|
|
13400
|
-
|
|
13401
|
-
|
|
13402
|
-
|
|
13403
|
-
|
|
13404
|
-
|
|
13405
|
-
|
|
13406
|
-
|
|
13407
|
-
|
|
13408
|
-
|
|
13409
|
-
|
|
13410
|
-
maps,
|
|
13411
|
-
requestedMaxPages: maxPages,
|
|
13412
|
-
requestedMaxReviews: LOCAL_SOURCEBOOK_MAX_REVIEWS,
|
|
13413
|
-
siteError,
|
|
13414
|
-
mapsError,
|
|
13415
|
-
reviewBillingLimited
|
|
13416
|
-
});
|
|
13417
|
-
const coverage = {
|
|
13418
|
-
...compiled.coverage,
|
|
13419
|
-
billing: {
|
|
13420
|
-
websiteHeldMc: siteHoldMc,
|
|
13421
|
-
websiteUsedMc: siteActualMc,
|
|
13422
|
-
mapsUsedMc: mapsRefunded ? 0 : MC_COSTS.maps_place,
|
|
13423
|
-
reviewsUsedMc: reviewsBilledMc,
|
|
13424
|
-
reviewBillingLimited
|
|
13564
|
+
const current = await getLocalSourcebookSubmission(submission.id).catch(() => null);
|
|
13565
|
+
const revisionAlreadyPublished = current?.status === "published" && current.publishedRevision != null && current.draftRevision > submission.draftRevision;
|
|
13566
|
+
if (!revisionAlreadyPublished) {
|
|
13567
|
+
await refundPersistedListingWrite({
|
|
13568
|
+
surface: "local_sourcebook",
|
|
13569
|
+
kind,
|
|
13570
|
+
userId: submission.ownerUserId,
|
|
13571
|
+
debitKey: debitKey2,
|
|
13572
|
+
reason: `Local Sourcebook acquisition failed: ${errorMessage2(error)}`
|
|
13573
|
+
}).catch(() => null);
|
|
13425
13574
|
}
|
|
13426
|
-
|
|
13427
|
-
|
|
13428
|
-
if (!completed) throw new Error("The Local Sourcebook draft could not be saved after acquisition.");
|
|
13429
|
-
return completed;
|
|
13575
|
+
throw error;
|
|
13576
|
+
}
|
|
13430
13577
|
}
|
|
13431
13578
|
async function processLocalSourcebookSubmission(submissionId) {
|
|
13432
13579
|
const submission = await claimLocalSourcebookAcquisition(submissionId);
|
|
@@ -21322,6 +21469,7 @@ var DirectoryIdentitySchema = z25.object({
|
|
|
21322
21469
|
state: z25.string().trim().length(2).transform((value) => value.toLowerCase()),
|
|
21323
21470
|
businessName: z25.string().trim().min(2).max(160),
|
|
21324
21471
|
websiteUrl: z25.url().refine((value) => /^https?:\/\//.test(value), "websiteUrl must use http or https"),
|
|
21472
|
+
schemaOrgType: LocalSourcebookSchemaTypeSchema.optional(),
|
|
21325
21473
|
slug: z25.string().trim().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/).max(100).optional(),
|
|
21326
21474
|
tags: z25.array(z25.string().trim().min(1).max(60)).max(20).default([]),
|
|
21327
21475
|
idempotencyKey: z25.string().trim().min(8).max(200)
|
|
@@ -21406,7 +21554,7 @@ async function resolveLocalSourcebookTags(candidates) {
|
|
|
21406
21554
|
function getLocalSourcebookContract(category) {
|
|
21407
21555
|
return {
|
|
21408
21556
|
name: "Local Sourcebook listing",
|
|
21409
|
-
version: "
|
|
21557
|
+
version: "local_sourcebook_listing_v3",
|
|
21410
21558
|
purpose: "An owner-scoped, evidence-backed local business record with a separate public profile and complete captured-review archive.",
|
|
21411
21559
|
categories: LOCAL_SOURCEBOOK_CATEGORIES,
|
|
21412
21560
|
selectedCategory: category ?? null,
|
|
@@ -21417,11 +21565,14 @@ function getLocalSourcebookContract(category) {
|
|
|
21417
21565
|
reviews: "https://{category}.localsourcebook.com/{state}/{business-slug}/reviews"
|
|
21418
21566
|
},
|
|
21419
21567
|
requiredCreateFields: ["category", "state", "businessName", "websiteUrl", "tags", "idempotencyKey"],
|
|
21420
|
-
|
|
21568
|
+
optionalCreateFields: ["schemaOrgType"],
|
|
21569
|
+
systemCompiledSections: ["identity", "LocalBusiness JSON-LD", "services/products", "service areas/locations", "genuine media", "reviews and theme counts", "evidence", "answered FAQ", "collection receipts"],
|
|
21421
21570
|
workflow: ["list-local-sourcebook-tags", "get-local-sourcebook-contract", "prepare-local-sourcebook-write", "validate-local-sourcebook-write", "local-sourcebook-capture", "local_sourcebook_submission_status"],
|
|
21422
21571
|
ownership: "Only the MCP Scraper account that captures a listing can read it or request a paid evidence refresh.",
|
|
21572
|
+
billing: { validationCredits: 0, createCredits: 10, editCredits: 2, acquisitionIncluded: true, chargeEvent: "accepted_write" },
|
|
21423
21573
|
acquisition: { websitePagesMaximum: 100, reviewsMaximum: 500, exactMapsIdentityRequired: true, partialResultsRequireStoppingReasons: true },
|
|
21424
21574
|
publication: { automatic: true, evidenceCompiledOnly: true, ownerAuthoredClaimsAccepted: false, adminReviewRequired: false, lastPublishedRevisionRemainsPublicDuringRefresh: true },
|
|
21575
|
+
structuredData: localSourcebookSchemaContract(category),
|
|
21425
21576
|
tagPolicy: { inspectVocabularyFirst: true, minimum: 1, maximum: 20, newTagsRequire: ["central=true", "reusable=true", "description"], newTagsStartAs: "pending" }
|
|
21426
21577
|
};
|
|
21427
21578
|
}
|
|
@@ -21436,17 +21587,20 @@ async function prepareLocalSourcebookWrite(input) {
|
|
|
21436
21587
|
];
|
|
21437
21588
|
const tagResolutions = await resolveLocalSourcebookTags(candidates);
|
|
21438
21589
|
const publicUrls = localSourcebookPublicUrls(parsed.category, parsed.state, slug2);
|
|
21590
|
+
const schemaOrgType = parsed.schemaOrgType ?? defaultLocalSourcebookSchemaType(parsed.category);
|
|
21439
21591
|
return {
|
|
21440
21592
|
ok: true,
|
|
21441
21593
|
route: { category: parsed.category, state: parsed.state, slug: slug2, ...publicUrls },
|
|
21442
|
-
proposedWrite: { ...parsed, slug: slug2, tags: tagResolutions.filter((item) => item.action === "reuse").map((item) => item.tag) },
|
|
21594
|
+
proposedWrite: { ...parsed, schemaOrgType, slug: slug2, tags: tagResolutions.filter((item) => item.action === "reuse").map((item) => item.tag) },
|
|
21443
21595
|
contract: getLocalSourcebookContract(parsed.category),
|
|
21444
21596
|
tagResolutions,
|
|
21445
21597
|
instructions: [
|
|
21446
21598
|
"Resolve every review candidate before capture; do not silently create a near-duplicate tag.",
|
|
21447
21599
|
"Call validate-local-sourcebook-write with the proposed identity, resolved tags, and any required tag decisions.",
|
|
21600
|
+
"Keep the proposed schemaOrgType or choose a more specific accepted LocalBusiness subtype from the contract. Arbitrary JSON-LD is not accepted.",
|
|
21448
21601
|
"Call local-sourcebook-capture only after validation returns valid=true. Capture queues paid acquisition and automatically publishes the resulting system-compiled evidence revision.",
|
|
21449
|
-
"
|
|
21602
|
+
"Creating the listing costs 10 Credits total, including acquisition. Validation is free.",
|
|
21603
|
+
"Owners cannot supply public listing claims directly. A source-backed local_sourcebook_refresh costs 2 Credits."
|
|
21450
21604
|
]
|
|
21451
21605
|
};
|
|
21452
21606
|
}
|
|
@@ -21456,7 +21610,10 @@ async function validateLocalSourcebookWrite(input) {
|
|
|
21456
21610
|
let normalizedIdentity2;
|
|
21457
21611
|
const parsedIdentity = DirectoryIdentitySchema.safeParse(input.identity);
|
|
21458
21612
|
if (!parsedIdentity.success) errors.push(...parsedIdentity.error.issues.map((issue) => `identity.${issue.path.join(".")}: ${issue.message}`));
|
|
21459
|
-
else normalizedIdentity2 =
|
|
21613
|
+
else normalizedIdentity2 = {
|
|
21614
|
+
...parsedIdentity.data,
|
|
21615
|
+
schemaOrgType: parsedIdentity.data.schemaOrgType ?? defaultLocalSourcebookSchemaType(parsedIdentity.data.category)
|
|
21616
|
+
};
|
|
21460
21617
|
const candidates = input.tagCandidates ?? (normalizedIdentity2?.tags.map((tag) => ({ tag })) ?? []);
|
|
21461
21618
|
const resolutions = await resolveLocalSourcebookTags(candidates);
|
|
21462
21619
|
const decisions = new Map((input.tagDecisions ?? []).map((decision) => [normalizeTag(decision.tag), decision]));
|
|
@@ -21505,6 +21662,7 @@ var SubmitSchema = z26.object({
|
|
|
21505
21662
|
state: z26.string().trim().length(2).transform((value) => value.toLowerCase()),
|
|
21506
21663
|
businessName: z26.string().trim().min(2).max(160),
|
|
21507
21664
|
websiteUrl: z26.url().refine((value) => /^https?:\/\//.test(value), "websiteUrl must use http or https"),
|
|
21665
|
+
schemaOrgType: LocalSourcebookSchemaTypeSchema.optional(),
|
|
21508
21666
|
slug: z26.string().trim().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/).max(100).optional(),
|
|
21509
21667
|
idempotencyKey: z26.string().trim().min(8).max(200).optional()
|
|
21510
21668
|
}).strict();
|
|
@@ -21561,13 +21719,19 @@ localSourcebookApp.post("/capture", createApiKeyAuth(), async (c) => {
|
|
|
21561
21719
|
if (!parsed.success) return c.json({ error: "invalid_capture_request", issues: parsed.error.issues }, 400);
|
|
21562
21720
|
const validation = await validateLocalSourcebookWrite(parsed.data);
|
|
21563
21721
|
if (!validation.valid) return c.json({ error: "invalid_directory_write", ...validation }, 400);
|
|
21722
|
+
let billing = null;
|
|
21723
|
+
let persisted = false;
|
|
21564
21724
|
try {
|
|
21565
21725
|
const identity = validation.normalizedIdentity;
|
|
21566
|
-
|
|
21726
|
+
const idempotencyKey3 = parsed.data.idempotencyKey ?? identity.idempotencyKey;
|
|
21727
|
+
if (!idempotencyKey3) {
|
|
21567
21728
|
return c.json({ error: "idempotency_key_required", message: "A stable idempotencyKey is required for a new Local Sourcebook capture." }, 400);
|
|
21568
21729
|
}
|
|
21569
21730
|
const listingSlug = identity.slug ?? slug(identity.businessName);
|
|
21570
21731
|
const publicUrls = localSourcebookPublicUrls(identity.category, identity.state, listingSlug);
|
|
21732
|
+
const charged = await chargeListingWrite({ surface: "local_sourcebook", kind: "create", userId: Number(user.id), idempotencyKey: idempotencyKey3 });
|
|
21733
|
+
if ("insufficient" in charged) return c.json(charged.insufficient, 402);
|
|
21734
|
+
billing = charged;
|
|
21571
21735
|
const submission = await createLocalSourcebookSubmission({
|
|
21572
21736
|
ownerUserId: Number(user.id),
|
|
21573
21737
|
category: identity.category,
|
|
@@ -21575,7 +21739,9 @@ localSourcebookApp.post("/capture", createApiKeyAuth(), async (c) => {
|
|
|
21575
21739
|
slug: listingSlug,
|
|
21576
21740
|
businessName: identity.businessName,
|
|
21577
21741
|
websiteUrl: identity.websiteUrl,
|
|
21578
|
-
|
|
21742
|
+
schemaOrgType: identity.schemaOrgType,
|
|
21743
|
+
idempotencyKey: idempotencyKey3,
|
|
21744
|
+
billing: { kind: billing.kind, credits: billing.credits, amountMc: billing.amountMc, debitKey: billing.debitKey },
|
|
21579
21745
|
payload: {
|
|
21580
21746
|
id: "",
|
|
21581
21747
|
category: identity.category,
|
|
@@ -21583,6 +21749,7 @@ localSourcebookApp.post("/capture", createApiKeyAuth(), async (c) => {
|
|
|
21583
21749
|
slug: listingSlug,
|
|
21584
21750
|
name: identity.businessName,
|
|
21585
21751
|
websiteUrl: identity.websiteUrl,
|
|
21752
|
+
schemaOrgType: identity.schemaOrgType,
|
|
21586
21753
|
tags: validation.normalizedTags,
|
|
21587
21754
|
canonicalPaths: { profile: publicUrls.profilePath, reviews: publicUrls.reviewsPath },
|
|
21588
21755
|
canonicalUrls: { profile: publicUrls.profileUrl, reviews: publicUrls.reviewsUrl },
|
|
@@ -21593,10 +21760,12 @@ localSourcebookApp.post("/capture", createApiKeyAuth(), async (c) => {
|
|
|
21593
21760
|
reviews: []
|
|
21594
21761
|
}
|
|
21595
21762
|
});
|
|
21763
|
+
persisted = true;
|
|
21596
21764
|
await persistLocalSourcebookTags(submission.id, Number(user.id), validation.normalizedTags, parsed.data.tagDecisions);
|
|
21597
21765
|
const dispatched = await dispatchLocalSourcebookAcquisition(submission.id).catch(() => false);
|
|
21598
|
-
return c.json({ ok: true, valid: true, submission, publicUrls: localSourcebookPublicationLinks(submission), normalizedTags: validation.normalizedTags, acquisition: { queued: true, dispatched }, acquisitionPlan: acquisitionPlan(identity.websiteUrl) }, 202);
|
|
21766
|
+
return c.json({ ok: true, valid: true, submission, publicUrls: localSourcebookPublicationLinks(submission), normalizedTags: validation.normalizedTags, acquisition: { queued: true, dispatched }, acquisitionPlan: acquisitionPlan(identity.websiteUrl), billing: publicListingWriteBilling(billing) }, 202);
|
|
21599
21767
|
} catch (error) {
|
|
21768
|
+
if (billing && !persisted) await refundListingWrite(billing, "Local Sourcebook creation failed before persistence.").catch(() => null);
|
|
21600
21769
|
const message = error instanceof Error ? error.message : String(error);
|
|
21601
21770
|
const conflict = message.includes("revision conflict") ? "revision_conflict" : message.includes("idempotency key conflicts") ? "idempotency_conflict" : null;
|
|
21602
21771
|
return c.json({ error: conflict ?? "capture_failed", message }, conflict ? 409 : 400);
|
|
@@ -21608,13 +21777,22 @@ localSourcebookApp.post("/submissions", createApiKeyAuth(), async (c) => {
|
|
|
21608
21777
|
const parsed = SubmitSchema.safeParse(await c.req.json().catch(() => null));
|
|
21609
21778
|
if (!parsed.success) return c.json({ error: "invalid_submission", issues: parsed.error.issues }, 400);
|
|
21610
21779
|
const input = parsed.data;
|
|
21780
|
+
if (!input.idempotencyKey) return c.json({ error: "idempotency_key_required", message: "A stable idempotencyKey is required for a new Local Sourcebook listing." }, 400);
|
|
21611
21781
|
const listingSlug = input.slug ?? slug(input.businessName);
|
|
21782
|
+
const schemaOrgType = input.schemaOrgType ?? defaultLocalSourcebookSchemaType(input.category);
|
|
21612
21783
|
const publicUrls = localSourcebookPublicUrls(input.category, input.state, listingSlug);
|
|
21784
|
+
let billing = null;
|
|
21785
|
+
let persisted = false;
|
|
21613
21786
|
try {
|
|
21787
|
+
const charged = await chargeListingWrite({ surface: "local_sourcebook", kind: "create", userId: Number(user.id), idempotencyKey: input.idempotencyKey });
|
|
21788
|
+
if ("insufficient" in charged) return c.json(charged.insufficient, 402);
|
|
21789
|
+
billing = charged;
|
|
21614
21790
|
const submission = await createLocalSourcebookSubmission({
|
|
21615
21791
|
ownerUserId: Number(user.id),
|
|
21616
21792
|
...input,
|
|
21793
|
+
schemaOrgType,
|
|
21617
21794
|
slug: listingSlug,
|
|
21795
|
+
billing: { kind: billing.kind, credits: billing.credits, amountMc: billing.amountMc, debitKey: billing.debitKey },
|
|
21618
21796
|
payload: {
|
|
21619
21797
|
id: "",
|
|
21620
21798
|
category: input.category,
|
|
@@ -21622,6 +21800,7 @@ localSourcebookApp.post("/submissions", createApiKeyAuth(), async (c) => {
|
|
|
21622
21800
|
slug: listingSlug,
|
|
21623
21801
|
name: input.businessName,
|
|
21624
21802
|
websiteUrl: input.websiteUrl,
|
|
21803
|
+
schemaOrgType,
|
|
21625
21804
|
canonicalPaths: { profile: publicUrls.profilePath, reviews: publicUrls.reviewsPath },
|
|
21626
21805
|
canonicalUrls: { profile: publicUrls.profileUrl, reviews: publicUrls.reviewsUrl },
|
|
21627
21806
|
verificationState: "submitted",
|
|
@@ -21631,9 +21810,11 @@ localSourcebookApp.post("/submissions", createApiKeyAuth(), async (c) => {
|
|
|
21631
21810
|
reviews: []
|
|
21632
21811
|
}
|
|
21633
21812
|
});
|
|
21813
|
+
persisted = true;
|
|
21634
21814
|
const dispatched = await dispatchLocalSourcebookAcquisition(submission.id).catch(() => false);
|
|
21635
|
-
return c.json({ ok: true, submission, publicUrls: localSourcebookPublicationLinks(submission), acquisition: { queued: true, dispatched }, acquisitionPlan: acquisitionPlan(input.websiteUrl) }, 202);
|
|
21815
|
+
return c.json({ ok: true, submission, publicUrls: localSourcebookPublicationLinks(submission), acquisition: { queued: true, dispatched }, acquisitionPlan: acquisitionPlan(input.websiteUrl), billing: publicListingWriteBilling(billing) }, 202);
|
|
21636
21816
|
} catch (error) {
|
|
21817
|
+
if (billing && !persisted) await refundListingWrite(billing, "Local Sourcebook creation failed before persistence.").catch(() => null);
|
|
21637
21818
|
return c.json({ error: "submission_conflict", message: error instanceof Error ? error.message : String(error) }, 409);
|
|
21638
21819
|
}
|
|
21639
21820
|
});
|
|
@@ -21650,12 +21831,35 @@ localSourcebookApp.on(["PATCH", "POST"], "/submissions/:id/draft", createApiKeyA
|
|
|
21650
21831
|
localSourcebookApp.post("/submissions/:id/refresh", createApiKeyAuth(), async (c) => {
|
|
21651
21832
|
const user = c.get("user");
|
|
21652
21833
|
if (!user.subscription_tier) return c.json({ error: "subscription_required" }, 402);
|
|
21834
|
+
const userId = Number(user.id);
|
|
21835
|
+
const existing = await getLocalSourcebookSubmission(c.req.param("id"), userId);
|
|
21836
|
+
if (!existing) return c.json({ error: "not_found" }, 404);
|
|
21837
|
+
if (existing.status === "rejected" || existing.status === "unpublished") {
|
|
21838
|
+
return c.json({ error: "moderated_listing", message: `A ${existing.status} Local Sourcebook listing cannot be refreshed by its owner.` }, 409);
|
|
21839
|
+
}
|
|
21840
|
+
if (existing.status === "queued" || existing.status === "enriching") {
|
|
21841
|
+
return c.json({ error: "acquisition_in_progress", message: "This listing already has an acquisition in progress." }, 409);
|
|
21842
|
+
}
|
|
21843
|
+
const idempotencyKey3 = `refresh:${existing.id}:revision:${existing.draftRevision}`;
|
|
21844
|
+
let billing = null;
|
|
21653
21845
|
try {
|
|
21654
|
-
const
|
|
21655
|
-
if (
|
|
21846
|
+
const charged = await chargeListingWrite({ surface: "local_sourcebook", kind: "edit", userId, idempotencyKey: idempotencyKey3 });
|
|
21847
|
+
if ("insufficient" in charged) return c.json(charged.insufficient, 402);
|
|
21848
|
+
billing = charged;
|
|
21849
|
+
const submission = await queueLocalSourcebookRefresh(c.req.param("id"), userId, {
|
|
21850
|
+
kind: billing.kind,
|
|
21851
|
+
credits: billing.credits,
|
|
21852
|
+
amountMc: billing.amountMc,
|
|
21853
|
+
debitKey: billing.debitKey
|
|
21854
|
+
});
|
|
21855
|
+
if (!submission) {
|
|
21856
|
+
await refundListingWrite(billing, "Local Sourcebook listing was not found before refresh persistence.").catch(() => null);
|
|
21857
|
+
return c.json({ error: "not_found" }, 404);
|
|
21858
|
+
}
|
|
21656
21859
|
const dispatched = await dispatchLocalSourcebookAcquisition(submission.id).catch(() => false);
|
|
21657
|
-
return c.json({ ok: true, submission, publicUrls: localSourcebookPublicationLinks(submission), acquisition: { queued: true, dispatched }, acquisitionPlan: acquisitionPlan(submission.websiteUrl) }, 202);
|
|
21860
|
+
return c.json({ ok: true, submission, publicUrls: localSourcebookPublicationLinks(submission), acquisition: { queued: true, dispatched }, acquisitionPlan: acquisitionPlan(submission.websiteUrl), billing: publicListingWriteBilling(billing) }, 202);
|
|
21658
21861
|
} catch (error) {
|
|
21862
|
+
if (billing) await refundListingWrite(billing, "Local Sourcebook refresh failed before persistence.").catch(() => null);
|
|
21659
21863
|
return c.json({ error: "moderated_listing", message: error instanceof Error ? error.message : String(error) }, 409);
|
|
21660
21864
|
}
|
|
21661
21865
|
});
|
|
@@ -31027,6 +31231,12 @@ async function prepareCommonsEntity(input, user) {
|
|
|
31027
31231
|
writePath: "commons_prepare_entity -> commons_validate_entity -> commons_submit_entity"
|
|
31028
31232
|
},
|
|
31029
31233
|
contract,
|
|
31234
|
+
billing: {
|
|
31235
|
+
validationCredits: LISTING_WRITE_RATE_POLICY.validationCredits,
|
|
31236
|
+
createCredits: LISTING_WRITE_RATE_POLICY.create.credits,
|
|
31237
|
+
editCredits: LISTING_WRITE_RATE_POLICY.edit.credits,
|
|
31238
|
+
chargeEvent: LISTING_WRITE_RATE_POLICY.chargeEvent
|
|
31239
|
+
},
|
|
31030
31240
|
duplicateCandidates,
|
|
31031
31241
|
tagResolutions,
|
|
31032
31242
|
headingPlan: {
|
|
@@ -31041,7 +31251,8 @@ async function prepareCommonsEntity(input, user) {
|
|
|
31041
31251
|
"Use the returned profile as a contract, not a rigid empty template: do not include History, Reception, Pricing, Timeline, or similar sections unless source evidence supports them.",
|
|
31042
31252
|
"Write neutral encyclopedia prose. Do not paste raw scrape Markdown, marketing layout fragments, or empty/filler headings.",
|
|
31043
31253
|
"Attach a featured image for auto-publish eligibility, preserve original URL/source byline/relCanonical when republishing, and put media assets in the media array.",
|
|
31044
|
-
"Call commons_validate_entity with the composed payload before commons_submit_entity."
|
|
31254
|
+
"Call commons_validate_entity with the composed payload before commons_submit_entity.",
|
|
31255
|
+
"Validation is free. Submitting a new entity costs 10 Credits; submitting an edit to an existing entity costs 2 Credits."
|
|
31045
31256
|
]
|
|
31046
31257
|
};
|
|
31047
31258
|
}
|
|
@@ -31130,6 +31341,14 @@ async function getCommonsEntity(idOrSlug) {
|
|
|
31130
31341
|
});
|
|
31131
31342
|
return result.rows[0] ? rowToEntity(result.rows[0]) : null;
|
|
31132
31343
|
}
|
|
31344
|
+
async function classifyCommonsWrite(input, user) {
|
|
31345
|
+
await ensureCommonsSchema();
|
|
31346
|
+
const normalized = normalizeSubmitInput(input);
|
|
31347
|
+
const existingProposal = await findProposalByIdempotency(user.id, normalized.idempotencyKey);
|
|
31348
|
+
if (existingProposal) return existingProposal.proposalType === "edit_entity" ? "edit" : "create";
|
|
31349
|
+
const existing = await findEntityBySlugOrIdAnyStatus(normalized.slug, normalized.entityId);
|
|
31350
|
+
return existing ? "edit" : "create";
|
|
31351
|
+
}
|
|
31133
31352
|
async function submitCommonsEntity(input, user) {
|
|
31134
31353
|
await ensureCommonsSchema();
|
|
31135
31354
|
const normalized = normalizeSubmitInput(input);
|
|
@@ -31394,7 +31613,8 @@ function commonsDatabaseReport() {
|
|
|
31394
31613
|
normalWorkflow: ["commons_prepare_entity", "commons_validate_entity", "commons_submit_entity"],
|
|
31395
31614
|
writesCreate: ["commons_proposals", "commons_ledger", "commons_entities when accepted"],
|
|
31396
31615
|
safeAutoPublish: "New non-conflicting entities with title, description, featured image, and body/source evidence.",
|
|
31397
|
-
conflictBehavior: "Existing entity edits require baseRevision. Otherwise the proposal is pending_review."
|
|
31616
|
+
conflictBehavior: "Existing entity edits require baseRevision. Otherwise the proposal is pending_review.",
|
|
31617
|
+
billing: { validationCredits: 0, createCredits: LISTING_WRITE_RATE_POLICY.create.credits, editCredits: LISTING_WRITE_RATE_POLICY.edit.credits }
|
|
31398
31618
|
},
|
|
31399
31619
|
searchCorpus: ["title", "disambiguationName", "description", "tags", "keywords", "relatedEntities", "bodyMarkdown", "sections", "media", "source metadata", "jsonLd"],
|
|
31400
31620
|
indexing: commonsIndexingPolicy(),
|
|
@@ -32685,10 +32905,22 @@ commonsApp.post("/entities/validate", auth2, async (c) => {
|
|
|
32685
32905
|
commonsApp.post("/entities/propose", auth2, async (c) => {
|
|
32686
32906
|
const parsed = SubmitEntitySchema.safeParse(await c.req.json().catch(() => ({})));
|
|
32687
32907
|
if (!parsed.success) return validationError(c, parsed.error);
|
|
32908
|
+
let billing = null;
|
|
32688
32909
|
try {
|
|
32910
|
+
const user = c.get("user");
|
|
32911
|
+
const kind = await classifyCommonsWrite(parsed.data, user);
|
|
32912
|
+
const charged = await chargeListingWrite({
|
|
32913
|
+
surface: "transparent_commons",
|
|
32914
|
+
kind,
|
|
32915
|
+
userId: Number(user.id),
|
|
32916
|
+
idempotencyKey: parsed.data.idempotencyKey
|
|
32917
|
+
});
|
|
32918
|
+
if ("insufficient" in charged) return c.json(charged.insufficient, 402);
|
|
32919
|
+
billing = charged;
|
|
32689
32920
|
const result = await submitCommonsEntity(parsed.data, c.get("user"));
|
|
32690
|
-
return c.json({ ok: true, data: result }, result.applied ? 201 : 202);
|
|
32921
|
+
return c.json({ ok: true, data: result, billing: publicListingWriteBilling(billing) }, result.applied ? 201 : 202);
|
|
32691
32922
|
} catch (error) {
|
|
32923
|
+
if (billing) await refundListingWrite(billing, "Transparent Commons write failed before persistence.").catch(() => null);
|
|
32692
32924
|
if (error instanceof CommonsRepositoryError) {
|
|
32693
32925
|
return c.json({ ok: false, error: error.code, message: error.message }, repositoryStatus(error));
|
|
32694
32926
|
}
|
|
@@ -40772,7 +41004,7 @@ app.get("/cron/tick", async (c) => {
|
|
|
40772
41004
|
if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
|
|
40773
41005
|
return c.json({ error: "Unauthorized" }, 401);
|
|
40774
41006
|
}
|
|
40775
|
-
const { drainQueue } = await import("./worker-
|
|
41007
|
+
const { drainQueue } = await import("./worker-4LZGRHDA.js");
|
|
40776
41008
|
const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
|
|
40777
41009
|
const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
|
|
40778
41010
|
const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup, connectedAccountBilling, directoryWorkflowDispatch, directoryWorkflowReconciliation, directoryArtifactCleanup, siteExtractArtifactCleanup, siteExtractRedispatch, siteExtractReconciliation, paaHarvestRedispatch, paaHarvestReconciliation, localSourcebookRedispatch] = await Promise.all([
|
|
@@ -40827,8 +41059,8 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
40827
41059
|
finishExtractJob: finishExtractJob2,
|
|
40828
41060
|
terminalExtractJobStatus: terminalExtractJobStatus2,
|
|
40829
41061
|
extractJobLimitInfo: extractJobLimitInfo2
|
|
40830
|
-
} = await import("./site-extract-repository-
|
|
40831
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
41062
|
+
} = await import("./site-extract-repository-G723PZIL.js");
|
|
41063
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-DYMQRQWG.js");
|
|
40832
41064
|
const existing = await getExtractJob2(jobId2);
|
|
40833
41065
|
if (!existing) return c.json({ error: "job not found" }, 404);
|
|
40834
41066
|
if (existing.status !== "failed") {
|
|
@@ -40855,7 +41087,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
40855
41087
|
if (!updated) return c.json({ error: "job state changed during re-finalization" }, 409);
|
|
40856
41088
|
let settlement = "already_settled_or_refunded";
|
|
40857
41089
|
if (job.billedMc == null && job.userId != null) {
|
|
40858
|
-
const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-
|
|
41090
|
+
const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-G723PZIL.js");
|
|
40859
41091
|
const heldMc = Number(job.options.heldMc ?? 0);
|
|
40860
41092
|
const successful = await countSuccessfulPages2(jobId2);
|
|
40861
41093
|
const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
|
|
@@ -41039,4 +41271,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
41039
41271
|
export {
|
|
41040
41272
|
app
|
|
41041
41273
|
};
|
|
41042
|
-
//# sourceMappingURL=server-
|
|
41274
|
+
//# sourceMappingURL=server-7B75SDNG.js.map
|