mcp-scraper 0.49.0 → 0.51.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -3
- package/dist/{analytics-repository-WWXVU3VW.js → analytics-repository-IKE53S3E.js} +10 -2
- package/dist/bin/api-server.cjs +840 -177
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +4 -4
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +141 -35
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +4 -4
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +3 -3
- package/dist/{chunk-MFNGUM4L.js → chunk-2XTYLJQK.js} +15 -1
- package/dist/chunk-2XTYLJQK.js.map +1 -0
- package/dist/{chunk-YW2LXLDU.js → chunk-5BEPOFKG.js} +160 -15
- package/dist/chunk-5BEPOFKG.js.map +1 -0
- package/dist/chunk-73MUQSWC.js +7 -0
- package/dist/chunk-73MUQSWC.js.map +1 -0
- package/dist/{chunk-MFE6RMIA.js → chunk-GOZIG6HD.js} +2 -2
- package/dist/{chunk-R66PJOZW.js → chunk-NNZXZTJ2.js} +2 -2
- package/dist/{chunk-64WBUDPC.js → chunk-O63VVCCE.js} +143 -37
- package/dist/chunk-O63VVCCE.js.map +1 -0
- package/dist/{chunk-6DPIE262.js → chunk-QHVFKFME.js} +2 -2
- package/dist/{chunk-PODASLGT.js → chunk-QXAY44SA.js} +2 -2
- package/dist/{chunk-ZEMHL36J.js → chunk-YMSZO62N.js} +2 -2
- package/dist/{db-A4YUPV4Z.js → db-UPSS5BGM.js} +2 -2
- package/dist/editorial-reading-room/assets/app.js +62 -0
- package/dist/editorial-reading-room/assets/index.html +8 -0
- package/dist/editorial-reading-room/assets/styles.css +39 -0
- package/dist/{extract-bundle-2HONVC5T.js → extract-bundle-RCTNANCH.js} +3 -3
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +3 -3
- package/dist/{location-data-repository-YWFJGNIP.js → location-data-repository-UUSHH5M2.js} +3 -3
- package/dist/{server-JE7SLZLJ.js → server-IGIIEHXY.js} +518 -135
- package/dist/server-IGIIEHXY.js.map +1 -0
- package/dist/{site-extract-repository-STNOI5OH.js → site-extract-repository-3VQAALNR.js} +3 -3
- package/dist/{worker-4MHZS3OB.js → worker-KQN673JF.js} +5 -5
- package/package.json +1 -1
- package/dist/chunk-64WBUDPC.js.map +0 -1
- package/dist/chunk-MFNGUM4L.js.map +0 -1
- package/dist/chunk-XLEDHV3J.js +0 -7
- package/dist/chunk-XLEDHV3J.js.map +0 -1
- package/dist/chunk-YW2LXLDU.js.map +0 -1
- package/dist/server-JE7SLZLJ.js.map +0 -1
- /package/dist/{analytics-repository-WWXVU3VW.js.map → analytics-repository-IKE53S3E.js.map} +0 -0
- /package/dist/{chunk-MFE6RMIA.js.map → chunk-GOZIG6HD.js.map} +0 -0
- /package/dist/{chunk-R66PJOZW.js.map → chunk-NNZXZTJ2.js.map} +0 -0
- /package/dist/{chunk-6DPIE262.js.map → chunk-QHVFKFME.js.map} +0 -0
- /package/dist/{chunk-PODASLGT.js.map → chunk-QXAY44SA.js.map} +0 -0
- /package/dist/{chunk-ZEMHL36J.js.map → chunk-YMSZO62N.js.map} +0 -0
- /package/dist/{db-A4YUPV4Z.js.map → db-UPSS5BGM.js.map} +0 -0
- /package/dist/{extract-bundle-2HONVC5T.js.map → extract-bundle-RCTNANCH.js.map} +0 -0
- /package/dist/{location-data-repository-YWFJGNIP.js.map → location-data-repository-UUSHH5M2.js.map} +0 -0
- /package/dist/{site-extract-repository-STNOI5OH.js.map → site-extract-repository-3VQAALNR.js.map} +0 -0
- /package/dist/{worker-4MHZS3OB.js.map → worker-KQN673JF.js.map} +0 -0
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
CREDIT_LOT_TTL,
|
|
9
9
|
getDb,
|
|
10
10
|
settleDebitMcIdempotent
|
|
11
|
-
} from "./chunk-
|
|
11
|
+
} from "./chunk-2XTYLJQK.js";
|
|
12
12
|
|
|
13
13
|
// src/api/site-extract-repository.ts
|
|
14
14
|
function extractJobLimitInfo(job) {
|
|
@@ -496,4 +496,4 @@ export {
|
|
|
496
496
|
failUnfundedExtractJob,
|
|
497
497
|
settleExtractJob
|
|
498
498
|
};
|
|
499
|
-
//# sourceMappingURL=chunk-
|
|
499
|
+
//# sourceMappingURL=chunk-QHVFKFME.js.map
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
getDb
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-2XTYLJQK.js";
|
|
4
4
|
|
|
5
5
|
// src/lib/browser-service-env.ts
|
|
6
6
|
function browserServiceApiKey() {
|
|
@@ -275,4 +275,4 @@ export {
|
|
|
275
275
|
recordKernelSession,
|
|
276
276
|
recordVendorUsage
|
|
277
277
|
};
|
|
278
|
-
//# sourceMappingURL=chunk-
|
|
278
|
+
//# sourceMappingURL=chunk-QXAY44SA.js.map
|
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
import {
|
|
9
9
|
finishHarvestAttempt,
|
|
10
10
|
startHarvestAttempt
|
|
11
|
-
} from "./chunk-
|
|
11
|
+
} from "./chunk-2XTYLJQK.js";
|
|
12
12
|
|
|
13
13
|
// src/api/webhook.ts
|
|
14
14
|
async function deliverWebhook(url, payload, retries = 3) {
|
|
@@ -191,4 +191,4 @@ export {
|
|
|
191
191
|
harvestProblemResponse,
|
|
192
192
|
createHarvestAttemptRecorder
|
|
193
193
|
};
|
|
194
|
-
//# sourceMappingURL=chunk-
|
|
194
|
+
//# sourceMappingURL=chunk-YMSZO62N.js.map
|
|
@@ -124,7 +124,7 @@ import {
|
|
|
124
124
|
updateKpoJobState,
|
|
125
125
|
upsertPageSnapshot,
|
|
126
126
|
verifyPassword
|
|
127
|
-
} from "./chunk-
|
|
127
|
+
} from "./chunk-2XTYLJQK.js";
|
|
128
128
|
export {
|
|
129
129
|
CORE_SCHEMA_VERSION,
|
|
130
130
|
CREDIT_LOTS_EPOCH,
|
|
@@ -252,4 +252,4 @@ export {
|
|
|
252
252
|
upsertPageSnapshot,
|
|
253
253
|
verifyPassword
|
|
254
254
|
};
|
|
255
|
-
//# sourceMappingURL=db-
|
|
255
|
+
//# sourceMappingURL=db-UPSS5BGM.js.map
|
|
@@ -32,6 +32,39 @@ function escapeHtml(value = "") {
|
|
|
32
32
|
.replace(/"/g, """);
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
+
function socialImage(article = null) {
|
|
36
|
+
return article?.ogImage || article?.image || state.data.site.ogImage || null;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function setMeta(selector, attribute, value) {
|
|
40
|
+
const element = document.querySelector(selector);
|
|
41
|
+
if (!element) return;
|
|
42
|
+
if (value) element.setAttribute(attribute, value);
|
|
43
|
+
else element.removeAttribute(attribute);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function updateSocialMeta(article = null) {
|
|
47
|
+
const title = article ? `${article.title} — ${state.data.site.title}` : state.data.site.title;
|
|
48
|
+
const description = article?.summary || state.data.deck || state.data.site.heroTitle;
|
|
49
|
+
const image = socialImage(article);
|
|
50
|
+
setMeta('meta[property="og:type"]', "content", article ? "article" : "website");
|
|
51
|
+
setMeta('meta[property="og:title"]', "content", title);
|
|
52
|
+
setMeta('meta[property="og:description"]', "content", description);
|
|
53
|
+
setMeta('meta[name="twitter:title"]', "content", title);
|
|
54
|
+
setMeta('meta[name="twitter:description"]', "content", description);
|
|
55
|
+
setMeta('meta[property="og:image"]', "content", image?.url || "");
|
|
56
|
+
setMeta('meta[property="og:image:alt"]', "content", image?.alt || "");
|
|
57
|
+
setMeta('meta[property="og:image:width"]', "content", image?.width ? String(image.width) : "");
|
|
58
|
+
setMeta('meta[property="og:image:height"]', "content", image?.height ? String(image.height) : "");
|
|
59
|
+
setMeta('meta[name="twitter:image"]', "content", image?.url || "");
|
|
60
|
+
setMeta('meta[name="twitter:image:alt"]', "content", image?.alt || "");
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function imageMarkup(image, className) {
|
|
64
|
+
if (!image) return "";
|
|
65
|
+
return `<img class="${className}" src="${escapeHtml(image.url)}" alt="${escapeHtml(image.alt)}" loading="lazy">`;
|
|
66
|
+
}
|
|
67
|
+
|
|
35
68
|
function setTypeMode() {
|
|
36
69
|
const mode = typeModes[state.typeIndex % typeModes.length];
|
|
37
70
|
if (mode === "default") document.documentElement.removeAttribute("data-type");
|
|
@@ -132,6 +165,7 @@ function renderHome() {
|
|
|
132
165
|
elements.article.hidden = true;
|
|
133
166
|
elements.progress.style.width = "0";
|
|
134
167
|
document.title = state.data.site.title;
|
|
168
|
+
updateSocialMeta();
|
|
135
169
|
document.querySelector("#masthead-product").textContent = state.data.site.product;
|
|
136
170
|
document.querySelector("#masthead-title").textContent = state.data.site.edition;
|
|
137
171
|
document.querySelector("#edition-label").textContent = state.data.site.editionLabel;
|
|
@@ -151,6 +185,7 @@ function renderHome() {
|
|
|
151
185
|
.map(
|
|
152
186
|
(article) => `<article class="feature-card">
|
|
153
187
|
<button data-article="${article.slug}">
|
|
188
|
+
${imageMarkup(article.image, "feature-card-image")}
|
|
154
189
|
<p class="card-kicker">${escapeHtml(article.kicker)}</p>
|
|
155
190
|
<h2>${escapeHtml(article.title)}</h2>
|
|
156
191
|
<p>${escapeHtml(article.summary)}</p>
|
|
@@ -190,6 +225,7 @@ function renderArticle(article) {
|
|
|
190
225
|
elements.home.hidden = true;
|
|
191
226
|
elements.article.hidden = false;
|
|
192
227
|
document.title = `${article.title} — ${state.data.site.title}`;
|
|
228
|
+
updateSocialMeta(article);
|
|
193
229
|
document.querySelector("#article-category").textContent = article.category;
|
|
194
230
|
document.querySelector("#article-kicker").textContent = article.kicker;
|
|
195
231
|
document.querySelector("#article-title").textContent = article.title;
|
|
@@ -199,6 +235,32 @@ function renderArticle(article) {
|
|
|
199
235
|
document.querySelector("#article-revision").textContent = article.revision
|
|
200
236
|
? `Revision ${article.revision}`
|
|
201
237
|
: article.sourceType || "Source material";
|
|
238
|
+
const imageFigure = document.querySelector("#article-image");
|
|
239
|
+
const imageElement = document.querySelector("#article-image-src");
|
|
240
|
+
const imageCaption = document.querySelector("#article-image-caption");
|
|
241
|
+
const imageCaptionText = document.querySelector("#article-image-caption-text");
|
|
242
|
+
const imageSource = document.querySelector("#article-image-source");
|
|
243
|
+
if (article.image) {
|
|
244
|
+
imageFigure.hidden = false;
|
|
245
|
+
imageElement.src = article.image.url;
|
|
246
|
+
imageElement.alt = article.image.alt;
|
|
247
|
+
imageCaptionText.textContent = [article.image.caption, article.image.credit].filter(Boolean).join(" — ");
|
|
248
|
+
if (article.image.sourceUrl) {
|
|
249
|
+
imageSource.href = article.image.sourceUrl;
|
|
250
|
+
imageSource.hidden = false;
|
|
251
|
+
} else {
|
|
252
|
+
imageSource.removeAttribute("href");
|
|
253
|
+
imageSource.hidden = true;
|
|
254
|
+
}
|
|
255
|
+
imageCaption.hidden = !imageCaptionText.textContent && !article.image.sourceUrl;
|
|
256
|
+
} else {
|
|
257
|
+
imageFigure.hidden = true;
|
|
258
|
+
imageElement.removeAttribute("src");
|
|
259
|
+
imageElement.alt = "";
|
|
260
|
+
imageCaptionText.textContent = "";
|
|
261
|
+
imageSource.removeAttribute("href");
|
|
262
|
+
imageSource.hidden = true;
|
|
263
|
+
}
|
|
202
264
|
document.querySelector("#article-body").innerHTML = article.html;
|
|
203
265
|
document.querySelector("#article-source").textContent = article.sourceLabel;
|
|
204
266
|
renderDesktopJumps(article);
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
content="A mobile-first editorial reading room."
|
|
9
9
|
/>
|
|
10
10
|
<meta name="theme-color" content="#f5f1e8" />
|
|
11
|
+
<!--__READING_ROOM_SOCIAL_META__-->
|
|
11
12
|
<title>Editorial Reading Room</title>
|
|
12
13
|
<style>/*__READING_ROOM_STYLES__*/</style>
|
|
13
14
|
</head>
|
|
@@ -99,6 +100,13 @@
|
|
|
99
100
|
<span id="article-word-count"></span>
|
|
100
101
|
<span id="article-revision"></span>
|
|
101
102
|
</div>
|
|
103
|
+
<figure class="article-image" id="article-image" hidden>
|
|
104
|
+
<img id="article-image-src" alt="" />
|
|
105
|
+
<figcaption id="article-image-caption">
|
|
106
|
+
<span id="article-image-caption-text"></span>
|
|
107
|
+
<a id="article-image-source" target="_blank" rel="noopener noreferrer">Image source</a>
|
|
108
|
+
</figcaption>
|
|
109
|
+
</figure>
|
|
102
110
|
</header>
|
|
103
111
|
<div class="article-body" id="article-body"></div>
|
|
104
112
|
<footer class="article-source">
|
|
@@ -277,6 +277,14 @@ a {
|
|
|
277
277
|
text-align: left;
|
|
278
278
|
}
|
|
279
279
|
|
|
280
|
+
.feature-card-image {
|
|
281
|
+
width: 100%;
|
|
282
|
+
aspect-ratio: 16 / 9;
|
|
283
|
+
margin-bottom: 20px;
|
|
284
|
+
border: 1px solid var(--rule);
|
|
285
|
+
object-fit: cover;
|
|
286
|
+
}
|
|
287
|
+
|
|
280
288
|
.card-kicker {
|
|
281
289
|
margin: 0 0 18px;
|
|
282
290
|
color: var(--accent);
|
|
@@ -594,6 +602,37 @@ body.drawer-open .drawer-backdrop {
|
|
|
594
602
|
text-transform: uppercase;
|
|
595
603
|
}
|
|
596
604
|
|
|
605
|
+
.article-image {
|
|
606
|
+
margin: 34px 0 0;
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
.article-image img {
|
|
610
|
+
display: block;
|
|
611
|
+
width: 100%;
|
|
612
|
+
max-height: 680px;
|
|
613
|
+
border: 1px solid var(--rule);
|
|
614
|
+
object-fit: cover;
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
.article-image figcaption {
|
|
618
|
+
margin-top: 9px;
|
|
619
|
+
color: var(--muted);
|
|
620
|
+
font-size: 12px;
|
|
621
|
+
line-height: 1.45;
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
.article-image figcaption a {
|
|
625
|
+
margin-left: 0.45em;
|
|
626
|
+
color: inherit;
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
.article-body img {
|
|
630
|
+
display: block;
|
|
631
|
+
max-width: 100%;
|
|
632
|
+
height: auto;
|
|
633
|
+
margin: 1.8em auto;
|
|
634
|
+
}
|
|
635
|
+
|
|
597
636
|
.article-body {
|
|
598
637
|
padding-top: 28px;
|
|
599
638
|
font-family: var(--serif);
|
|
@@ -10,12 +10,12 @@ import {
|
|
|
10
10
|
} from "./chunk-2K74LVV7.js";
|
|
11
11
|
import {
|
|
12
12
|
extractJobLimitInfo
|
|
13
|
-
} from "./chunk-
|
|
13
|
+
} from "./chunk-QHVFKFME.js";
|
|
14
14
|
import "./chunk-SOEYDWJU.js";
|
|
15
15
|
import "./chunk-5UN33CGU.js";
|
|
16
16
|
import {
|
|
17
17
|
getDb
|
|
18
|
-
} from "./chunk-
|
|
18
|
+
} from "./chunk-2XTYLJQK.js";
|
|
19
19
|
|
|
20
20
|
// src/api/extract-bundle.ts
|
|
21
21
|
import { createWriteStream, mkdirSync, readdirSync, rmSync, statSync, writeFileSync } from "fs";
|
|
@@ -565,4 +565,4 @@ export {
|
|
|
565
565
|
SITE_EXTRACT_PAGE_CHUNK,
|
|
566
566
|
assembleExtractArtifacts
|
|
567
567
|
};
|
|
568
|
-
//# sourceMappingURL=extract-bundle-
|
|
568
|
+
//# sourceMappingURL=extract-bundle-RCTNANCH.js.map
|