mcp-scraper 0.49.0 → 0.51.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/README.md +11 -3
  2. package/dist/{analytics-repository-WWXVU3VW.js → analytics-repository-IKE53S3E.js} +10 -2
  3. package/dist/bin/api-server.cjs +840 -177
  4. package/dist/bin/api-server.cjs.map +1 -1
  5. package/dist/bin/api-server.js +4 -4
  6. package/dist/bin/mcp-scraper-cli.cjs +1 -1
  7. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  8. package/dist/bin/mcp-scraper-cli.js +1 -1
  9. package/dist/bin/mcp-scraper-install.cjs +1 -1
  10. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  11. package/dist/bin/mcp-scraper-install.js +1 -1
  12. package/dist/bin/mcp-stdio-server.cjs +141 -35
  13. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  14. package/dist/bin/mcp-stdio-server.js +4 -4
  15. package/dist/bin/paa-harvest.cjs.map +1 -1
  16. package/dist/bin/paa-harvest.js +3 -3
  17. package/dist/{chunk-MFNGUM4L.js → chunk-2XTYLJQK.js} +15 -1
  18. package/dist/chunk-2XTYLJQK.js.map +1 -0
  19. package/dist/{chunk-YW2LXLDU.js → chunk-5BEPOFKG.js} +160 -15
  20. package/dist/chunk-5BEPOFKG.js.map +1 -0
  21. package/dist/chunk-73MUQSWC.js +7 -0
  22. package/dist/chunk-73MUQSWC.js.map +1 -0
  23. package/dist/{chunk-MFE6RMIA.js → chunk-GOZIG6HD.js} +2 -2
  24. package/dist/{chunk-R66PJOZW.js → chunk-NNZXZTJ2.js} +2 -2
  25. package/dist/{chunk-64WBUDPC.js → chunk-O63VVCCE.js} +143 -37
  26. package/dist/chunk-O63VVCCE.js.map +1 -0
  27. package/dist/{chunk-6DPIE262.js → chunk-QHVFKFME.js} +2 -2
  28. package/dist/{chunk-PODASLGT.js → chunk-QXAY44SA.js} +2 -2
  29. package/dist/{chunk-ZEMHL36J.js → chunk-YMSZO62N.js} +2 -2
  30. package/dist/{db-A4YUPV4Z.js → db-UPSS5BGM.js} +2 -2
  31. package/dist/editorial-reading-room/assets/app.js +62 -0
  32. package/dist/editorial-reading-room/assets/index.html +8 -0
  33. package/dist/editorial-reading-room/assets/styles.css +39 -0
  34. package/dist/{extract-bundle-2HONVC5T.js → extract-bundle-RCTNANCH.js} +3 -3
  35. package/dist/index.cjs.map +1 -1
  36. package/dist/index.js +3 -3
  37. package/dist/{location-data-repository-YWFJGNIP.js → location-data-repository-UUSHH5M2.js} +3 -3
  38. package/dist/{server-JE7SLZLJ.js → server-IGIIEHXY.js} +518 -135
  39. package/dist/server-IGIIEHXY.js.map +1 -0
  40. package/dist/{site-extract-repository-STNOI5OH.js → site-extract-repository-3VQAALNR.js} +3 -3
  41. package/dist/{worker-4MHZS3OB.js → worker-KQN673JF.js} +5 -5
  42. package/package.json +1 -1
  43. package/dist/chunk-64WBUDPC.js.map +0 -1
  44. package/dist/chunk-MFNGUM4L.js.map +0 -1
  45. package/dist/chunk-XLEDHV3J.js +0 -7
  46. package/dist/chunk-XLEDHV3J.js.map +0 -1
  47. package/dist/chunk-YW2LXLDU.js.map +0 -1
  48. package/dist/server-JE7SLZLJ.js.map +0 -1
  49. /package/dist/{analytics-repository-WWXVU3VW.js.map → analytics-repository-IKE53S3E.js.map} +0 -0
  50. /package/dist/{chunk-MFE6RMIA.js.map → chunk-GOZIG6HD.js.map} +0 -0
  51. /package/dist/{chunk-R66PJOZW.js.map → chunk-NNZXZTJ2.js.map} +0 -0
  52. /package/dist/{chunk-6DPIE262.js.map → chunk-QHVFKFME.js.map} +0 -0
  53. /package/dist/{chunk-PODASLGT.js.map → chunk-QXAY44SA.js.map} +0 -0
  54. /package/dist/{chunk-ZEMHL36J.js.map → chunk-YMSZO62N.js.map} +0 -0
  55. /package/dist/{db-A4YUPV4Z.js.map → db-UPSS5BGM.js.map} +0 -0
  56. /package/dist/{extract-bundle-2HONVC5T.js.map → extract-bundle-RCTNANCH.js.map} +0 -0
  57. /package/dist/{location-data-repository-YWFJGNIP.js.map → location-data-repository-UUSHH5M2.js.map} +0 -0
  58. /package/dist/{site-extract-repository-STNOI5OH.js.map → site-extract-repository-3VQAALNR.js.map} +0 -0
  59. /package/dist/{worker-4MHZS3OB.js.map → worker-KQN673JF.js.map} +0 -0
@@ -8,7 +8,7 @@ import {
8
8
  CREDIT_LOT_TTL,
9
9
  getDb,
10
10
  settleDebitMcIdempotent
11
- } from "./chunk-MFNGUM4L.js";
11
+ } from "./chunk-2XTYLJQK.js";
12
12
 
13
13
  // src/api/site-extract-repository.ts
14
14
  function extractJobLimitInfo(job) {
@@ -496,4 +496,4 @@ export {
496
496
  failUnfundedExtractJob,
497
497
  settleExtractJob
498
498
  };
499
- //# sourceMappingURL=chunk-6DPIE262.js.map
499
+ //# sourceMappingURL=chunk-QHVFKFME.js.map
@@ -1,6 +1,6 @@
1
1
  import {
2
2
  getDb
3
- } from "./chunk-MFNGUM4L.js";
3
+ } from "./chunk-2XTYLJQK.js";
4
4
 
5
5
  // src/lib/browser-service-env.ts
6
6
  function browserServiceApiKey() {
@@ -275,4 +275,4 @@ export {
275
275
  recordKernelSession,
276
276
  recordVendorUsage
277
277
  };
278
- //# sourceMappingURL=chunk-PODASLGT.js.map
278
+ //# sourceMappingURL=chunk-QXAY44SA.js.map
@@ -8,7 +8,7 @@ import {
8
8
  import {
9
9
  finishHarvestAttempt,
10
10
  startHarvestAttempt
11
- } from "./chunk-MFNGUM4L.js";
11
+ } from "./chunk-2XTYLJQK.js";
12
12
 
13
13
  // src/api/webhook.ts
14
14
  async function deliverWebhook(url, payload, retries = 3) {
@@ -191,4 +191,4 @@ export {
191
191
  harvestProblemResponse,
192
192
  createHarvestAttemptRecorder
193
193
  };
194
- //# sourceMappingURL=chunk-ZEMHL36J.js.map
194
+ //# sourceMappingURL=chunk-YMSZO62N.js.map
@@ -124,7 +124,7 @@ import {
124
124
  updateKpoJobState,
125
125
  upsertPageSnapshot,
126
126
  verifyPassword
127
- } from "./chunk-MFNGUM4L.js";
127
+ } from "./chunk-2XTYLJQK.js";
128
128
  export {
129
129
  CORE_SCHEMA_VERSION,
130
130
  CREDIT_LOTS_EPOCH,
@@ -252,4 +252,4 @@ export {
252
252
  upsertPageSnapshot,
253
253
  verifyPassword
254
254
  };
255
- //# sourceMappingURL=db-A4YUPV4Z.js.map
255
+ //# sourceMappingURL=db-UPSS5BGM.js.map
@@ -32,6 +32,39 @@ function escapeHtml(value = "") {
32
32
  .replace(/"/g, """);
33
33
  }
34
34
 
35
+ function socialImage(article = null) {
36
+ return article?.ogImage || article?.image || state.data.site.ogImage || null;
37
+ }
38
+
39
+ function setMeta(selector, attribute, value) {
40
+ const element = document.querySelector(selector);
41
+ if (!element) return;
42
+ if (value) element.setAttribute(attribute, value);
43
+ else element.removeAttribute(attribute);
44
+ }
45
+
46
+ function updateSocialMeta(article = null) {
47
+ const title = article ? `${article.title} — ${state.data.site.title}` : state.data.site.title;
48
+ const description = article?.summary || state.data.deck || state.data.site.heroTitle;
49
+ const image = socialImage(article);
50
+ setMeta('meta[property="og:type"]', "content", article ? "article" : "website");
51
+ setMeta('meta[property="og:title"]', "content", title);
52
+ setMeta('meta[property="og:description"]', "content", description);
53
+ setMeta('meta[name="twitter:title"]', "content", title);
54
+ setMeta('meta[name="twitter:description"]', "content", description);
55
+ setMeta('meta[property="og:image"]', "content", image?.url || "");
56
+ setMeta('meta[property="og:image:alt"]', "content", image?.alt || "");
57
+ setMeta('meta[property="og:image:width"]', "content", image?.width ? String(image.width) : "");
58
+ setMeta('meta[property="og:image:height"]', "content", image?.height ? String(image.height) : "");
59
+ setMeta('meta[name="twitter:image"]', "content", image?.url || "");
60
+ setMeta('meta[name="twitter:image:alt"]', "content", image?.alt || "");
61
+ }
62
+
63
+ function imageMarkup(image, className) {
64
+ if (!image) return "";
65
+ return `<img class="${className}" src="${escapeHtml(image.url)}" alt="${escapeHtml(image.alt)}" loading="lazy">`;
66
+ }
67
+
35
68
  function setTypeMode() {
36
69
  const mode = typeModes[state.typeIndex % typeModes.length];
37
70
  if (mode === "default") document.documentElement.removeAttribute("data-type");
@@ -132,6 +165,7 @@ function renderHome() {
132
165
  elements.article.hidden = true;
133
166
  elements.progress.style.width = "0";
134
167
  document.title = state.data.site.title;
168
+ updateSocialMeta();
135
169
  document.querySelector("#masthead-product").textContent = state.data.site.product;
136
170
  document.querySelector("#masthead-title").textContent = state.data.site.edition;
137
171
  document.querySelector("#edition-label").textContent = state.data.site.editionLabel;
@@ -151,6 +185,7 @@ function renderHome() {
151
185
  .map(
152
186
  (article) => `<article class="feature-card">
153
187
  <button data-article="${article.slug}">
188
+ ${imageMarkup(article.image, "feature-card-image")}
154
189
  <p class="card-kicker">${escapeHtml(article.kicker)}</p>
155
190
  <h2>${escapeHtml(article.title)}</h2>
156
191
  <p>${escapeHtml(article.summary)}</p>
@@ -190,6 +225,7 @@ function renderArticle(article) {
190
225
  elements.home.hidden = true;
191
226
  elements.article.hidden = false;
192
227
  document.title = `${article.title} — ${state.data.site.title}`;
228
+ updateSocialMeta(article);
193
229
  document.querySelector("#article-category").textContent = article.category;
194
230
  document.querySelector("#article-kicker").textContent = article.kicker;
195
231
  document.querySelector("#article-title").textContent = article.title;
@@ -199,6 +235,32 @@ function renderArticle(article) {
199
235
  document.querySelector("#article-revision").textContent = article.revision
200
236
  ? `Revision ${article.revision}`
201
237
  : article.sourceType || "Source material";
238
+ const imageFigure = document.querySelector("#article-image");
239
+ const imageElement = document.querySelector("#article-image-src");
240
+ const imageCaption = document.querySelector("#article-image-caption");
241
+ const imageCaptionText = document.querySelector("#article-image-caption-text");
242
+ const imageSource = document.querySelector("#article-image-source");
243
+ if (article.image) {
244
+ imageFigure.hidden = false;
245
+ imageElement.src = article.image.url;
246
+ imageElement.alt = article.image.alt;
247
+ imageCaptionText.textContent = [article.image.caption, article.image.credit].filter(Boolean).join(" — ");
248
+ if (article.image.sourceUrl) {
249
+ imageSource.href = article.image.sourceUrl;
250
+ imageSource.hidden = false;
251
+ } else {
252
+ imageSource.removeAttribute("href");
253
+ imageSource.hidden = true;
254
+ }
255
+ imageCaption.hidden = !imageCaptionText.textContent && !article.image.sourceUrl;
256
+ } else {
257
+ imageFigure.hidden = true;
258
+ imageElement.removeAttribute("src");
259
+ imageElement.alt = "";
260
+ imageCaptionText.textContent = "";
261
+ imageSource.removeAttribute("href");
262
+ imageSource.hidden = true;
263
+ }
202
264
  document.querySelector("#article-body").innerHTML = article.html;
203
265
  document.querySelector("#article-source").textContent = article.sourceLabel;
204
266
  renderDesktopJumps(article);
@@ -8,6 +8,7 @@
8
8
  content="A mobile-first editorial reading room."
9
9
  />
10
10
  <meta name="theme-color" content="#f5f1e8" />
11
+ <!--__READING_ROOM_SOCIAL_META__-->
11
12
  <title>Editorial Reading Room</title>
12
13
  <style>/*__READING_ROOM_STYLES__*/</style>
13
14
  </head>
@@ -99,6 +100,13 @@
99
100
  <span id="article-word-count"></span>
100
101
  <span id="article-revision"></span>
101
102
  </div>
103
+ <figure class="article-image" id="article-image" hidden>
104
+ <img id="article-image-src" alt="" />
105
+ <figcaption id="article-image-caption">
106
+ <span id="article-image-caption-text"></span>
107
+ <a id="article-image-source" target="_blank" rel="noopener noreferrer">Image source</a>
108
+ </figcaption>
109
+ </figure>
102
110
  </header>
103
111
  <div class="article-body" id="article-body"></div>
104
112
  <footer class="article-source">
@@ -277,6 +277,14 @@ a {
277
277
  text-align: left;
278
278
  }
279
279
 
280
+ .feature-card-image {
281
+ width: 100%;
282
+ aspect-ratio: 16 / 9;
283
+ margin-bottom: 20px;
284
+ border: 1px solid var(--rule);
285
+ object-fit: cover;
286
+ }
287
+
280
288
  .card-kicker {
281
289
  margin: 0 0 18px;
282
290
  color: var(--accent);
@@ -594,6 +602,37 @@ body.drawer-open .drawer-backdrop {
594
602
  text-transform: uppercase;
595
603
  }
596
604
 
605
+ .article-image {
606
+ margin: 34px 0 0;
607
+ }
608
+
609
+ .article-image img {
610
+ display: block;
611
+ width: 100%;
612
+ max-height: 680px;
613
+ border: 1px solid var(--rule);
614
+ object-fit: cover;
615
+ }
616
+
617
+ .article-image figcaption {
618
+ margin-top: 9px;
619
+ color: var(--muted);
620
+ font-size: 12px;
621
+ line-height: 1.45;
622
+ }
623
+
624
+ .article-image figcaption a {
625
+ margin-left: 0.45em;
626
+ color: inherit;
627
+ }
628
+
629
+ .article-body img {
630
+ display: block;
631
+ max-width: 100%;
632
+ height: auto;
633
+ margin: 1.8em auto;
634
+ }
635
+
597
636
  .article-body {
598
637
  padding-top: 28px;
599
638
  font-family: var(--serif);
@@ -10,12 +10,12 @@ import {
10
10
  } from "./chunk-2K74LVV7.js";
11
11
  import {
12
12
  extractJobLimitInfo
13
- } from "./chunk-6DPIE262.js";
13
+ } from "./chunk-QHVFKFME.js";
14
14
  import "./chunk-SOEYDWJU.js";
15
15
  import "./chunk-5UN33CGU.js";
16
16
  import {
17
17
  getDb
18
- } from "./chunk-MFNGUM4L.js";
18
+ } from "./chunk-2XTYLJQK.js";
19
19
 
20
20
  // src/api/extract-bundle.ts
21
21
  import { createWriteStream, mkdirSync, readdirSync, rmSync, statSync, writeFileSync } from "fs";
@@ -565,4 +565,4 @@ export {
565
565
  SITE_EXTRACT_PAGE_CHUNK,
566
566
  assembleExtractArtifacts
567
567
  };
568
- //# sourceMappingURL=extract-bundle-2HONVC5T.js.map
568
+ //# sourceMappingURL=extract-bundle-RCTNANCH.js.map