@blankgrimoire/runtime 1.3.1 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/engine.js CHANGED
@@ -1,6 +1,7 @@
1
1
  // SPDX-License-Identifier: MIT
2
2
  import { API_VERSION, } from '@blankgrimoire/types';
3
3
  import { attr, firstAttribute, queryAll, resolveURL, textOf } from './dom.js';
4
+ import { console } from './prelude.js';
4
5
  import { runStrategy } from './strategies.js';
5
6
  /** Raised when an adapter bundle targets a different contract version. */
6
7
  export class UnsupportedApiVersionError extends Error {
@@ -81,6 +82,27 @@ function splitList(value) {
81
82
  .filter((item) => item !== '');
82
83
  return items.length > 0 ? items : undefined;
83
84
  }
85
+ /**
86
+ * Read a list of labels. A list selector (`*Item`) collects every match in
87
+ * document order, dropping empty values and duplicates; otherwise the single
88
+ * node selector is split on the usual separators.
89
+ */
90
+ function readList(root, listSelector, singleSelector) {
91
+ if (listSelector) {
92
+ const values = [];
93
+ const seen = new Set();
94
+ for (const element of queryAll(root, listSelector)) {
95
+ const value = textOf(element);
96
+ if (value === '' || seen.has(value)) {
97
+ continue;
98
+ }
99
+ seen.add(value);
100
+ values.push(value);
101
+ }
102
+ return values.length > 0 ? values : undefined;
103
+ }
104
+ return splitList(readText(root, singleSelector));
105
+ }
84
106
  /** Normalise site text into the closed `MangaStatus` union. */
85
107
  export function normalizeStatus(value) {
86
108
  const text = value.trim().toLowerCase();
@@ -190,19 +212,19 @@ export async function extractDetails(runtime, key) {
190
212
  details[field] = value;
191
213
  }
192
214
  }
193
- const altTitles = splitList(readText(document, detailsSelectors.altTitles));
215
+ const altTitles = readList(document, detailsSelectors.altTitlesItem, detailsSelectors.altTitles);
194
216
  if (altTitles) {
195
217
  details.altTitles = altTitles;
196
218
  }
197
- const authors = splitList(readText(document, detailsSelectors.authors));
219
+ const authors = readList(document, detailsSelectors.authorsItem, detailsSelectors.authors);
198
220
  if (authors) {
199
221
  details.authors = authors;
200
222
  }
201
- const artists = splitList(readText(document, detailsSelectors.artists));
223
+ const artists = readList(document, detailsSelectors.artistsItem, detailsSelectors.artists);
202
224
  if (artists) {
203
225
  details.artists = artists;
204
226
  }
205
- const tags = splitList(readText(document, detailsSelectors.tags));
227
+ const tags = readList(document, detailsSelectors.tagsItem, detailsSelectors.tags);
206
228
  if (tags) {
207
229
  details.tags = tags;
208
230
  }
@@ -233,8 +255,16 @@ export async function extractChapters(runtime, key) {
233
255
  }
234
256
  const selectors = selectorsFor(runtime).chapters;
235
257
  const baseURL = runtime.context.baseURL;
258
+ const excludedSelector = selectors.excludeItem ?? selectors.premiumMarker;
236
259
  const chapters = [];
260
+ let dropped = 0;
237
261
  for (const element of queryAll(runtime.document, selectors.item)) {
262
+ // A locked row is dropped before anything is read from it, so paid content
263
+ // never reaches the host.
264
+ if (excludedSelector && element.querySelector(excludedSelector)) {
265
+ dropped += 1;
266
+ continue;
267
+ }
238
268
  const link = selectLink(element, selectors.link);
239
269
  const href = link ? attr(link, 'href') : null;
240
270
  if (!href) {
@@ -262,6 +292,9 @@ export async function extractChapters(runtime, key) {
262
292
  }
263
293
  chapters.push(chapter);
264
294
  }
295
+ if (dropped > 0) {
296
+ console.log(`chapters: dropped ${dropped} paid ${dropped === 1 ? 'row' : 'rows'}`);
297
+ }
265
298
  const numeric = chapters.every((chapter) => chapter.number !== undefined);
266
299
  if (numeric) {
267
300
  chapters.sort((a, b) => {
@@ -283,7 +316,13 @@ export async function extractPages(runtime, keys) {
283
316
  return hook(runtime.context, keys.mangaKey, keys.chapterKey);
284
317
  }
285
318
  const selectors = selectorsFor(runtime);
286
- const environment = { baseURL: runtime.context.baseURL };
319
+ let blocked = false;
320
+ const environment = {
321
+ baseURL: runtime.context.baseURL,
322
+ onBlocked: () => {
323
+ blocked = true;
324
+ },
325
+ };
287
326
  if (runtime.window) {
288
327
  environment.window = runtime.window;
289
328
  }
@@ -292,6 +331,12 @@ export async function extractPages(runtime, keys) {
292
331
  }
293
332
  for (const strategy of selectors.pages.strategies) {
294
333
  const pages = await runStrategy(strategy, runtime.document, environment);
334
+ // A paid-content guard stops extraction: falling through to another
335
+ // strategy could still return the locked pages.
336
+ if (blocked) {
337
+ console.log('pages: skipped, the payload is flagged as paid');
338
+ return [];
339
+ }
295
340
  if (pages.length > 0) {
296
341
  return pages;
297
342
  }
package/dist/runtime.js CHANGED
@@ -333,6 +333,23 @@ var BlankGrimoireRuntime = (() => {
333
333
  }
334
334
  return current;
335
335
  }
336
+ function isPaidPayload(parsed, skipWhen) {
337
+ if (!skipWhen) {
338
+ return false;
339
+ }
340
+ const value = readPath(parsed, skipWhen);
341
+ if (typeof value === "boolean") {
342
+ return value;
343
+ }
344
+ if (typeof value === "number") {
345
+ return Number.isFinite(value) && value !== 0;
346
+ }
347
+ if (typeof value === "string") {
348
+ const text = value.trim().toLowerCase();
349
+ return text !== "" && text !== "false" && text !== "0";
350
+ }
351
+ return false;
352
+ }
336
353
  function pageURLFromItem(item) {
337
354
  if (typeof item === "string") {
338
355
  return item.trim() === "" ? null : item;
@@ -367,7 +384,7 @@ var BlankGrimoireRuntime = (() => {
367
384
  }
368
385
  return pages;
369
386
  }
370
- async function extractScriptJSON(strategy, document, baseURL, parseHTML) {
387
+ async function extractScriptJSON(strategy, document, baseURL, parseHTML, onBlocked) {
371
388
  const regex = new RegExp(strategy.pattern);
372
389
  for (const script of queryAll(document, "script")) {
373
390
  const text = textOf(script);
@@ -394,6 +411,10 @@ var BlankGrimoireRuntime = (() => {
394
411
  if (parsed === void 0) {
395
412
  continue;
396
413
  }
414
+ if (isPaidPayload(parsed, strategy.skipWhen)) {
415
+ onBlocked?.();
416
+ return [];
417
+ }
397
418
  const payload = readPath(parsed, strategy.path);
398
419
  if (payload === void 0 || payload === null) {
399
420
  continue;
@@ -427,11 +448,15 @@ var BlankGrimoireRuntime = (() => {
427
448
  }
428
449
  return void 0;
429
450
  }
430
- function extractHydration(strategy, document, baseURL, windowObj) {
451
+ function extractHydration(strategy, document, baseURL, windowObj, onBlocked) {
431
452
  const payload = hydrationPayload(strategy, document, windowObj);
432
453
  if (payload === void 0) {
433
454
  return [];
434
455
  }
456
+ if (isPaidPayload(payload, strategy.skipWhen)) {
457
+ onBlocked?.();
458
+ return [];
459
+ }
435
460
  let value = payload;
436
461
  if (strategy.path) {
437
462
  value = readPath(payload, strategy.path);
@@ -483,14 +508,16 @@ var BlankGrimoireRuntime = (() => {
483
508
  strategy,
484
509
  document,
485
510
  environment.baseURL,
486
- environment.parseHTML
511
+ environment.parseHTML,
512
+ environment.onBlocked
487
513
  );
488
514
  case "hydration":
489
515
  return extractHydration(
490
516
  strategy,
491
517
  document,
492
518
  environment.baseURL,
493
- environment.window
519
+ environment.window,
520
+ environment.onBlocked
494
521
  );
495
522
  case "ajax":
496
523
  return extractAjax(strategy, document, environment, depth);
@@ -562,6 +589,22 @@ var BlankGrimoireRuntime = (() => {
562
589
  const items = value.split(/[,|\n]/).map((item) => item.trim()).filter((item) => item !== "");
563
590
  return items.length > 0 ? items : void 0;
564
591
  }
592
+ function readList(root, listSelector, singleSelector) {
593
+ if (listSelector) {
594
+ const values = [];
595
+ const seen = /* @__PURE__ */ new Set();
596
+ for (const element of queryAll(root, listSelector)) {
597
+ const value = textOf(element);
598
+ if (value === "" || seen.has(value)) {
599
+ continue;
600
+ }
601
+ seen.add(value);
602
+ values.push(value);
603
+ }
604
+ return values.length > 0 ? values : void 0;
605
+ }
606
+ return splitList(readText(root, singleSelector));
607
+ }
565
608
  function normalizeStatus(value) {
566
609
  const text = value.trim().toLowerCase();
567
610
  if (text === "") {
@@ -666,19 +709,23 @@ var BlankGrimoireRuntime = (() => {
666
709
  details[field] = value;
667
710
  }
668
711
  }
669
- const altTitles = splitList(readText(document, detailsSelectors.altTitles));
712
+ const altTitles = readList(
713
+ document,
714
+ detailsSelectors.altTitlesItem,
715
+ detailsSelectors.altTitles
716
+ );
670
717
  if (altTitles) {
671
718
  details.altTitles = altTitles;
672
719
  }
673
- const authors = splitList(readText(document, detailsSelectors.authors));
720
+ const authors = readList(document, detailsSelectors.authorsItem, detailsSelectors.authors);
674
721
  if (authors) {
675
722
  details.authors = authors;
676
723
  }
677
- const artists = splitList(readText(document, detailsSelectors.artists));
724
+ const artists = readList(document, detailsSelectors.artistsItem, detailsSelectors.artists);
678
725
  if (artists) {
679
726
  details.artists = artists;
680
727
  }
681
- const tags = splitList(readText(document, detailsSelectors.tags));
728
+ const tags = readList(document, detailsSelectors.tagsItem, detailsSelectors.tags);
682
729
  if (tags) {
683
730
  details.tags = tags;
684
731
  }
@@ -708,8 +755,14 @@ var BlankGrimoireRuntime = (() => {
708
755
  }
709
756
  const selectors = selectorsFor(runtime).chapters;
710
757
  const baseURL = runtime.context.baseURL;
758
+ const excludedSelector = selectors.excludeItem ?? selectors.premiumMarker;
711
759
  const chapters = [];
760
+ let dropped = 0;
712
761
  for (const element of queryAll(runtime.document, selectors.item)) {
762
+ if (excludedSelector && element.querySelector(excludedSelector)) {
763
+ dropped += 1;
764
+ continue;
765
+ }
713
766
  const link = selectLink(element, selectors.link);
714
767
  const href = link ? attr(link, "href") : null;
715
768
  if (!href) {
@@ -733,6 +786,11 @@ var BlankGrimoireRuntime = (() => {
733
786
  }
734
787
  chapters.push(chapter);
735
788
  }
789
+ if (dropped > 0) {
790
+ console.log(
791
+ `chapters: dropped ${dropped} paid ${dropped === 1 ? "row" : "rows"}`
792
+ );
793
+ }
736
794
  const numeric = chapters.every((chapter) => chapter.number !== void 0);
737
795
  if (numeric) {
738
796
  chapters.sort((a, b) => {
@@ -752,7 +810,13 @@ var BlankGrimoireRuntime = (() => {
752
810
  return hook(runtime.context, keys.mangaKey, keys.chapterKey);
753
811
  }
754
812
  const selectors = selectorsFor(runtime);
755
- const environment = { baseURL: runtime.context.baseURL };
813
+ let blocked = false;
814
+ const environment = {
815
+ baseURL: runtime.context.baseURL,
816
+ onBlocked: () => {
817
+ blocked = true;
818
+ }
819
+ };
756
820
  if (runtime.window) {
757
821
  environment.window = runtime.window;
758
822
  }
@@ -761,6 +825,10 @@ var BlankGrimoireRuntime = (() => {
761
825
  }
762
826
  for (const strategy of selectors.pages.strategies) {
763
827
  const pages = await runStrategy(strategy, runtime.document, environment);
828
+ if (blocked) {
829
+ console.log("pages: skipped, the payload is flagged as paid");
830
+ return [];
831
+ }
764
832
  if (pages.length > 0) {
765
833
  return pages;
766
834
  }
@@ -6,11 +6,15 @@ export interface ScriptJSONStrategy {
6
6
  pattern: string;
7
7
  path: string;
8
8
  base64?: boolean;
9
+ /** Paid-content guard evaluated against the parsed payload. */
10
+ skipWhen?: string;
9
11
  }
10
12
  export interface HydrationStrategy {
11
13
  kind: 'hydration';
12
14
  framework: 'next' | 'nuxt';
13
15
  path?: string;
16
+ /** Paid-content guard evaluated against the parsed payload. */
17
+ skipWhen?: string;
14
18
  }
15
19
  export interface AjaxStrategy {
16
20
  kind: 'ajax';
@@ -23,6 +27,8 @@ export interface StrategyEnvironment {
23
27
  baseURL: string;
24
28
  window?: WindowLike;
25
29
  parseHTML?: ParseHTML;
30
+ /** Called when a `skipWhen` guard reports paid or locked content. */
31
+ onBlocked?: () => void;
26
32
  }
27
33
  /**
28
34
  * Parse a JSON value from script text. Inline scripts often wrap the payload
@@ -32,6 +38,12 @@ export interface StrategyEnvironment {
32
38
  export declare function parseJSONText(text: string): unknown;
33
39
  /** Dot/bracket path lookup, e.g. `props.pageProps.pages` or `items[0].url`. */
34
40
  export declare function readPath(value: unknown, path: string): unknown;
41
+ /**
42
+ * Evaluate a paid-content guard against a parsed payload. A guard is truthy for
43
+ * booleans, non-zero numbers and non-empty strings other than `"false"` and
44
+ * `"0"`, which is how sites mark locked entries in JSON.
45
+ */
46
+ export declare function isPaidPayload(parsed: unknown, skipWhen: string | undefined): boolean;
35
47
  /** Normalise a strategy payload (URL strings or page objects) into `Page[]`. */
36
48
  export declare function toPages(payload: unknown, baseURL: string, headers?: Record<string, string>): Page[];
37
49
  /**
@@ -39,9 +51,9 @@ export declare function toPages(payload: unknown, baseURL: string, headers?: Rec
39
51
  * matching external script is fetched through the native bridge. Failed
40
52
  * scripts are skipped so later candidates can still succeed.
41
53
  */
42
- export declare function extractScriptJSON(strategy: ScriptJSONStrategy, document: DocumentLike, baseURL: string, parseHTML?: ParseHTML): Promise<Page[]>;
54
+ export declare function extractScriptJSON(strategy: ScriptJSONStrategy, document: DocumentLike, baseURL: string, parseHTML?: ParseHTML, onBlocked?: () => void): Promise<Page[]>;
43
55
  /** Extract pages from a Next.js or Nuxt hydration payload. */
44
- export declare function extractHydration(strategy: HydrationStrategy, document: DocumentLike, baseURL: string, windowObj?: WindowLike): Page[];
56
+ export declare function extractHydration(strategy: HydrationStrategy, document: DocumentLike, baseURL: string, windowObj?: WindowLike, onBlocked?: () => void): Page[];
45
57
  /** Fetch a URL and apply the nested strategy to the response HTML. */
46
58
  export declare function extractAjax(strategy: AjaxStrategy, document: DocumentLike, environment: StrategyEnvironment, depth?: number): Promise<Page[]>;
47
59
  /** Run one strategy. A strategy returns `[]` when it does not match. */
@@ -106,6 +106,28 @@ export function readPath(value, path) {
106
106
  }
107
107
  return current;
108
108
  }
109
+ /**
110
+ * Evaluate a paid-content guard against a parsed payload. A guard is truthy for
111
+ * booleans, non-zero numbers and non-empty strings other than `"false"` and
112
+ * `"0"`, which is how sites mark locked entries in JSON.
113
+ */
114
+ export function isPaidPayload(parsed, skipWhen) {
115
+ if (!skipWhen) {
116
+ return false;
117
+ }
118
+ const value = readPath(parsed, skipWhen);
119
+ if (typeof value === 'boolean') {
120
+ return value;
121
+ }
122
+ if (typeof value === 'number') {
123
+ return Number.isFinite(value) && value !== 0;
124
+ }
125
+ if (typeof value === 'string') {
126
+ const text = value.trim().toLowerCase();
127
+ return text !== '' && text !== 'false' && text !== '0';
128
+ }
129
+ return false;
130
+ }
109
131
  function pageURLFromItem(item) {
110
132
  if (typeof item === 'string') {
111
133
  return item.trim() === '' ? null : item;
@@ -146,7 +168,7 @@ export function toPages(payload, baseURL, headers) {
146
168
  * matching external script is fetched through the native bridge. Failed
147
169
  * scripts are skipped so later candidates can still succeed.
148
170
  */
149
- export async function extractScriptJSON(strategy, document, baseURL, parseHTML) {
171
+ export async function extractScriptJSON(strategy, document, baseURL, parseHTML, onBlocked) {
150
172
  void parseHTML;
151
173
  const regex = new RegExp(strategy.pattern);
152
174
  for (const script of queryAll(document, 'script')) {
@@ -177,6 +199,10 @@ export async function extractScriptJSON(strategy, document, baseURL, parseHTML)
177
199
  if (parsed === undefined) {
178
200
  continue;
179
201
  }
202
+ if (isPaidPayload(parsed, strategy.skipWhen)) {
203
+ onBlocked?.();
204
+ return [];
205
+ }
180
206
  const payload = readPath(parsed, strategy.path);
181
207
  if (payload === undefined || payload === null) {
182
208
  continue;
@@ -215,11 +241,15 @@ function hydrationPayload(strategy, document, windowObj) {
215
241
  return undefined;
216
242
  }
217
243
  /** Extract pages from a Next.js or Nuxt hydration payload. */
218
- export function extractHydration(strategy, document, baseURL, windowObj) {
244
+ export function extractHydration(strategy, document, baseURL, windowObj, onBlocked) {
219
245
  const payload = hydrationPayload(strategy, document, windowObj);
220
246
  if (payload === undefined) {
221
247
  return [];
222
248
  }
249
+ if (isPaidPayload(payload, strategy.skipWhen)) {
250
+ onBlocked?.();
251
+ return [];
252
+ }
223
253
  let value = payload;
224
254
  if (strategy.path) {
225
255
  value = readPath(payload, strategy.path);
@@ -271,9 +301,9 @@ export async function runStrategy(strategy, document, environment, depth = 0) {
271
301
  case 'css':
272
302
  return extractCss(strategy, document, environment.baseURL);
273
303
  case 'scriptJSON':
274
- return extractScriptJSON(strategy, document, environment.baseURL, environment.parseHTML);
304
+ return extractScriptJSON(strategy, document, environment.baseURL, environment.parseHTML, environment.onBlocked);
275
305
  case 'hydration':
276
- return extractHydration(strategy, document, environment.baseURL, environment.window);
306
+ return extractHydration(strategy, document, environment.baseURL, environment.window, environment.onBlocked);
277
307
  case 'ajax':
278
308
  return extractAjax(strategy, document, environment, depth);
279
309
  default:
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@blankgrimoire/runtime",
3
- "version": "1.3.1",
3
+ "version": "1.4.0",
4
4
  "description": "Declarative extraction runtime for BlankGrimoire theme adapters.",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -26,7 +26,7 @@
26
26
  "directory": "packages/runtime"
27
27
  },
28
28
  "dependencies": {
29
- "@blankgrimoire/types": "1.3.1"
29
+ "@blankgrimoire/types": "1.4.0"
30
30
  },
31
31
  "devDependencies": {
32
32
  "esbuild": "^0.24.0",