feedscout 1.4.0 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. package/README.md +12 -17
  2. package/dist/blogrolls/defaults.cjs +1 -3
  3. package/dist/blogrolls/defaults.js +1 -2
  4. package/dist/blogrolls/extractors.cjs +6 -9
  5. package/dist/blogrolls/extractors.js +6 -9
  6. package/dist/blogrolls/index.cjs +6 -9
  7. package/dist/blogrolls/index.d.cts +1 -1
  8. package/dist/blogrolls/index.d.ts +1 -1
  9. package/dist/blogrolls/index.js +1 -4
  10. package/dist/blogrolls.cjs +4 -5
  11. package/dist/blogrolls.js +1 -2
  12. package/dist/common/discover/index.cjs +60 -21
  13. package/dist/common/discover/index.js +58 -19
  14. package/dist/common/discover/utils.cjs +30 -9
  15. package/dist/common/discover/utils.js +28 -8
  16. package/dist/common/locales.cjs +127 -8
  17. package/dist/common/locales.js +116 -2
  18. package/dist/common/types.cjs +10 -0
  19. package/dist/common/types.d.cts +23 -5
  20. package/dist/common/types.d.ts +23 -5
  21. package/dist/common/types.js +10 -0
  22. package/dist/common/uris/feed/index.cjs +12 -0
  23. package/dist/common/uris/feed/index.d.cts +6 -0
  24. package/dist/common/uris/feed/index.d.ts +6 -0
  25. package/dist/common/uris/feed/index.js +12 -0
  26. package/dist/common/uris/feed/types.d.cts +9 -0
  27. package/dist/common/uris/feed/types.d.ts +9 -0
  28. package/dist/common/uris/guess/index.cjs +2 -4
  29. package/dist/common/uris/guess/index.d.cts +2 -1
  30. package/dist/common/uris/guess/index.d.ts +2 -1
  31. package/dist/common/uris/guess/index.js +1 -3
  32. package/dist/common/uris/guess/types.d.cts +3 -1
  33. package/dist/common/uris/guess/types.d.ts +3 -1
  34. package/dist/common/uris/guess/utils.cjs +5 -4
  35. package/dist/common/uris/guess/utils.d.cts +3 -1
  36. package/dist/common/uris/guess/utils.d.ts +3 -1
  37. package/dist/common/uris/guess/utils.js +5 -3
  38. package/dist/common/uris/headers/index.cjs +2 -4
  39. package/dist/common/uris/headers/index.js +1 -3
  40. package/dist/common/uris/html/handlers.cjs +2 -4
  41. package/dist/common/uris/html/handlers.js +1 -3
  42. package/dist/common/uris/html/index.cjs +2 -4
  43. package/dist/common/uris/html/index.js +1 -3
  44. package/dist/common/uris/index.cjs +23 -17
  45. package/dist/common/uris/index.js +19 -13
  46. package/dist/common/uris/platform/index.cjs +3 -5
  47. package/dist/common/uris/platform/index.d.cts +7 -0
  48. package/dist/common/uris/platform/index.d.ts +7 -0
  49. package/dist/common/uris/platform/index.js +3 -4
  50. package/dist/common/uris/platform/types.d.cts +4 -2
  51. package/dist/common/uris/platform/types.d.ts +4 -2
  52. package/dist/common/utils.cjs +25 -8
  53. package/dist/common/utils.d.cts +2 -2
  54. package/dist/common/utils.d.ts +2 -2
  55. package/dist/common/utils.js +23 -7
  56. package/dist/favicons/defaults.cjs +60 -0
  57. package/dist/favicons/defaults.d.cts +18 -0
  58. package/dist/favicons/defaults.d.ts +18 -0
  59. package/dist/favicons/defaults.js +53 -0
  60. package/dist/favicons/extractors.cjs +13 -0
  61. package/dist/favicons/extractors.d.cts +7 -0
  62. package/dist/favicons/extractors.d.ts +7 -0
  63. package/dist/favicons/extractors.js +13 -0
  64. package/dist/favicons/index.cjs +29 -0
  65. package/dist/favicons/index.d.cts +7 -0
  66. package/dist/favicons/index.d.ts +7 -0
  67. package/dist/favicons/index.js +29 -0
  68. package/dist/favicons/platform/handlers/bluesky.cjs +27 -0
  69. package/dist/favicons/platform/handlers/bluesky.js +27 -0
  70. package/dist/favicons/platform/handlers/codeberg.cjs +18 -0
  71. package/dist/favicons/platform/handlers/codeberg.js +18 -0
  72. package/dist/favicons/platform/handlers/deviantart.cjs +29 -0
  73. package/dist/favicons/platform/handlers/deviantart.js +29 -0
  74. package/dist/favicons/platform/handlers/github.cjs +18 -0
  75. package/dist/favicons/platform/handlers/github.js +18 -0
  76. package/dist/favicons/platform/handlers/githubGist.cjs +18 -0
  77. package/dist/favicons/platform/handlers/githubGist.js +18 -0
  78. package/dist/favicons/platform/handlers/lobsters.cjs +21 -0
  79. package/dist/favicons/platform/handlers/lobsters.js +21 -0
  80. package/dist/favicons/platform/handlers/mastodon.cjs +36 -0
  81. package/dist/favicons/platform/handlers/mastodon.js +34 -0
  82. package/dist/favicons/platform/handlers/reddit.cjs +41 -0
  83. package/dist/favicons/platform/handlers/reddit.js +41 -0
  84. package/dist/favicons/platform/handlers/sourceforge.cjs +21 -0
  85. package/dist/favicons/platform/handlers/sourceforge.js +21 -0
  86. package/dist/favicons/platform/handlers/tumblr.cjs +16 -0
  87. package/dist/favicons/platform/handlers/tumblr.js +16 -0
  88. package/dist/favicons/types.d.cts +4 -0
  89. package/dist/favicons/types.d.ts +4 -0
  90. package/dist/favicons.cjs +12 -0
  91. package/dist/favicons.d.cts +4 -0
  92. package/dist/favicons.d.ts +4 -0
  93. package/dist/favicons.js +3 -0
  94. package/dist/feeds/defaults.cjs +66 -37
  95. package/dist/feeds/defaults.d.cts +2 -2
  96. package/dist/feeds/defaults.d.ts +2 -2
  97. package/dist/feeds/defaults.js +52 -23
  98. package/dist/feeds/extractors.cjs +1 -3
  99. package/dist/feeds/extractors.js +1 -3
  100. package/dist/feeds/index.cjs +6 -8
  101. package/dist/feeds/index.d.cts +1 -1
  102. package/dist/feeds/index.d.ts +1 -1
  103. package/dist/feeds/index.js +1 -3
  104. package/dist/feeds/platform/handlers/behance.cjs +10 -6
  105. package/dist/feeds/platform/handlers/behance.js +10 -6
  106. package/dist/feeds/platform/handlers/blogspot.cjs +19 -8
  107. package/dist/feeds/platform/handlers/blogspot.js +19 -7
  108. package/dist/feeds/platform/handlers/bluesky.cjs +7 -6
  109. package/dist/feeds/platform/handlers/bluesky.js +7 -6
  110. package/dist/feeds/platform/handlers/codeberg.cjs +69 -0
  111. package/dist/feeds/platform/handlers/codeberg.js +67 -0
  112. package/dist/feeds/platform/handlers/csdn.cjs +19 -0
  113. package/dist/feeds/platform/handlers/csdn.js +19 -0
  114. package/dist/feeds/platform/handlers/dailymotion.cjs +10 -6
  115. package/dist/feeds/platform/handlers/dailymotion.js +10 -6
  116. package/dist/feeds/platform/handlers/deviantart.cjs +20 -8
  117. package/dist/feeds/platform/handlers/deviantart.js +18 -8
  118. package/dist/feeds/platform/handlers/devto.cjs +10 -6
  119. package/dist/feeds/platform/handlers/devto.js +10 -6
  120. package/dist/feeds/platform/handlers/douban.cjs +54 -0
  121. package/dist/feeds/platform/handlers/douban.js +54 -0
  122. package/dist/feeds/platform/handlers/github.cjs +37 -16
  123. package/dist/feeds/platform/handlers/github.js +35 -16
  124. package/dist/feeds/platform/handlers/githubGist.cjs +17 -17
  125. package/dist/feeds/platform/handlers/githubGist.js +15 -17
  126. package/dist/feeds/platform/handlers/gitlab.cjs +18 -8
  127. package/dist/feeds/platform/handlers/gitlab.js +18 -8
  128. package/dist/feeds/platform/handlers/goodreads.cjs +39 -0
  129. package/dist/feeds/platform/handlers/goodreads.js +39 -0
  130. package/dist/feeds/platform/handlers/hashnode.cjs +16 -0
  131. package/dist/feeds/platform/handlers/hashnode.js +16 -0
  132. package/dist/feeds/platform/handlers/hatenablog.cjs +53 -0
  133. package/dist/feeds/platform/handlers/hatenablog.js +53 -0
  134. package/dist/feeds/platform/handlers/itchio.cjs +88 -0
  135. package/dist/feeds/platform/handlers/itchio.js +88 -0
  136. package/dist/feeds/platform/handlers/kickstarter.cjs +10 -6
  137. package/dist/feeds/platform/handlers/kickstarter.js +10 -6
  138. package/dist/feeds/platform/handlers/letterboxd.cjs +42 -0
  139. package/dist/feeds/platform/handlers/letterboxd.js +42 -0
  140. package/dist/feeds/platform/handlers/lobsters.cjs +35 -12
  141. package/dist/feeds/platform/handlers/lobsters.js +34 -12
  142. package/dist/feeds/platform/handlers/mastodon.cjs +40 -0
  143. package/dist/feeds/platform/handlers/mastodon.js +40 -0
  144. package/dist/feeds/platform/handlers/medium.cjs +26 -10
  145. package/dist/feeds/platform/handlers/medium.js +26 -10
  146. package/dist/feeds/platform/handlers/paragraph.cjs +21 -0
  147. package/dist/feeds/platform/handlers/paragraph.js +21 -0
  148. package/dist/feeds/platform/handlers/pinterest.cjs +6 -10
  149. package/dist/feeds/platform/handlers/pinterest.js +6 -10
  150. package/dist/feeds/platform/handlers/producthunt.cjs +14 -7
  151. package/dist/feeds/platform/handlers/producthunt.js +14 -7
  152. package/dist/feeds/platform/handlers/reddit.cjs +35 -12
  153. package/dist/feeds/platform/handlers/reddit.js +34 -12
  154. package/dist/feeds/platform/handlers/soundcloud.cjs +6 -5
  155. package/dist/feeds/platform/handlers/soundcloud.js +6 -5
  156. package/dist/feeds/platform/handlers/sourceforge.cjs +20 -0
  157. package/dist/feeds/platform/handlers/sourceforge.js +19 -0
  158. package/dist/feeds/platform/handlers/stackExchange.cjs +37 -0
  159. package/dist/feeds/platform/handlers/stackExchange.js +37 -0
  160. package/dist/feeds/platform/handlers/steam.cjs +26 -0
  161. package/dist/feeds/platform/handlers/steam.js +26 -0
  162. package/dist/feeds/platform/handlers/substack.cjs +6 -5
  163. package/dist/feeds/platform/handlers/substack.js +6 -5
  164. package/dist/feeds/platform/handlers/tumblr.cjs +13 -7
  165. package/dist/feeds/platform/handlers/tumblr.js +12 -7
  166. package/dist/feeds/platform/handlers/v2ex.cjs +33 -0
  167. package/dist/feeds/platform/handlers/v2ex.js +33 -0
  168. package/dist/feeds/platform/handlers/vimeo.cjs +68 -0
  169. package/dist/feeds/platform/handlers/vimeo.js +68 -0
  170. package/dist/feeds/platform/handlers/wordpress.cjs +46 -12
  171. package/dist/feeds/platform/handlers/wordpress.js +46 -12
  172. package/dist/feeds/platform/handlers/wpengine.cjs +10 -0
  173. package/dist/feeds/platform/handlers/wpengine.js +11 -0
  174. package/dist/feeds/platform/handlers/ximalaya.cjs +19 -0
  175. package/dist/feeds/platform/handlers/ximalaya.js +19 -0
  176. package/dist/feeds/platform/handlers/youtube.cjs +35 -13
  177. package/dist/feeds/platform/handlers/youtube.js +35 -13
  178. package/dist/feeds.cjs +4 -5
  179. package/dist/feeds.js +1 -2
  180. package/dist/hubs/discover/index.cjs +7 -9
  181. package/dist/hubs/discover/index.js +1 -3
  182. package/dist/hubs/discover/utils.cjs +1 -3
  183. package/dist/hubs/discover/utils.js +1 -2
  184. package/dist/hubs/feed/index.cjs +1 -3
  185. package/dist/hubs/feed/index.js +1 -3
  186. package/dist/hubs/headers/index.cjs +3 -5
  187. package/dist/hubs/headers/index.js +1 -3
  188. package/dist/hubs/html/index.cjs +3 -5
  189. package/dist/hubs/html/index.js +1 -3
  190. package/dist/hubs.js +1 -1
  191. package/dist/index.cjs +8 -7
  192. package/dist/index.d.cts +3 -2
  193. package/dist/index.d.ts +3 -2
  194. package/dist/index.js +2 -2
  195. package/dist/methods.cjs +13 -10
  196. package/dist/methods.d.cts +3 -1
  197. package/dist/methods.d.ts +3 -1
  198. package/dist/methods.js +3 -2
  199. package/dist/utils.cjs +3 -4
  200. package/dist/utils.js +1 -2
  201. package/package.json +15 -5
package/README.md CHANGED
@@ -18,31 +18,26 @@ Finds feeds by scanning links and anchors in HTML content, parsing HTTP headers,
18
18
 
19
19
  ### Supported Content
20
20
 
21
- - **Feeds** RSS, Atom, JSON Feed, and RDF. Each feed is validated and returns metadata like format, title, description, and site URL.
22
- - **Blogrolls** OPML files containing feed subscriptions. Validated and returns title.
23
- - **WebSub hubs** Find hubs for real-time feed update notifications.
21
+ | Type | Description |
22
+ | --- | --- |
23
+ | Feeds | RSS, Atom, JSON Feed, and RDF. Each feed is validated and returns metadata like format, title, description, and site URL. |
24
+ | Blogrolls | OPML files containing feed subscriptions. Validated and returns title. |
25
+ | WebSub hubs | Find hubs for real-time feed update notifications. |
24
26
 
25
27
  ### Discovery Methods
26
28
 
27
- - **Platform** Generates feed URLs for YouTube, GitHub, WordPress, and other popular platforms using URL pattern matching.
28
- - **HTML** Scans `<link>` elements with feed MIME types and `<a>` elements matching feed patterns or labels like "RSS", "Subscribe".
29
- - **Headers** Parses HTTP `Link` headers for `rel="alternate"` with feed MIME types per RFC 8288.
30
- - **Guess** Tests common paths (e.g. `/feed`, `/rss.xml`, `/atom.xml`) against the base URL as a fallback.
29
+ | Method | Description |
30
+ | --- | --- |
31
+ | Platform | Generates feed URLs for YouTube, GitHub, WordPress, and 30+ other popular platforms using URL pattern matching. |
32
+ | HTML | Scans `<link>` elements with feed MIME types and `<a>` elements matching feed patterns or labels like "RSS", "Subscribe". |
33
+ | Headers | Parses HTTP `Link` headers for `rel="alternate"` with feed MIME types per RFC 8288. |
34
+ | Guess | Tests common paths (e.g. `/feed`, `/rss.xml`, `/atom.xml`) against the base URL as a fallback. |
31
35
 
32
36
  ### Customization
33
37
 
34
- - **Custom extractors** Override the default parser to extract additional metadata from feeds and blogrolls.
35
- - **Configurable methods** — Enable/disable discovery methods or customize their options.
36
- - **Adapter system** — Use native fetch or easily integrate with Axios, Got, or Ky.
37
- - **Concurrency control** — Limit parallel requests during validation.
38
- - **Progress tracking** — Monitor discovery progress with callbacks.
39
- - **Type-safe** — Full TypeScript support with exported types.
40
- - **Tree-shakable** — Import only what you need.
41
-
38
+ Uses native fetch by default but supports custom adapters for Axios, Got, Ky, or any other HTTP client. Discovery methods can be individually enabled or disabled and their options adjusted. Custom extractors let you override the default parser to pull additional metadata from feeds and blogrolls.
42
39
  ## Quick Start
43
40
 
44
- This is a short guide on how to get you up and running with Feedscout.
45
-
46
41
  For a full overview of all the features, [visit the documentation](https://feedscout.dev).
47
42
 
48
43
  ### Installation
@@ -1,4 +1,3 @@
1
-
2
1
  //#region src/blogrolls/defaults.ts
3
2
  const mimeTypes = [
4
3
  "text/x-opml",
@@ -40,7 +39,6 @@ const defaultHtmlOptions = {
40
39
  };
41
40
  const defaultHeadersOptions = { linkSelectors };
42
41
  const defaultGuessOptions = { uris: urisBalanced };
43
-
44
42
  //#endregion
45
43
  exports.anchorLabels = anchorLabels;
46
44
  exports.defaultGuessOptions = defaultGuessOptions;
@@ -50,4 +48,4 @@ exports.linkSelectors = linkSelectors;
50
48
  exports.mimeTypes = mimeTypes;
51
49
  exports.urisBalanced = urisBalanced;
52
50
  exports.urisComprehensive = urisComprehensive;
53
- exports.urisMinimal = urisMinimal;
51
+ exports.urisMinimal = urisMinimal;
@@ -39,6 +39,5 @@ const defaultHtmlOptions = {
39
39
  };
40
40
  const defaultHeadersOptions = { linkSelectors };
41
41
  const defaultGuessOptions = { uris: urisBalanced };
42
-
43
42
  //#endregion
44
- export { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
43
+ export { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
@@ -1,5 +1,4 @@
1
1
  let feedsmith = require("feedsmith");
2
-
3
2
  //#region src/blogrolls/extractors.ts
4
3
  const defaultExtractor = async ({ content, url }) => {
5
4
  if (!content) return {
@@ -12,13 +11,11 @@ const defaultExtractor = async ({ content, url }) => {
12
11
  isValid: true,
13
12
  title: (0, feedsmith.parseOpml)(content).head?.title
14
13
  };
15
- } catch {
16
- return {
17
- url,
18
- isValid: false
19
- };
20
- }
14
+ } catch {}
15
+ return {
16
+ url,
17
+ isValid: false
18
+ };
21
19
  };
22
-
23
20
  //#endregion
24
- exports.defaultExtractor = defaultExtractor;
21
+ exports.defaultExtractor = defaultExtractor;
@@ -1,5 +1,4 @@
1
1
  import { parseOpml } from "feedsmith";
2
-
3
2
  //#region src/blogrolls/extractors.ts
4
3
  const defaultExtractor = async ({ content, url }) => {
5
4
  if (!content) return {
@@ -12,13 +11,11 @@ const defaultExtractor = async ({ content, url }) => {
12
11
  isValid: true,
13
12
  title: parseOpml(content).head?.title
14
13
  };
15
- } catch {
16
- return {
17
- url,
18
- isValid: false
19
- };
20
- }
14
+ } catch {}
15
+ return {
16
+ url,
17
+ isValid: false
18
+ };
21
19
  };
22
-
23
20
  //#endregion
24
- export { defaultExtractor };
21
+ export { defaultExtractor };
@@ -1,9 +1,8 @@
1
- const require_utils = require('../common/utils.cjs');
2
- const require_utils$1 = require('../common/discover/utils.cjs');
3
- const require_index = require('../common/discover/index.cjs');
4
- const require_defaults = require('./defaults.cjs');
5
- const require_extractors = require('./extractors.cjs');
6
-
1
+ const require_utils = require("../common/utils.cjs");
2
+ const require_utils$1 = require("../common/discover/utils.cjs");
3
+ const require_index = require("../common/discover/index.cjs");
4
+ const require_defaults = require("./defaults.cjs");
5
+ const require_extractors = require("./extractors.cjs");
7
6
  //#region src/blogrolls/index.ts
8
7
  const discoverBlogrolls = async (input, options = {}) => {
9
8
  return require_index.discover(input, {
@@ -17,12 +16,10 @@ const discoverBlogrolls = async (input, options = {}) => {
17
16
  extractFn: options.extractFn ?? require_extractors.defaultExtractor,
18
17
  normalizeUrlFn: options.normalizeUrlFn ?? require_utils.normalizeUrl
19
18
  }, {
20
- platform: { handlers: [] },
21
19
  html: require_defaults.defaultHtmlOptions,
22
20
  headers: require_defaults.defaultHeadersOptions,
23
21
  guess: require_defaults.defaultGuessOptions
24
22
  });
25
23
  };
26
-
27
24
  //#endregion
28
- exports.discoverBlogrolls = discoverBlogrolls;
25
+ exports.discoverBlogrolls = discoverBlogrolls;
@@ -2,6 +2,6 @@ import { DiscoverInput, DiscoverOptions, DiscoverResult } from "../common/types.
2
2
  import { BlogrollResult } from "./types.cjs";
3
3
 
4
4
  //#region src/blogrolls/index.d.ts
5
- declare const discoverBlogrolls: <TValid extends BlogrollResult = BlogrollResult>(input: DiscoverInput, options?: DiscoverOptions<TValid>) => Promise<Array<DiscoverResult<TValid>>>;
5
+ declare const discoverBlogrolls: <TValid extends BlogrollResult = BlogrollResult>(input: DiscoverInput, options?: DiscoverOptions<TValid, "html" | "headers" | "guess">) => Promise<Array<DiscoverResult<TValid>>>;
6
6
  //#endregion
7
7
  export { discoverBlogrolls };
@@ -2,6 +2,6 @@ import { DiscoverInput, DiscoverOptions, DiscoverResult } from "../common/types.
2
2
  import { BlogrollResult } from "./types.js";
3
3
 
4
4
  //#region src/blogrolls/index.d.ts
5
- declare const discoverBlogrolls: <TValid extends BlogrollResult = BlogrollResult>(input: DiscoverInput, options?: DiscoverOptions<TValid>) => Promise<Array<DiscoverResult<TValid>>>;
5
+ declare const discoverBlogrolls: <TValid extends BlogrollResult = BlogrollResult>(input: DiscoverInput, options?: DiscoverOptions<TValid, "html" | "headers" | "guess">) => Promise<Array<DiscoverResult<TValid>>>;
6
6
  //#endregion
7
7
  export { discoverBlogrolls };
@@ -3,7 +3,6 @@ import { defaultFetchFn } from "../common/discover/utils.js";
3
3
  import { discover } from "../common/discover/index.js";
4
4
  import { defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions } from "./defaults.js";
5
5
  import { defaultExtractor } from "./extractors.js";
6
-
7
6
  //#region src/blogrolls/index.ts
8
7
  const discoverBlogrolls = async (input, options = {}) => {
9
8
  return discover(input, {
@@ -17,12 +16,10 @@ const discoverBlogrolls = async (input, options = {}) => {
17
16
  extractFn: options.extractFn ?? defaultExtractor,
18
17
  normalizeUrlFn: options.normalizeUrlFn ?? normalizeUrl
19
18
  }, {
20
- platform: { handlers: [] },
21
19
  html: defaultHtmlOptions,
22
20
  headers: defaultHeadersOptions,
23
21
  guess: defaultGuessOptions
24
22
  });
25
23
  };
26
-
27
24
  //#endregion
28
- export { discoverBlogrolls };
25
+ export { discoverBlogrolls };
@@ -1,7 +1,6 @@
1
- Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });
2
- const require_defaults = require('./blogrolls/defaults.cjs');
3
- const require_extractors = require('./blogrolls/extractors.cjs');
4
-
1
+ Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
2
+ const require_defaults = require("./blogrolls/defaults.cjs");
3
+ const require_extractors = require("./blogrolls/extractors.cjs");
5
4
  exports.anchorLabels = require_defaults.anchorLabels;
6
5
  exports.defaultExtractor = require_extractors.defaultExtractor;
7
6
  exports.defaultGuessOptions = require_defaults.defaultGuessOptions;
@@ -11,4 +10,4 @@ exports.linkSelectors = require_defaults.linkSelectors;
11
10
  exports.mimeTypes = require_defaults.mimeTypes;
12
11
  exports.urisBalanced = require_defaults.urisBalanced;
13
12
  exports.urisComprehensive = require_defaults.urisComprehensive;
14
- exports.urisMinimal = require_defaults.urisMinimal;
13
+ exports.urisMinimal = require_defaults.urisMinimal;
package/dist/blogrolls.js CHANGED
@@ -1,4 +1,3 @@
1
1
  import { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal } from "./blogrolls/defaults.js";
2
2
  import { defaultExtractor } from "./blogrolls/extractors.js";
3
-
4
- export { anchorLabels, defaultExtractor, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
3
+ export { anchorLabels, defaultExtractor, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
@@ -1,7 +1,7 @@
1
- const require_utils = require('../utils.cjs');
2
- const require_index = require('../uris/index.cjs');
3
- const require_utils$1 = require('./utils.cjs');
4
-
1
+ const require_types = require("../types.cjs");
2
+ const require_utils = require("../utils.cjs");
3
+ const require_index = require("../uris/index.cjs");
4
+ const require_utils$1 = require("./utils.cjs");
5
5
  //#region src/common/discover/index.ts
6
6
  const discover = async (input, options, defaults) => {
7
7
  const { methods, fetchFn, extractFn, normalizeUrlFn, stopOnFirstMethod = false, stopOnFirstResult = false, concurrency = 3, includeInvalid = false, onProgress } = options;
@@ -9,46 +9,85 @@ const discover = async (input, options, defaults) => {
9
9
  if (normalizedInput.content) {
10
10
  const result = await extractFn({
11
11
  url: normalizedInput.url,
12
- content: normalizedInput.content
12
+ content: normalizedInput.content,
13
+ headers: normalizedInput.headers
13
14
  });
14
15
  if (result.isValid) return [result];
15
16
  }
16
- const rawUris = require_index.discoverUris(require_utils$1.normalizeMethodsConfig(normalizedInput, methods, defaults), stopOnFirstMethod);
17
- const uris = [...new Set(rawUris.map((uri) => normalizeUrlFn(uri, normalizedInput.url)))];
17
+ const urisByMethod = await require_index.discoverUris(require_utils$1.normalizeMethodsConfig(normalizedInput, methods, defaults), fetchFn);
18
+ const seen = /* @__PURE__ */ new Set();
19
+ const methodGroups = [];
20
+ for (const method of require_types.discoverMethodOrder) {
21
+ const rawUris = urisByMethod[method];
22
+ if (!rawUris?.length) continue;
23
+ const unique = rawUris.map((entry) => {
24
+ return require_utils$1.normalizeUriEntry(entry, normalizeUrlFn, normalizedInput.url);
25
+ }).filter((entry) => {
26
+ const key = typeof entry.uri === "string" ? entry.uri : entry.uri.join("\0");
27
+ if (seen.has(key)) return false;
28
+ seen.add(key);
29
+ return true;
30
+ });
31
+ if (unique.length > 0) methodGroups.push({
32
+ method,
33
+ entries: unique
34
+ });
35
+ }
36
+ const total = methodGroups.reduce((sum, group) => sum + group.entries.length, 0);
18
37
  const results = [];
19
38
  let tested = 0;
20
39
  let found = 0;
21
- const processUri = async (url) => {
40
+ const fetchAndExtract = async (url) => {
22
41
  try {
23
42
  const fetchResult = await fetchFn(url);
24
- const extractResult = await extractFn({
43
+ return await extractFn({
25
44
  url: fetchResult.url,
26
- content: typeof fetchResult.body === "string" ? fetchResult.body : ""
45
+ content: typeof fetchResult.body === "string" ? fetchResult.body : "",
46
+ headers: fetchResult.headers,
47
+ status: fetchResult.status
27
48
  });
28
- results.push(extractResult);
29
- if (extractResult.isValid) found += 1;
30
49
  } catch (error) {
31
- results.push({
50
+ return {
32
51
  url,
33
52
  isValid: false,
34
53
  error
54
+ };
55
+ }
56
+ };
57
+ const processUri = async (entry, method) => {
58
+ const alternatives = typeof entry.uri === "string" ? [entry.uri] : entry.uri;
59
+ for (const url of alternatives) {
60
+ const result = await fetchAndExtract(url);
61
+ results.push(entry.hint ? {
62
+ ...result,
63
+ method,
64
+ hint: entry.hint
65
+ } : {
66
+ ...result,
67
+ method
35
68
  });
36
- } finally {
37
69
  tested += 1;
70
+ if (result.isValid) found += 1;
38
71
  onProgress?.({
39
72
  tested,
40
- total: uris.length,
73
+ total,
41
74
  found,
42
75
  current: url
43
76
  });
77
+ if (result.isValid) break;
44
78
  }
45
79
  };
46
- await require_utils.processConcurrently(uris, processUri, {
47
- concurrency,
48
- shouldStop: () => stopOnFirstResult && found > 0
49
- });
80
+ for (const { method, entries } of methodGroups) {
81
+ const foundBefore = found;
82
+ await require_utils.processConcurrently(entries, (entry) => processUri(entry, method), {
83
+ concurrency,
84
+ shouldStop: () => {
85
+ return stopOnFirstResult && found > 0;
86
+ }
87
+ });
88
+ if (stopOnFirstMethod && found > foundBefore) break;
89
+ }
50
90
  return includeInvalid ? results : results.filter((result) => result.isValid);
51
91
  };
52
-
53
92
  //#endregion
54
- exports.discover = discover;
93
+ exports.discover = discover;
@@ -1,7 +1,7 @@
1
+ import { discoverMethodOrder } from "../types.js";
1
2
  import { processConcurrently } from "../utils.js";
2
3
  import { discoverUris } from "../uris/index.js";
3
- import { normalizeInput, normalizeMethodsConfig } from "./utils.js";
4
-
4
+ import { normalizeInput, normalizeMethodsConfig, normalizeUriEntry } from "./utils.js";
5
5
  //#region src/common/discover/index.ts
6
6
  const discover = async (input, options, defaults) => {
7
7
  const { methods, fetchFn, extractFn, normalizeUrlFn, stopOnFirstMethod = false, stopOnFirstResult = false, concurrency = 3, includeInvalid = false, onProgress } = options;
@@ -9,46 +9,85 @@ const discover = async (input, options, defaults) => {
9
9
  if (normalizedInput.content) {
10
10
  const result = await extractFn({
11
11
  url: normalizedInput.url,
12
- content: normalizedInput.content
12
+ content: normalizedInput.content,
13
+ headers: normalizedInput.headers
13
14
  });
14
15
  if (result.isValid) return [result];
15
16
  }
16
- const rawUris = discoverUris(normalizeMethodsConfig(normalizedInput, methods, defaults), stopOnFirstMethod);
17
- const uris = [...new Set(rawUris.map((uri) => normalizeUrlFn(uri, normalizedInput.url)))];
17
+ const urisByMethod = await discoverUris(normalizeMethodsConfig(normalizedInput, methods, defaults), fetchFn);
18
+ const seen = /* @__PURE__ */ new Set();
19
+ const methodGroups = [];
20
+ for (const method of discoverMethodOrder) {
21
+ const rawUris = urisByMethod[method];
22
+ if (!rawUris?.length) continue;
23
+ const unique = rawUris.map((entry) => {
24
+ return normalizeUriEntry(entry, normalizeUrlFn, normalizedInput.url);
25
+ }).filter((entry) => {
26
+ const key = typeof entry.uri === "string" ? entry.uri : entry.uri.join("\0");
27
+ if (seen.has(key)) return false;
28
+ seen.add(key);
29
+ return true;
30
+ });
31
+ if (unique.length > 0) methodGroups.push({
32
+ method,
33
+ entries: unique
34
+ });
35
+ }
36
+ const total = methodGroups.reduce((sum, group) => sum + group.entries.length, 0);
18
37
  const results = [];
19
38
  let tested = 0;
20
39
  let found = 0;
21
- const processUri = async (url) => {
40
+ const fetchAndExtract = async (url) => {
22
41
  try {
23
42
  const fetchResult = await fetchFn(url);
24
- const extractResult = await extractFn({
43
+ return await extractFn({
25
44
  url: fetchResult.url,
26
- content: typeof fetchResult.body === "string" ? fetchResult.body : ""
45
+ content: typeof fetchResult.body === "string" ? fetchResult.body : "",
46
+ headers: fetchResult.headers,
47
+ status: fetchResult.status
27
48
  });
28
- results.push(extractResult);
29
- if (extractResult.isValid) found += 1;
30
49
  } catch (error) {
31
- results.push({
50
+ return {
32
51
  url,
33
52
  isValid: false,
34
53
  error
54
+ };
55
+ }
56
+ };
57
+ const processUri = async (entry, method) => {
58
+ const alternatives = typeof entry.uri === "string" ? [entry.uri] : entry.uri;
59
+ for (const url of alternatives) {
60
+ const result = await fetchAndExtract(url);
61
+ results.push(entry.hint ? {
62
+ ...result,
63
+ method,
64
+ hint: entry.hint
65
+ } : {
66
+ ...result,
67
+ method
35
68
  });
36
- } finally {
37
69
  tested += 1;
70
+ if (result.isValid) found += 1;
38
71
  onProgress?.({
39
72
  tested,
40
- total: uris.length,
73
+ total,
41
74
  found,
42
75
  current: url
43
76
  });
77
+ if (result.isValid) break;
44
78
  }
45
79
  };
46
- await processConcurrently(uris, processUri, {
47
- concurrency,
48
- shouldStop: () => stopOnFirstResult && found > 0
49
- });
80
+ for (const { method, entries } of methodGroups) {
81
+ const foundBefore = found;
82
+ await processConcurrently(entries, (entry) => processUri(entry, method), {
83
+ concurrency,
84
+ shouldStop: () => {
85
+ return stopOnFirstResult && found > 0;
86
+ }
87
+ });
88
+ if (stopOnFirstMethod && found > foundBefore) break;
89
+ }
50
90
  return includeInvalid ? results : results.filter((result) => result.isValid);
51
91
  };
52
-
53
92
  //#endregion
54
- export { discover };
93
+ export { discover };
@@ -1,5 +1,4 @@
1
- const require_locales = require('../locales.cjs');
2
-
1
+ const require_locales = require("../locales.cjs");
3
2
  //#region src/common/discover/utils.ts
4
3
  const defaultFetchFn = async (url, options) => {
5
4
  const response = await fetch(url, {
@@ -23,14 +22,25 @@ const normalizeInput = async (input, fetchFn) => {
23
22
  headers: response.headers
24
23
  };
25
24
  };
25
+ const normalizeUriEntry = (entry, normalizeUrlFn, baseUrl) => {
26
+ if (typeof entry.uri === "string") return {
27
+ ...entry,
28
+ uri: normalizeUrlFn(entry.uri, baseUrl)
29
+ };
30
+ return {
31
+ ...entry,
32
+ uri: entry.uri.map((uri) => normalizeUrlFn(uri, baseUrl))
33
+ };
34
+ };
26
35
  const normalizeMethodsConfig = (input, methods, defaults) => {
27
36
  const methodsObj = Array.isArray(methods) ? Object.fromEntries(methods.map((method) => [method, true])) : methods;
28
37
  const methodsConfig = {};
29
- if (methodsObj.platform) {
38
+ if (methodsObj.platform && defaults.platform) {
30
39
  if (!input.url || input.url === "") throw new Error(require_locales.errors.platformMethodRequiresUrl);
31
40
  const platformOptions = methodsObj.platform === true ? {} : methodsObj.platform;
32
41
  methodsConfig.platform = {
33
- html: input.content ?? "",
42
+ content: input.content,
43
+ headers: input.headers,
34
44
  options: {
35
45
  ...defaults.platform,
36
46
  ...platformOptions,
@@ -38,7 +48,18 @@ const normalizeMethodsConfig = (input, methods, defaults) => {
38
48
  }
39
49
  };
40
50
  }
41
- if (methodsObj.html) {
51
+ if (methodsObj.feed && defaults.feed) {
52
+ if (input.content === void 0) throw new Error(require_locales.errors.feedMethodRequiresContent);
53
+ const feedOptions = methodsObj.feed === true ? {} : methodsObj.feed;
54
+ methodsConfig.feed = {
55
+ content: input.content,
56
+ options: {
57
+ ...defaults.feed,
58
+ ...feedOptions
59
+ }
60
+ };
61
+ }
62
+ if (methodsObj.html && defaults.html) {
42
63
  if (input.content === void 0) throw new Error(require_locales.errors.htmlMethodRequiresContent);
43
64
  const htmlOptions = methodsObj.html === true ? {} : methodsObj.html;
44
65
  methodsConfig.html = {
@@ -50,7 +71,7 @@ const normalizeMethodsConfig = (input, methods, defaults) => {
50
71
  }
51
72
  };
52
73
  }
53
- if (methodsObj.headers) {
74
+ if (methodsObj.headers && defaults.headers) {
54
75
  if (input.headers === void 0) throw new Error(require_locales.errors.headersMethodRequiresHeaders);
55
76
  const headersOptions = methodsObj.headers === true ? {} : methodsObj.headers;
56
77
  methodsConfig.headers = {
@@ -62,7 +83,7 @@ const normalizeMethodsConfig = (input, methods, defaults) => {
62
83
  }
63
84
  };
64
85
  }
65
- if (methodsObj.guess) {
86
+ if (methodsObj.guess && defaults.guess) {
66
87
  if (!input.url || input.url === "") throw new Error(require_locales.errors.guessMethodRequiresUrl);
67
88
  const guessOptions = methodsObj.guess === true ? {} : methodsObj.guess;
68
89
  methodsConfig.guess = { options: {
@@ -73,8 +94,8 @@ const normalizeMethodsConfig = (input, methods, defaults) => {
73
94
  }
74
95
  return methodsConfig;
75
96
  };
76
-
77
97
  //#endregion
78
98
  exports.defaultFetchFn = defaultFetchFn;
79
99
  exports.normalizeInput = normalizeInput;
80
- exports.normalizeMethodsConfig = normalizeMethodsConfig;
100
+ exports.normalizeMethodsConfig = normalizeMethodsConfig;
101
+ exports.normalizeUriEntry = normalizeUriEntry;
@@ -1,5 +1,4 @@
1
1
  import { errors } from "../locales.js";
2
-
3
2
  //#region src/common/discover/utils.ts
4
3
  const defaultFetchFn = async (url, options) => {
5
4
  const response = await fetch(url, {
@@ -23,14 +22,25 @@ const normalizeInput = async (input, fetchFn) => {
23
22
  headers: response.headers
24
23
  };
25
24
  };
25
+ const normalizeUriEntry = (entry, normalizeUrlFn, baseUrl) => {
26
+ if (typeof entry.uri === "string") return {
27
+ ...entry,
28
+ uri: normalizeUrlFn(entry.uri, baseUrl)
29
+ };
30
+ return {
31
+ ...entry,
32
+ uri: entry.uri.map((uri) => normalizeUrlFn(uri, baseUrl))
33
+ };
34
+ };
26
35
  const normalizeMethodsConfig = (input, methods, defaults) => {
27
36
  const methodsObj = Array.isArray(methods) ? Object.fromEntries(methods.map((method) => [method, true])) : methods;
28
37
  const methodsConfig = {};
29
- if (methodsObj.platform) {
38
+ if (methodsObj.platform && defaults.platform) {
30
39
  if (!input.url || input.url === "") throw new Error(errors.platformMethodRequiresUrl);
31
40
  const platformOptions = methodsObj.platform === true ? {} : methodsObj.platform;
32
41
  methodsConfig.platform = {
33
- html: input.content ?? "",
42
+ content: input.content,
43
+ headers: input.headers,
34
44
  options: {
35
45
  ...defaults.platform,
36
46
  ...platformOptions,
@@ -38,7 +48,18 @@ const normalizeMethodsConfig = (input, methods, defaults) => {
38
48
  }
39
49
  };
40
50
  }
41
- if (methodsObj.html) {
51
+ if (methodsObj.feed && defaults.feed) {
52
+ if (input.content === void 0) throw new Error(errors.feedMethodRequiresContent);
53
+ const feedOptions = methodsObj.feed === true ? {} : methodsObj.feed;
54
+ methodsConfig.feed = {
55
+ content: input.content,
56
+ options: {
57
+ ...defaults.feed,
58
+ ...feedOptions
59
+ }
60
+ };
61
+ }
62
+ if (methodsObj.html && defaults.html) {
42
63
  if (input.content === void 0) throw new Error(errors.htmlMethodRequiresContent);
43
64
  const htmlOptions = methodsObj.html === true ? {} : methodsObj.html;
44
65
  methodsConfig.html = {
@@ -50,7 +71,7 @@ const normalizeMethodsConfig = (input, methods, defaults) => {
50
71
  }
51
72
  };
52
73
  }
53
- if (methodsObj.headers) {
74
+ if (methodsObj.headers && defaults.headers) {
54
75
  if (input.headers === void 0) throw new Error(errors.headersMethodRequiresHeaders);
55
76
  const headersOptions = methodsObj.headers === true ? {} : methodsObj.headers;
56
77
  methodsConfig.headers = {
@@ -62,7 +83,7 @@ const normalizeMethodsConfig = (input, methods, defaults) => {
62
83
  }
63
84
  };
64
85
  }
65
- if (methodsObj.guess) {
86
+ if (methodsObj.guess && defaults.guess) {
66
87
  if (!input.url || input.url === "") throw new Error(errors.guessMethodRequiresUrl);
67
88
  const guessOptions = methodsObj.guess === true ? {} : methodsObj.guess;
68
89
  methodsConfig.guess = { options: {
@@ -73,6 +94,5 @@ const normalizeMethodsConfig = (input, methods, defaults) => {
73
94
  }
74
95
  return methodsConfig;
75
96
  };
76
-
77
97
  //#endregion
78
- export { defaultFetchFn, normalizeInput, normalizeMethodsConfig };
98
+ export { defaultFetchFn, normalizeInput, normalizeMethodsConfig, normalizeUriEntry };