feedscout 1.5.0 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (193) hide show
  1. package/README.md +12 -17
  2. package/dist/blogrolls/defaults.cjs +1 -3
  3. package/dist/blogrolls/defaults.js +1 -2
  4. package/dist/blogrolls/extractors.cjs +6 -9
  5. package/dist/blogrolls/extractors.js +6 -9
  6. package/dist/blogrolls/index.cjs +6 -9
  7. package/dist/blogrolls/index.d.cts +1 -1
  8. package/dist/blogrolls/index.d.ts +1 -1
  9. package/dist/blogrolls/index.js +1 -4
  10. package/dist/blogrolls.cjs +4 -5
  11. package/dist/blogrolls.js +1 -2
  12. package/dist/common/discover/index.cjs +24 -16
  13. package/dist/common/discover/index.js +20 -12
  14. package/dist/common/discover/utils.cjs +19 -9
  15. package/dist/common/discover/utils.js +18 -8
  16. package/dist/common/locales.cjs +59 -15
  17. package/dist/common/locales.js +49 -4
  18. package/dist/common/types.cjs +2 -3
  19. package/dist/common/types.d.cts +11 -6
  20. package/dist/common/types.d.ts +11 -6
  21. package/dist/common/types.js +2 -2
  22. package/dist/common/uris/feed/index.cjs +12 -0
  23. package/dist/common/uris/feed/index.d.cts +6 -0
  24. package/dist/common/uris/feed/index.d.ts +6 -0
  25. package/dist/common/uris/feed/index.js +12 -0
  26. package/dist/common/uris/feed/types.d.cts +9 -0
  27. package/dist/common/uris/feed/types.d.ts +9 -0
  28. package/dist/common/uris/guess/index.cjs +2 -4
  29. package/dist/common/uris/guess/index.js +1 -3
  30. package/dist/common/uris/guess/utils.cjs +1 -3
  31. package/dist/common/uris/guess/utils.js +1 -2
  32. package/dist/common/uris/headers/index.cjs +2 -4
  33. package/dist/common/uris/headers/index.js +1 -3
  34. package/dist/common/uris/html/handlers.cjs +2 -4
  35. package/dist/common/uris/html/handlers.js +1 -3
  36. package/dist/common/uris/html/index.cjs +2 -4
  37. package/dist/common/uris/html/index.js +1 -3
  38. package/dist/common/uris/index.cjs +15 -12
  39. package/dist/common/uris/index.js +8 -5
  40. package/dist/common/uris/platform/index.cjs +3 -5
  41. package/dist/common/uris/platform/index.d.cts +7 -0
  42. package/dist/common/uris/platform/index.d.ts +7 -0
  43. package/dist/common/uris/platform/index.js +3 -4
  44. package/dist/common/uris/platform/types.d.cts +3 -3
  45. package/dist/common/uris/platform/types.d.ts +3 -3
  46. package/dist/common/utils.cjs +20 -9
  47. package/dist/common/utils.d.cts +2 -2
  48. package/dist/common/utils.d.ts +2 -2
  49. package/dist/common/utils.js +18 -8
  50. package/dist/favicons/defaults.cjs +60 -0
  51. package/dist/favicons/defaults.d.cts +18 -0
  52. package/dist/favicons/defaults.d.ts +18 -0
  53. package/dist/favicons/defaults.js +53 -0
  54. package/dist/favicons/extractors.cjs +13 -0
  55. package/dist/favicons/extractors.d.cts +7 -0
  56. package/dist/favicons/extractors.d.ts +7 -0
  57. package/dist/favicons/extractors.js +13 -0
  58. package/dist/favicons/index.cjs +29 -0
  59. package/dist/favicons/index.d.cts +7 -0
  60. package/dist/favicons/index.d.ts +7 -0
  61. package/dist/favicons/index.js +29 -0
  62. package/dist/favicons/platform/handlers/bluesky.cjs +27 -0
  63. package/dist/favicons/platform/handlers/bluesky.js +27 -0
  64. package/dist/favicons/platform/handlers/codeberg.cjs +18 -0
  65. package/dist/favicons/platform/handlers/codeberg.js +18 -0
  66. package/dist/favicons/platform/handlers/deviantart.cjs +29 -0
  67. package/dist/favicons/platform/handlers/deviantart.js +29 -0
  68. package/dist/favicons/platform/handlers/github.cjs +18 -0
  69. package/dist/favicons/platform/handlers/github.js +18 -0
  70. package/dist/favicons/platform/handlers/githubGist.cjs +18 -0
  71. package/dist/favicons/platform/handlers/githubGist.js +18 -0
  72. package/dist/favicons/platform/handlers/lobsters.cjs +21 -0
  73. package/dist/favicons/platform/handlers/lobsters.js +21 -0
  74. package/dist/favicons/platform/handlers/mastodon.cjs +36 -0
  75. package/dist/favicons/platform/handlers/mastodon.js +34 -0
  76. package/dist/favicons/platform/handlers/reddit.cjs +41 -0
  77. package/dist/favicons/platform/handlers/reddit.js +41 -0
  78. package/dist/favicons/platform/handlers/sourceforge.cjs +21 -0
  79. package/dist/favicons/platform/handlers/sourceforge.js +21 -0
  80. package/dist/favicons/platform/handlers/tumblr.cjs +16 -0
  81. package/dist/favicons/platform/handlers/tumblr.js +16 -0
  82. package/dist/favicons/types.d.cts +4 -0
  83. package/dist/favicons/types.d.ts +4 -0
  84. package/dist/favicons.cjs +12 -0
  85. package/dist/favicons.d.cts +4 -0
  86. package/dist/favicons.d.ts +4 -0
  87. package/dist/favicons.js +3 -0
  88. package/dist/feeds/defaults.cjs +55 -25
  89. package/dist/feeds/defaults.js +40 -10
  90. package/dist/feeds/extractors.cjs +1 -3
  91. package/dist/feeds/extractors.js +1 -3
  92. package/dist/feeds/index.cjs +6 -8
  93. package/dist/feeds/index.d.cts +1 -1
  94. package/dist/feeds/index.d.ts +1 -1
  95. package/dist/feeds/index.js +1 -3
  96. package/dist/feeds/platform/handlers/behance.cjs +2 -4
  97. package/dist/feeds/platform/handlers/behance.js +1 -3
  98. package/dist/feeds/platform/handlers/blogspot.cjs +7 -6
  99. package/dist/feeds/platform/handlers/blogspot.js +6 -5
  100. package/dist/feeds/platform/handlers/bluesky.cjs +3 -5
  101. package/dist/feeds/platform/handlers/bluesky.js +2 -4
  102. package/dist/feeds/platform/handlers/codeberg.cjs +69 -0
  103. package/dist/feeds/platform/handlers/codeberg.js +67 -0
  104. package/dist/feeds/platform/handlers/csdn.cjs +19 -0
  105. package/dist/feeds/platform/handlers/csdn.js +19 -0
  106. package/dist/feeds/platform/handlers/dailymotion.cjs +2 -4
  107. package/dist/feeds/platform/handlers/dailymotion.js +1 -3
  108. package/dist/feeds/platform/handlers/deviantart.cjs +4 -4
  109. package/dist/feeds/platform/handlers/deviantart.js +1 -3
  110. package/dist/feeds/platform/handlers/devto.cjs +2 -4
  111. package/dist/feeds/platform/handlers/devto.js +1 -3
  112. package/dist/feeds/platform/handlers/douban.cjs +54 -0
  113. package/dist/feeds/platform/handlers/douban.js +54 -0
  114. package/dist/feeds/platform/handlers/github.cjs +5 -4
  115. package/dist/feeds/platform/handlers/github.js +2 -3
  116. package/dist/feeds/platform/handlers/githubGist.cjs +4 -4
  117. package/dist/feeds/platform/handlers/githubGist.js +1 -3
  118. package/dist/feeds/platform/handlers/gitlab.cjs +2 -4
  119. package/dist/feeds/platform/handlers/gitlab.js +1 -3
  120. package/dist/feeds/platform/handlers/goodreads.cjs +39 -0
  121. package/dist/feeds/platform/handlers/goodreads.js +39 -0
  122. package/dist/feeds/platform/handlers/hashnode.cjs +16 -0
  123. package/dist/feeds/platform/handlers/hashnode.js +16 -0
  124. package/dist/feeds/platform/handlers/hatenablog.cjs +53 -0
  125. package/dist/feeds/platform/handlers/hatenablog.js +53 -0
  126. package/dist/feeds/platform/handlers/itchio.cjs +88 -0
  127. package/dist/feeds/platform/handlers/itchio.js +88 -0
  128. package/dist/feeds/platform/handlers/kickstarter.cjs +2 -4
  129. package/dist/feeds/platform/handlers/kickstarter.js +1 -3
  130. package/dist/feeds/platform/handlers/letterboxd.cjs +42 -0
  131. package/dist/feeds/platform/handlers/letterboxd.js +42 -0
  132. package/dist/feeds/platform/handlers/lobsters.cjs +3 -4
  133. package/dist/feeds/platform/handlers/lobsters.js +1 -3
  134. package/dist/feeds/platform/handlers/mastodon.cjs +40 -0
  135. package/dist/feeds/platform/handlers/mastodon.js +40 -0
  136. package/dist/feeds/platform/handlers/medium.cjs +2 -4
  137. package/dist/feeds/platform/handlers/medium.js +1 -3
  138. package/dist/feeds/platform/handlers/paragraph.cjs +21 -0
  139. package/dist/feeds/platform/handlers/paragraph.js +21 -0
  140. package/dist/feeds/platform/handlers/pinterest.cjs +2 -15
  141. package/dist/feeds/platform/handlers/pinterest.js +1 -14
  142. package/dist/feeds/platform/handlers/producthunt.cjs +2 -4
  143. package/dist/feeds/platform/handlers/producthunt.js +1 -3
  144. package/dist/feeds/platform/handlers/reddit.cjs +3 -4
  145. package/dist/feeds/platform/handlers/reddit.js +1 -3
  146. package/dist/feeds/platform/handlers/soundcloud.cjs +2 -4
  147. package/dist/feeds/platform/handlers/soundcloud.js +1 -3
  148. package/dist/feeds/platform/handlers/sourceforge.cjs +20 -0
  149. package/dist/feeds/platform/handlers/sourceforge.js +19 -0
  150. package/dist/feeds/platform/handlers/stackExchange.cjs +37 -0
  151. package/dist/feeds/platform/handlers/stackExchange.js +37 -0
  152. package/dist/feeds/platform/handlers/steam.cjs +26 -0
  153. package/dist/feeds/platform/handlers/steam.js +26 -0
  154. package/dist/feeds/platform/handlers/substack.cjs +2 -4
  155. package/dist/feeds/platform/handlers/substack.js +1 -3
  156. package/dist/feeds/platform/handlers/tumblr.cjs +5 -5
  157. package/dist/feeds/platform/handlers/tumblr.js +3 -4
  158. package/dist/feeds/platform/handlers/v2ex.cjs +33 -0
  159. package/dist/feeds/platform/handlers/v2ex.js +33 -0
  160. package/dist/feeds/platform/handlers/vimeo.cjs +68 -0
  161. package/dist/feeds/platform/handlers/vimeo.js +68 -0
  162. package/dist/feeds/platform/handlers/wordpress.cjs +5 -7
  163. package/dist/feeds/platform/handlers/wordpress.js +4 -6
  164. package/dist/feeds/platform/handlers/wpengine.cjs +4 -7
  165. package/dist/feeds/platform/handlers/wpengine.js +2 -4
  166. package/dist/feeds/platform/handlers/ximalaya.cjs +19 -0
  167. package/dist/feeds/platform/handlers/ximalaya.js +19 -0
  168. package/dist/feeds/platform/handlers/youtube.cjs +19 -8
  169. package/dist/feeds/platform/handlers/youtube.js +18 -7
  170. package/dist/feeds.cjs +4 -5
  171. package/dist/feeds.js +1 -2
  172. package/dist/hubs/discover/index.cjs +7 -9
  173. package/dist/hubs/discover/index.js +1 -3
  174. package/dist/hubs/discover/utils.cjs +1 -3
  175. package/dist/hubs/discover/utils.js +1 -2
  176. package/dist/hubs/feed/index.cjs +1 -3
  177. package/dist/hubs/feed/index.js +1 -3
  178. package/dist/hubs/headers/index.cjs +3 -5
  179. package/dist/hubs/headers/index.js +1 -3
  180. package/dist/hubs/html/index.cjs +3 -5
  181. package/dist/hubs/html/index.js +1 -3
  182. package/dist/hubs.js +1 -1
  183. package/dist/index.cjs +8 -7
  184. package/dist/index.d.cts +3 -2
  185. package/dist/index.d.ts +3 -2
  186. package/dist/index.js +2 -2
  187. package/dist/methods.cjs +13 -10
  188. package/dist/methods.d.cts +3 -1
  189. package/dist/methods.d.ts +3 -1
  190. package/dist/methods.js +3 -2
  191. package/dist/utils.cjs +3 -4
  192. package/dist/utils.js +1 -2
  193. package/package.json +15 -5
package/README.md CHANGED
@@ -18,31 +18,26 @@ Finds feeds by scanning links and anchors in HTML content, parsing HTTP headers,
18
18
 
19
19
  ### Supported Content
20
20
 
21
- - **Feeds** RSS, Atom, JSON Feed, and RDF. Each feed is validated and returns metadata like format, title, description, and site URL.
22
- - **Blogrolls** OPML files containing feed subscriptions. Validated and returns title.
23
- - **WebSub hubs** Find hubs for real-time feed update notifications.
21
+ | Type | Description |
22
+ | --- | --- |
23
+ | Feeds | RSS, Atom, JSON Feed, and RDF. Each feed is validated and returns metadata like format, title, description, and site URL. |
24
+ | Blogrolls | OPML files containing feed subscriptions. Validated and returns title. |
25
+ | WebSub hubs | Find hubs for real-time feed update notifications. |
24
26
 
25
27
  ### Discovery Methods
26
28
 
27
- - **Platform** Generates feed URLs for YouTube, GitHub, WordPress, and other popular platforms using URL pattern matching.
28
- - **HTML** Scans `<link>` elements with feed MIME types and `<a>` elements matching feed patterns or labels like "RSS", "Subscribe".
29
- - **Headers** Parses HTTP `Link` headers for `rel="alternate"` with feed MIME types per RFC 8288.
30
- - **Guess** Tests common paths (e.g. `/feed`, `/rss.xml`, `/atom.xml`) against the base URL as a fallback.
29
+ | Method | Description |
30
+ | --- | --- |
31
+ | Platform | Generates feed URLs for YouTube, GitHub, WordPress, and 30+ other popular platforms using URL pattern matching. |
32
+ | HTML | Scans `<link>` elements with feed MIME types and `<a>` elements matching feed patterns or labels like "RSS", "Subscribe". |
33
+ | Headers | Parses HTTP `Link` headers for `rel="alternate"` with feed MIME types per RFC 8288. |
34
+ | Guess | Tests common paths (e.g. `/feed`, `/rss.xml`, `/atom.xml`) against the base URL as a fallback. |
31
35
 
32
36
  ### Customization
33
37
 
34
- - **Custom extractors** Override the default parser to extract additional metadata from feeds and blogrolls.
35
- - **Configurable methods** — Enable/disable discovery methods or customize their options.
36
- - **Adapter system** — Use native fetch or easily integrate with Axios, Got, or Ky.
37
- - **Concurrency control** — Limit parallel requests during validation.
38
- - **Progress tracking** — Monitor discovery progress with callbacks.
39
- - **Type-safe** — Full TypeScript support with exported types.
40
- - **Tree-shakable** — Import only what you need.
41
-
38
+ Uses native fetch by default but supports custom adapters for Axios, Got, Ky, or any other HTTP client. Discovery methods can be individually enabled or disabled and their options adjusted. Custom extractors let you override the default parser to pull additional metadata from feeds and blogrolls.
42
39
  ## Quick Start
43
40
 
44
- This is a short guide on how to get you up and running with Feedscout.
45
-
46
41
  For a full overview of all the features, [visit the documentation](https://feedscout.dev).
47
42
 
48
43
  ### Installation
@@ -1,4 +1,3 @@
1
-
2
1
  //#region src/blogrolls/defaults.ts
3
2
  const mimeTypes = [
4
3
  "text/x-opml",
@@ -40,7 +39,6 @@ const defaultHtmlOptions = {
40
39
  };
41
40
  const defaultHeadersOptions = { linkSelectors };
42
41
  const defaultGuessOptions = { uris: urisBalanced };
43
-
44
42
  //#endregion
45
43
  exports.anchorLabels = anchorLabels;
46
44
  exports.defaultGuessOptions = defaultGuessOptions;
@@ -50,4 +48,4 @@ exports.linkSelectors = linkSelectors;
50
48
  exports.mimeTypes = mimeTypes;
51
49
  exports.urisBalanced = urisBalanced;
52
50
  exports.urisComprehensive = urisComprehensive;
53
- exports.urisMinimal = urisMinimal;
51
+ exports.urisMinimal = urisMinimal;
@@ -39,6 +39,5 @@ const defaultHtmlOptions = {
39
39
  };
40
40
  const defaultHeadersOptions = { linkSelectors };
41
41
  const defaultGuessOptions = { uris: urisBalanced };
42
-
43
42
  //#endregion
44
- export { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
43
+ export { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
@@ -1,5 +1,4 @@
1
1
  let feedsmith = require("feedsmith");
2
-
3
2
  //#region src/blogrolls/extractors.ts
4
3
  const defaultExtractor = async ({ content, url }) => {
5
4
  if (!content) return {
@@ -12,13 +11,11 @@ const defaultExtractor = async ({ content, url }) => {
12
11
  isValid: true,
13
12
  title: (0, feedsmith.parseOpml)(content).head?.title
14
13
  };
15
- } catch {
16
- return {
17
- url,
18
- isValid: false
19
- };
20
- }
14
+ } catch {}
15
+ return {
16
+ url,
17
+ isValid: false
18
+ };
21
19
  };
22
-
23
20
  //#endregion
24
- exports.defaultExtractor = defaultExtractor;
21
+ exports.defaultExtractor = defaultExtractor;
@@ -1,5 +1,4 @@
1
1
  import { parseOpml } from "feedsmith";
2
-
3
2
  //#region src/blogrolls/extractors.ts
4
3
  const defaultExtractor = async ({ content, url }) => {
5
4
  if (!content) return {
@@ -12,13 +11,11 @@ const defaultExtractor = async ({ content, url }) => {
12
11
  isValid: true,
13
12
  title: parseOpml(content).head?.title
14
13
  };
15
- } catch {
16
- return {
17
- url,
18
- isValid: false
19
- };
20
- }
14
+ } catch {}
15
+ return {
16
+ url,
17
+ isValid: false
18
+ };
21
19
  };
22
-
23
20
  //#endregion
24
- export { defaultExtractor };
21
+ export { defaultExtractor };
@@ -1,9 +1,8 @@
1
- const require_utils = require('../common/utils.cjs');
2
- const require_utils$1 = require('../common/discover/utils.cjs');
3
- const require_index = require('../common/discover/index.cjs');
4
- const require_defaults = require('./defaults.cjs');
5
- const require_extractors = require('./extractors.cjs');
6
-
1
+ const require_utils = require("../common/utils.cjs");
2
+ const require_utils$1 = require("../common/discover/utils.cjs");
3
+ const require_index = require("../common/discover/index.cjs");
4
+ const require_defaults = require("./defaults.cjs");
5
+ const require_extractors = require("./extractors.cjs");
7
6
  //#region src/blogrolls/index.ts
8
7
  const discoverBlogrolls = async (input, options = {}) => {
9
8
  return require_index.discover(input, {
@@ -17,12 +16,10 @@ const discoverBlogrolls = async (input, options = {}) => {
17
16
  extractFn: options.extractFn ?? require_extractors.defaultExtractor,
18
17
  normalizeUrlFn: options.normalizeUrlFn ?? require_utils.normalizeUrl
19
18
  }, {
20
- platform: { handlers: [] },
21
19
  html: require_defaults.defaultHtmlOptions,
22
20
  headers: require_defaults.defaultHeadersOptions,
23
21
  guess: require_defaults.defaultGuessOptions
24
22
  });
25
23
  };
26
-
27
24
  //#endregion
28
- exports.discoverBlogrolls = discoverBlogrolls;
25
+ exports.discoverBlogrolls = discoverBlogrolls;
@@ -2,6 +2,6 @@ import { DiscoverInput, DiscoverOptions, DiscoverResult } from "../common/types.
2
2
  import { BlogrollResult } from "./types.cjs";
3
3
 
4
4
  //#region src/blogrolls/index.d.ts
5
- declare const discoverBlogrolls: <TValid extends BlogrollResult = BlogrollResult>(input: DiscoverInput, options?: DiscoverOptions<TValid>) => Promise<Array<DiscoverResult<TValid>>>;
5
+ declare const discoverBlogrolls: <TValid extends BlogrollResult = BlogrollResult>(input: DiscoverInput, options?: DiscoverOptions<TValid, "html" | "headers" | "guess">) => Promise<Array<DiscoverResult<TValid>>>;
6
6
  //#endregion
7
7
  export { discoverBlogrolls };
@@ -2,6 +2,6 @@ import { DiscoverInput, DiscoverOptions, DiscoverResult } from "../common/types.
2
2
  import { BlogrollResult } from "./types.js";
3
3
 
4
4
  //#region src/blogrolls/index.d.ts
5
- declare const discoverBlogrolls: <TValid extends BlogrollResult = BlogrollResult>(input: DiscoverInput, options?: DiscoverOptions<TValid>) => Promise<Array<DiscoverResult<TValid>>>;
5
+ declare const discoverBlogrolls: <TValid extends BlogrollResult = BlogrollResult>(input: DiscoverInput, options?: DiscoverOptions<TValid, "html" | "headers" | "guess">) => Promise<Array<DiscoverResult<TValid>>>;
6
6
  //#endregion
7
7
  export { discoverBlogrolls };
@@ -3,7 +3,6 @@ import { defaultFetchFn } from "../common/discover/utils.js";
3
3
  import { discover } from "../common/discover/index.js";
4
4
  import { defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions } from "./defaults.js";
5
5
  import { defaultExtractor } from "./extractors.js";
6
-
7
6
  //#region src/blogrolls/index.ts
8
7
  const discoverBlogrolls = async (input, options = {}) => {
9
8
  return discover(input, {
@@ -17,12 +16,10 @@ const discoverBlogrolls = async (input, options = {}) => {
17
16
  extractFn: options.extractFn ?? defaultExtractor,
18
17
  normalizeUrlFn: options.normalizeUrlFn ?? normalizeUrl
19
18
  }, {
20
- platform: { handlers: [] },
21
19
  html: defaultHtmlOptions,
22
20
  headers: defaultHeadersOptions,
23
21
  guess: defaultGuessOptions
24
22
  });
25
23
  };
26
-
27
24
  //#endregion
28
- export { discoverBlogrolls };
25
+ export { discoverBlogrolls };
@@ -1,7 +1,6 @@
1
- Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });
2
- const require_defaults = require('./blogrolls/defaults.cjs');
3
- const require_extractors = require('./blogrolls/extractors.cjs');
4
-
1
+ Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
2
+ const require_defaults = require("./blogrolls/defaults.cjs");
3
+ const require_extractors = require("./blogrolls/extractors.cjs");
5
4
  exports.anchorLabels = require_defaults.anchorLabels;
6
5
  exports.defaultExtractor = require_extractors.defaultExtractor;
7
6
  exports.defaultGuessOptions = require_defaults.defaultGuessOptions;
@@ -11,4 +10,4 @@ exports.linkSelectors = require_defaults.linkSelectors;
11
10
  exports.mimeTypes = require_defaults.mimeTypes;
12
11
  exports.urisBalanced = require_defaults.urisBalanced;
13
12
  exports.urisComprehensive = require_defaults.urisComprehensive;
14
- exports.urisMinimal = require_defaults.urisMinimal;
13
+ exports.urisMinimal = require_defaults.urisMinimal;
package/dist/blogrolls.js CHANGED
@@ -1,4 +1,3 @@
1
1
  import { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal } from "./blogrolls/defaults.js";
2
2
  import { defaultExtractor } from "./blogrolls/extractors.js";
3
-
4
- export { anchorLabels, defaultExtractor, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
3
+ export { anchorLabels, defaultExtractor, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
@@ -1,8 +1,7 @@
1
- const require_types = require('../types.cjs');
2
- const require_utils = require('../utils.cjs');
3
- const require_index = require('../uris/index.cjs');
4
- const require_utils$1 = require('./utils.cjs');
5
-
1
+ const require_types = require("../types.cjs");
2
+ const require_utils = require("../utils.cjs");
3
+ const require_index = require("../uris/index.cjs");
4
+ const require_utils$1 = require("./utils.cjs");
6
5
  //#region src/common/discover/index.ts
7
6
  const discover = async (input, options, defaults) => {
8
7
  const { methods, fetchFn, extractFn, normalizeUrlFn, stopOnFirstMethod = false, stopOnFirstResult = false, concurrency = 3, includeInvalid = false, onProgress } = options;
@@ -10,11 +9,12 @@ const discover = async (input, options, defaults) => {
10
9
  if (normalizedInput.content) {
11
10
  const result = await extractFn({
12
11
  url: normalizedInput.url,
13
- content: normalizedInput.content
12
+ content: normalizedInput.content,
13
+ headers: normalizedInput.headers
14
14
  });
15
15
  if (result.isValid) return [result];
16
16
  }
17
- const urisByMethod = require_index.discoverUris(require_utils$1.normalizeMethodsConfig(normalizedInput, methods, defaults));
17
+ const urisByMethod = await require_index.discoverUris(require_utils$1.normalizeMethodsConfig(normalizedInput, methods, defaults), fetchFn);
18
18
  const seen = /* @__PURE__ */ new Set();
19
19
  const methodGroups = [];
20
20
  for (const method of require_types.discoverMethodOrder) {
@@ -28,9 +28,12 @@ const discover = async (input, options, defaults) => {
28
28
  seen.add(key);
29
29
  return true;
30
30
  });
31
- if (unique.length > 0) methodGroups.push(unique);
31
+ if (unique.length > 0) methodGroups.push({
32
+ method,
33
+ entries: unique
34
+ });
32
35
  }
33
- const total = methodGroups.reduce((sum, group) => sum + group.length, 0);
36
+ const total = methodGroups.reduce((sum, group) => sum + group.entries.length, 0);
34
37
  const results = [];
35
38
  let tested = 0;
36
39
  let found = 0;
@@ -39,7 +42,9 @@ const discover = async (input, options, defaults) => {
39
42
  const fetchResult = await fetchFn(url);
40
43
  return await extractFn({
41
44
  url: fetchResult.url,
42
- content: typeof fetchResult.body === "string" ? fetchResult.body : ""
45
+ content: typeof fetchResult.body === "string" ? fetchResult.body : "",
46
+ headers: fetchResult.headers,
47
+ status: fetchResult.status
43
48
  });
44
49
  } catch (error) {
45
50
  return {
@@ -49,14 +54,18 @@ const discover = async (input, options, defaults) => {
49
54
  };
50
55
  }
51
56
  };
52
- const processUri = async (entry) => {
57
+ const processUri = async (entry, method) => {
53
58
  const alternatives = typeof entry.uri === "string" ? [entry.uri] : entry.uri;
54
59
  for (const url of alternatives) {
55
60
  const result = await fetchAndExtract(url);
56
61
  results.push(entry.hint ? {
57
62
  ...result,
63
+ method,
58
64
  hint: entry.hint
59
- } : result);
65
+ } : {
66
+ ...result,
67
+ method
68
+ });
60
69
  tested += 1;
61
70
  if (result.isValid) found += 1;
62
71
  onProgress?.({
@@ -68,9 +77,9 @@ const discover = async (input, options, defaults) => {
68
77
  if (result.isValid) break;
69
78
  }
70
79
  };
71
- for (const group of methodGroups) {
80
+ for (const { method, entries } of methodGroups) {
72
81
  const foundBefore = found;
73
- await require_utils.processConcurrently(group, processUri, {
82
+ await require_utils.processConcurrently(entries, (entry) => processUri(entry, method), {
74
83
  concurrency,
75
84
  shouldStop: () => {
76
85
  return stopOnFirstResult && found > 0;
@@ -80,6 +89,5 @@ const discover = async (input, options, defaults) => {
80
89
  }
81
90
  return includeInvalid ? results : results.filter((result) => result.isValid);
82
91
  };
83
-
84
92
  //#endregion
85
- exports.discover = discover;
93
+ exports.discover = discover;
@@ -2,7 +2,6 @@ import { discoverMethodOrder } from "../types.js";
2
2
  import { processConcurrently } from "../utils.js";
3
3
  import { discoverUris } from "../uris/index.js";
4
4
  import { normalizeInput, normalizeMethodsConfig, normalizeUriEntry } from "./utils.js";
5
-
6
5
  //#region src/common/discover/index.ts
7
6
  const discover = async (input, options, defaults) => {
8
7
  const { methods, fetchFn, extractFn, normalizeUrlFn, stopOnFirstMethod = false, stopOnFirstResult = false, concurrency = 3, includeInvalid = false, onProgress } = options;
@@ -10,11 +9,12 @@ const discover = async (input, options, defaults) => {
10
9
  if (normalizedInput.content) {
11
10
  const result = await extractFn({
12
11
  url: normalizedInput.url,
13
- content: normalizedInput.content
12
+ content: normalizedInput.content,
13
+ headers: normalizedInput.headers
14
14
  });
15
15
  if (result.isValid) return [result];
16
16
  }
17
- const urisByMethod = discoverUris(normalizeMethodsConfig(normalizedInput, methods, defaults));
17
+ const urisByMethod = await discoverUris(normalizeMethodsConfig(normalizedInput, methods, defaults), fetchFn);
18
18
  const seen = /* @__PURE__ */ new Set();
19
19
  const methodGroups = [];
20
20
  for (const method of discoverMethodOrder) {
@@ -28,9 +28,12 @@ const discover = async (input, options, defaults) => {
28
28
  seen.add(key);
29
29
  return true;
30
30
  });
31
- if (unique.length > 0) methodGroups.push(unique);
31
+ if (unique.length > 0) methodGroups.push({
32
+ method,
33
+ entries: unique
34
+ });
32
35
  }
33
- const total = methodGroups.reduce((sum, group) => sum + group.length, 0);
36
+ const total = methodGroups.reduce((sum, group) => sum + group.entries.length, 0);
34
37
  const results = [];
35
38
  let tested = 0;
36
39
  let found = 0;
@@ -39,7 +42,9 @@ const discover = async (input, options, defaults) => {
39
42
  const fetchResult = await fetchFn(url);
40
43
  return await extractFn({
41
44
  url: fetchResult.url,
42
- content: typeof fetchResult.body === "string" ? fetchResult.body : ""
45
+ content: typeof fetchResult.body === "string" ? fetchResult.body : "",
46
+ headers: fetchResult.headers,
47
+ status: fetchResult.status
43
48
  });
44
49
  } catch (error) {
45
50
  return {
@@ -49,14 +54,18 @@ const discover = async (input, options, defaults) => {
49
54
  };
50
55
  }
51
56
  };
52
- const processUri = async (entry) => {
57
+ const processUri = async (entry, method) => {
53
58
  const alternatives = typeof entry.uri === "string" ? [entry.uri] : entry.uri;
54
59
  for (const url of alternatives) {
55
60
  const result = await fetchAndExtract(url);
56
61
  results.push(entry.hint ? {
57
62
  ...result,
63
+ method,
58
64
  hint: entry.hint
59
- } : result);
65
+ } : {
66
+ ...result,
67
+ method
68
+ });
60
69
  tested += 1;
61
70
  if (result.isValid) found += 1;
62
71
  onProgress?.({
@@ -68,9 +77,9 @@ const discover = async (input, options, defaults) => {
68
77
  if (result.isValid) break;
69
78
  }
70
79
  };
71
- for (const group of methodGroups) {
80
+ for (const { method, entries } of methodGroups) {
72
81
  const foundBefore = found;
73
- await processConcurrently(group, processUri, {
82
+ await processConcurrently(entries, (entry) => processUri(entry, method), {
74
83
  concurrency,
75
84
  shouldStop: () => {
76
85
  return stopOnFirstResult && found > 0;
@@ -80,6 +89,5 @@ const discover = async (input, options, defaults) => {
80
89
  }
81
90
  return includeInvalid ? results : results.filter((result) => result.isValid);
82
91
  };
83
-
84
92
  //#endregion
85
- export { discover };
93
+ export { discover };
@@ -1,5 +1,4 @@
1
- const require_locales = require('../locales.cjs');
2
-
1
+ const require_locales = require("../locales.cjs");
3
2
  //#region src/common/discover/utils.ts
4
3
  const defaultFetchFn = async (url, options) => {
5
4
  const response = await fetch(url, {
@@ -36,11 +35,12 @@ const normalizeUriEntry = (entry, normalizeUrlFn, baseUrl) => {
36
35
  const normalizeMethodsConfig = (input, methods, defaults) => {
37
36
  const methodsObj = Array.isArray(methods) ? Object.fromEntries(methods.map((method) => [method, true])) : methods;
38
37
  const methodsConfig = {};
39
- if (methodsObj.platform) {
38
+ if (methodsObj.platform && defaults.platform) {
40
39
  if (!input.url || input.url === "") throw new Error(require_locales.errors.platformMethodRequiresUrl);
41
40
  const platformOptions = methodsObj.platform === true ? {} : methodsObj.platform;
42
41
  methodsConfig.platform = {
43
- html: input.content ?? "",
42
+ content: input.content,
43
+ headers: input.headers,
44
44
  options: {
45
45
  ...defaults.platform,
46
46
  ...platformOptions,
@@ -48,7 +48,18 @@ const normalizeMethodsConfig = (input, methods, defaults) => {
48
48
  }
49
49
  };
50
50
  }
51
- if (methodsObj.html) {
51
+ if (methodsObj.feed && defaults.feed) {
52
+ if (input.content === void 0) throw new Error(require_locales.errors.feedMethodRequiresContent);
53
+ const feedOptions = methodsObj.feed === true ? {} : methodsObj.feed;
54
+ methodsConfig.feed = {
55
+ content: input.content,
56
+ options: {
57
+ ...defaults.feed,
58
+ ...feedOptions
59
+ }
60
+ };
61
+ }
62
+ if (methodsObj.html && defaults.html) {
52
63
  if (input.content === void 0) throw new Error(require_locales.errors.htmlMethodRequiresContent);
53
64
  const htmlOptions = methodsObj.html === true ? {} : methodsObj.html;
54
65
  methodsConfig.html = {
@@ -60,7 +71,7 @@ const normalizeMethodsConfig = (input, methods, defaults) => {
60
71
  }
61
72
  };
62
73
  }
63
- if (methodsObj.headers) {
74
+ if (methodsObj.headers && defaults.headers) {
64
75
  if (input.headers === void 0) throw new Error(require_locales.errors.headersMethodRequiresHeaders);
65
76
  const headersOptions = methodsObj.headers === true ? {} : methodsObj.headers;
66
77
  methodsConfig.headers = {
@@ -72,7 +83,7 @@ const normalizeMethodsConfig = (input, methods, defaults) => {
72
83
  }
73
84
  };
74
85
  }
75
- if (methodsObj.guess) {
86
+ if (methodsObj.guess && defaults.guess) {
76
87
  if (!input.url || input.url === "") throw new Error(require_locales.errors.guessMethodRequiresUrl);
77
88
  const guessOptions = methodsObj.guess === true ? {} : methodsObj.guess;
78
89
  methodsConfig.guess = { options: {
@@ -83,9 +94,8 @@ const normalizeMethodsConfig = (input, methods, defaults) => {
83
94
  }
84
95
  return methodsConfig;
85
96
  };
86
-
87
97
  //#endregion
88
98
  exports.defaultFetchFn = defaultFetchFn;
89
99
  exports.normalizeInput = normalizeInput;
90
100
  exports.normalizeMethodsConfig = normalizeMethodsConfig;
91
- exports.normalizeUriEntry = normalizeUriEntry;
101
+ exports.normalizeUriEntry = normalizeUriEntry;
@@ -1,5 +1,4 @@
1
1
  import { errors } from "../locales.js";
2
-
3
2
  //#region src/common/discover/utils.ts
4
3
  const defaultFetchFn = async (url, options) => {
5
4
  const response = await fetch(url, {
@@ -36,11 +35,12 @@ const normalizeUriEntry = (entry, normalizeUrlFn, baseUrl) => {
36
35
  const normalizeMethodsConfig = (input, methods, defaults) => {
37
36
  const methodsObj = Array.isArray(methods) ? Object.fromEntries(methods.map((method) => [method, true])) : methods;
38
37
  const methodsConfig = {};
39
- if (methodsObj.platform) {
38
+ if (methodsObj.platform && defaults.platform) {
40
39
  if (!input.url || input.url === "") throw new Error(errors.platformMethodRequiresUrl);
41
40
  const platformOptions = methodsObj.platform === true ? {} : methodsObj.platform;
42
41
  methodsConfig.platform = {
43
- html: input.content ?? "",
42
+ content: input.content,
43
+ headers: input.headers,
44
44
  options: {
45
45
  ...defaults.platform,
46
46
  ...platformOptions,
@@ -48,7 +48,18 @@ const normalizeMethodsConfig = (input, methods, defaults) => {
48
48
  }
49
49
  };
50
50
  }
51
- if (methodsObj.html) {
51
+ if (methodsObj.feed && defaults.feed) {
52
+ if (input.content === void 0) throw new Error(errors.feedMethodRequiresContent);
53
+ const feedOptions = methodsObj.feed === true ? {} : methodsObj.feed;
54
+ methodsConfig.feed = {
55
+ content: input.content,
56
+ options: {
57
+ ...defaults.feed,
58
+ ...feedOptions
59
+ }
60
+ };
61
+ }
62
+ if (methodsObj.html && defaults.html) {
52
63
  if (input.content === void 0) throw new Error(errors.htmlMethodRequiresContent);
53
64
  const htmlOptions = methodsObj.html === true ? {} : methodsObj.html;
54
65
  methodsConfig.html = {
@@ -60,7 +71,7 @@ const normalizeMethodsConfig = (input, methods, defaults) => {
60
71
  }
61
72
  };
62
73
  }
63
- if (methodsObj.headers) {
74
+ if (methodsObj.headers && defaults.headers) {
64
75
  if (input.headers === void 0) throw new Error(errors.headersMethodRequiresHeaders);
65
76
  const headersOptions = methodsObj.headers === true ? {} : methodsObj.headers;
66
77
  methodsConfig.headers = {
@@ -72,7 +83,7 @@ const normalizeMethodsConfig = (input, methods, defaults) => {
72
83
  }
73
84
  };
74
85
  }
75
- if (methodsObj.guess) {
86
+ if (methodsObj.guess && defaults.guess) {
76
87
  if (!input.url || input.url === "") throw new Error(errors.guessMethodRequiresUrl);
77
88
  const guessOptions = methodsObj.guess === true ? {} : methodsObj.guess;
78
89
  methodsConfig.guess = { options: {
@@ -83,6 +94,5 @@ const normalizeMethodsConfig = (input, methods, defaults) => {
83
94
  }
84
95
  return methodsConfig;
85
96
  };
86
-
87
97
  //#endregion
88
- export { defaultFetchFn, normalizeInput, normalizeMethodsConfig, normalizeUriEntry };
98
+ export { defaultFetchFn, normalizeInput, normalizeMethodsConfig, normalizeUriEntry };
@@ -1,7 +1,7 @@
1
-
2
1
  //#region src/common/locales.json
3
2
  var errors = {
4
3
  "platformMethodRequiresUrl": "Platform method requires url to be provided in input",
4
+ "feedMethodRequiresContent": "Feed method requires content to be provided in input",
5
5
  "htmlMethodRequiresContent": "HTML method requires content to be provided in input",
6
6
  "headersMethodRequiresHeaders": "Headers method requires headers to be provided in input",
7
7
  "guessMethodRequiresUrl": "Guess method requires url to be provided in input"
@@ -10,6 +10,7 @@ var hints = {
10
10
  "youtube:all": "All uploads",
11
11
  "youtube:videos": "Videos only",
12
12
  "youtube:shorts": "Shorts only",
13
+ "youtube:live": "Live streams only",
13
14
  "youtube:playlist": "Playlist",
14
15
  "github:activity": "Activity",
15
16
  "github:releases": "Releases",
@@ -26,6 +27,11 @@ var hints = {
26
27
  "gitlab:activity": "Activity",
27
28
  "gitlab:releases": "Releases",
28
29
  "gitlab:tags": "Tags",
30
+ "codeberg:activity": "Activity",
31
+ "codeberg:releases": "Releases",
32
+ "codeberg:tags": "Tags",
33
+ "codeberg:branch-commits": "Branch commits",
34
+ "codeberg:file-history": "File history",
29
35
  "wordpress:category": "Category",
30
36
  "wordpress:tag": "Tag",
31
37
  "wordpress:author": "Author",
@@ -55,37 +61,75 @@ var hints = {
55
61
  "github-gist:starred": "Starred",
56
62
  "kickstarter:updates": "Updates",
57
63
  "kickstarter:projects": "Projects",
64
+ "letterboxd:diary": "Diary",
58
65
  "lobsters:tag": "Tag",
59
66
  "lobsters:domain": "Domain",
60
67
  "lobsters:stories": "Stories",
61
68
  "lobsters:top": "Top stories",
62
69
  "lobsters:newest": "Newest",
63
70
  "lobsters:comments": "Comments",
71
+ "mastodon:posts": "Posts",
72
+ "mastodon:tag": "Tag",
64
73
  "medium:posts": "Posts",
65
74
  "medium:tag": "Tag",
66
75
  "medium:tagged": "Tagged",
67
76
  "medium:publication": "Publication",
68
77
  "pinterest:pins": "Pins",
69
- "pinterest:board": "Board",
70
78
  "producthunt:topic": "Topic",
71
79
  "producthunt:category": "Category",
72
80
  "producthunt:products": "Products",
73
81
  "soundcloud:tracks": "Tracks",
82
+ "stackexchange:tag": "Tag",
83
+ "stackexchange:question": "Question",
84
+ "stackexchange:user": "User",
85
+ "steam:news": "News",
86
+ "steam:group": "Group",
74
87
  "substack:newsletter": "Newsletter",
75
88
  "tumblr:posts": "Posts",
76
- "tumblr:tag": "Tag"
89
+ "tumblr:tag": "Tag",
90
+ "goodreads:updates": "Updates",
91
+ "goodreads:reviews": "Reviews",
92
+ "sourceforge:activity": "Activity",
93
+ "vimeo:videos": "Videos",
94
+ "vimeo:likes": "Likes",
95
+ "vimeo:channel": "Channel",
96
+ "vimeo:group": "Group",
97
+ "hashnode:blog": "Blog",
98
+ "paragraph:blog": "Blog",
99
+ "hatenablog:posts-rss": "Posts (RSS)",
100
+ "hatenablog:posts-atom": "Posts (Atom)",
101
+ "hatenablog:category-rss": "Category (RSS)",
102
+ "hatenablog:category-atom": "Category (Atom)",
103
+ "hatenablog:author-rss": "Author (RSS)",
104
+ "hatenablog:author-atom": "Author (Atom)",
105
+ "itchio:devlog": "Devlog",
106
+ "itchio:games": "Games",
107
+ "itchio:tag": "Tag",
108
+ "itchio:section": "Section",
109
+ "itchio:featured": "Featured",
110
+ "itchio:new": "New",
111
+ "itchio:sales": "Sales",
112
+ "csdn:blog": "Blog",
113
+ "douban:interests": "Interests",
114
+ "douban:reviews": "Reviews",
115
+ "douban:notes": "Notes",
116
+ "douban:subjectReviews": "Subject reviews",
117
+ "v2ex:index": "Index",
118
+ "v2ex:node": "Node",
119
+ "v2ex:member": "Member",
120
+ "v2ex:tab": "Tab",
121
+ "ximalaya:album": "Album"
77
122
  };
78
-
79
123
  //#endregion
80
- Object.defineProperty(exports, 'errors', {
81
- enumerable: true,
82
- get: function () {
83
- return errors;
84
- }
124
+ Object.defineProperty(exports, "errors", {
125
+ enumerable: true,
126
+ get: function() {
127
+ return errors;
128
+ }
129
+ });
130
+ Object.defineProperty(exports, "hints", {
131
+ enumerable: true,
132
+ get: function() {
133
+ return hints;
134
+ }
85
135
  });
86
- Object.defineProperty(exports, 'hints', {
87
- enumerable: true,
88
- get: function () {
89
- return hints;
90
- }
91
- });