feedscout 1.8.0 → 2.0.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (236) hide show
  1. package/README.md +1 -1
  2. package/dist/blogrolls/extractors.d.ts +2 -2
  3. package/dist/blogrolls/extractors.js +2 -2
  4. package/dist/blogrolls/index.js +4 -5
  5. package/dist/blogrolls.d.ts +2 -2
  6. package/dist/blogrolls.js +2 -2
  7. package/dist/common/discover/defaults.d.ts +6 -0
  8. package/dist/common/discover/defaults.js +44 -0
  9. package/dist/common/discover/index.js +2 -2
  10. package/dist/common/discover/utils.d.ts +3 -6
  11. package/dist/common/discover/utils.js +1 -28
  12. package/dist/common/locales.js +214 -10
  13. package/dist/common/types.d.ts +3 -2
  14. package/dist/common/uris/feed/types.d.ts +2 -2
  15. package/dist/common/uris/html/types.d.ts +4 -4
  16. package/dist/common/utils.d.ts +6 -4
  17. package/dist/common/utils.js +13 -11
  18. package/dist/favicons/extractors.d.ts +2 -2
  19. package/dist/favicons/extractors.js +2 -2
  20. package/dist/favicons/index.js +4 -5
  21. package/dist/favicons.d.ts +2 -2
  22. package/dist/favicons.js +2 -2
  23. package/dist/feeds/defaults.d.ts +2 -1
  24. package/dist/feeds/defaults.js +116 -9
  25. package/dist/feeds/extractors.d.ts +2 -2
  26. package/dist/feeds/extractors.js +6 -6
  27. package/dist/feeds/index.js +4 -5
  28. package/dist/feeds/platform/handlers/acast.js +26 -0
  29. package/dist/feeds/platform/handlers/ameblo.js +36 -0
  30. package/dist/feeds/platform/handlers/applePodcasts.js +1 -1
  31. package/dist/feeds/platform/handlers/arena.js +42 -0
  32. package/dist/feeds/platform/handlers/artstation.js +52 -0
  33. package/dist/feeds/platform/handlers/audioboom.js +23 -0
  34. package/dist/feeds/platform/handlers/bearblog.js +45 -0
  35. package/dist/feeds/platform/handlers/behance.js +7 -0
  36. package/dist/feeds/platform/handlers/blogspot.js +38 -2
  37. package/dist/feeds/platform/handlers/bookwyrm.js +48 -0
  38. package/dist/feeds/platform/handlers/buttondown.js +43 -0
  39. package/dist/feeds/platform/handlers/buzzsprout.js +22 -0
  40. package/dist/feeds/platform/handlers/codeberg.js +5 -5
  41. package/dist/feeds/platform/handlers/dailymotion.js +16 -0
  42. package/dist/feeds/platform/handlers/deviantart.js +25 -6
  43. package/dist/feeds/platform/handlers/devto.js +8 -0
  44. package/dist/feeds/platform/handlers/discourse.js +70 -0
  45. package/dist/feeds/platform/handlers/dreamwidth.js +48 -0
  46. package/dist/feeds/platform/handlers/exblog.js +35 -0
  47. package/dist/feeds/platform/handlers/fireside.js +24 -0
  48. package/dist/feeds/platform/handlers/friendica.js +44 -0
  49. package/dist/feeds/platform/handlers/ghost.js +30 -0
  50. package/dist/feeds/platform/handlers/github.js +6 -0
  51. package/dist/feeds/platform/handlers/githubGist.js +16 -5
  52. package/dist/feeds/platform/handlers/gitlab.js +31 -13
  53. package/dist/feeds/platform/handlers/goodreads.js +18 -8
  54. package/dist/feeds/platform/handlers/hackernews.js +21 -0
  55. package/dist/feeds/platform/handlers/hashnode.js +1 -1
  56. package/dist/feeds/platform/handlers/hatenablog.js +4 -1
  57. package/dist/feeds/platform/handlers/hearthis.js +32 -0
  58. package/dist/feeds/platform/handlers/heyWorld.js +18 -0
  59. package/dist/feeds/platform/handlers/insanejournal.js +69 -0
  60. package/dist/feeds/platform/handlers/itchio.js +24 -1
  61. package/dist/feeds/platform/handlers/lemmy.js +46 -4
  62. package/dist/feeds/platform/handlers/letterboxd.js +4 -0
  63. package/dist/feeds/platform/handlers/libsyn.js +25 -0
  64. package/dist/feeds/platform/handlers/listed.js +20 -0
  65. package/dist/feeds/platform/handlers/livejournal.js +68 -0
  66. package/dist/feeds/platform/handlers/mastodon.js +32 -0
  67. package/dist/feeds/platform/handlers/mataroa.js +16 -0
  68. package/dist/feeds/platform/handlers/medium.js +2 -2
  69. package/dist/feeds/platform/handlers/microblog.js +55 -0
  70. package/dist/feeds/platform/handlers/misskey.js +40 -0
  71. package/dist/feeds/platform/handlers/myanimelist.js +43 -0
  72. package/dist/feeds/platform/handlers/naverBlog.js +21 -0
  73. package/dist/feeds/platform/handlers/nebula.js +68 -0
  74. package/dist/feeds/platform/handlers/note.js +53 -0
  75. package/dist/feeds/platform/handlers/observable.js +34 -0
  76. package/dist/feeds/platform/handlers/odysee.js +20 -0
  77. package/dist/feeds/platform/handlers/pagecord.js +16 -0
  78. package/dist/feeds/platform/handlers/paragraph.js +1 -2
  79. package/dist/feeds/platform/handlers/pika.js +35 -0
  80. package/dist/feeds/platform/handlers/pinterest.js +13 -0
  81. package/dist/feeds/platform/handlers/pixelfed.js +46 -0
  82. package/dist/feeds/platform/handlers/pleroma.js +34 -0
  83. package/dist/feeds/platform/handlers/podbean.js +29 -0
  84. package/dist/feeds/platform/handlers/podigee.js +29 -0
  85. package/dist/feeds/platform/handlers/posthaven.js +24 -0
  86. package/dist/feeds/platform/handlers/prose.js +26 -0
  87. package/dist/feeds/platform/handlers/qiita.js +58 -0
  88. package/dist/feeds/platform/handlers/reddit.js +83 -9
  89. package/dist/feeds/platform/handlers/rssCom.js +20 -0
  90. package/dist/feeds/platform/handlers/seesaa.js +22 -0
  91. package/dist/feeds/platform/handlers/sourceforge.js +37 -4
  92. package/dist/feeds/platform/handlers/spreaker.js +21 -0
  93. package/dist/feeds/platform/handlers/stackExchange.js +26 -2
  94. package/dist/feeds/platform/handlers/steam.js +7 -0
  95. package/dist/feeds/platform/handlers/tildes.js +41 -0
  96. package/dist/feeds/platform/handlers/tistory.js +16 -0
  97. package/dist/feeds/platform/handlers/transistor.js +29 -0
  98. package/dist/feeds/platform/handlers/velog.js +24 -0
  99. package/dist/feeds/platform/handlers/vimeo.js +6 -0
  100. package/dist/feeds/platform/handlers/weblogLol.js +26 -0
  101. package/dist/feeds/platform/handlers/weebly.js +25 -0
  102. package/dist/feeds/platform/handlers/wordpress.js +173 -28
  103. package/dist/feeds/platform/handlers/writeas.js +51 -0
  104. package/dist/feeds/platform/handlers/ximalaya.js +1 -1
  105. package/dist/feeds/platform/handlers/youtube.js +4 -1
  106. package/dist/feeds/platform/handlers/zenn.js +54 -0
  107. package/dist/feeds.d.ts +3 -3
  108. package/dist/feeds.js +3 -3
  109. package/dist/hubs/discover/index.js +3 -3
  110. package/dist/hubs/feed/index.js +2 -2
  111. package/dist/hubs/headers/index.js +2 -2
  112. package/dist/hubs/html/index.js +2 -2
  113. package/dist/index.d.ts +2 -1
  114. package/dist/index.js +2 -1
  115. package/dist/utils.d.ts +2 -1
  116. package/package.json +24 -64
  117. package/dist/blogrolls/defaults.cjs +0 -51
  118. package/dist/blogrolls/defaults.d.cts +0 -17
  119. package/dist/blogrolls/extractors.cjs +0 -21
  120. package/dist/blogrolls/extractors.d.cts +0 -7
  121. package/dist/blogrolls/index.cjs +0 -26
  122. package/dist/blogrolls/index.d.cts +0 -7
  123. package/dist/blogrolls/types.d.cts +0 -6
  124. package/dist/blogrolls.cjs +0 -13
  125. package/dist/blogrolls.d.cts +0 -4
  126. package/dist/common/discover/index.cjs +0 -105
  127. package/dist/common/discover/utils.cjs +0 -130
  128. package/dist/common/discover/utils.d.cts +0 -8
  129. package/dist/common/locales.cjs +0 -144
  130. package/dist/common/types.cjs +0 -10
  131. package/dist/common/types.d.cts +0 -89
  132. package/dist/common/uris/feed/index.cjs +0 -12
  133. package/dist/common/uris/feed/index.d.cts +0 -6
  134. package/dist/common/uris/feed/types.d.cts +0 -9
  135. package/dist/common/uris/guess/index.cjs +0 -8
  136. package/dist/common/uris/guess/index.d.cts +0 -7
  137. package/dist/common/uris/guess/types.d.cts +0 -10
  138. package/dist/common/uris/guess/utils.cjs +0 -42
  139. package/dist/common/uris/guess/utils.d.cts +0 -8
  140. package/dist/common/uris/headers/index.cjs +0 -26
  141. package/dist/common/uris/headers/index.d.cts +0 -6
  142. package/dist/common/uris/headers/types.d.cts +0 -9
  143. package/dist/common/uris/html/handlers.cjs +0 -45
  144. package/dist/common/uris/html/index.cjs +0 -19
  145. package/dist/common/uris/html/index.d.cts +0 -6
  146. package/dist/common/uris/html/types.d.cts +0 -12
  147. package/dist/common/uris/index.cjs +0 -32
  148. package/dist/common/uris/platform/index.cjs +0 -10
  149. package/dist/common/uris/platform/index.d.cts +0 -7
  150. package/dist/common/uris/platform/types.d.cts +0 -13
  151. package/dist/common/utils.cjs +0 -97
  152. package/dist/common/utils.d.cts +0 -10
  153. package/dist/favicons/defaults.cjs +0 -65
  154. package/dist/favicons/defaults.d.cts +0 -18
  155. package/dist/favicons/extractors.cjs +0 -25
  156. package/dist/favicons/extractors.d.cts +0 -7
  157. package/dist/favicons/index.cjs +0 -30
  158. package/dist/favicons/index.d.cts +0 -7
  159. package/dist/favicons/platform/handlers/bluesky.cjs +0 -27
  160. package/dist/favicons/platform/handlers/codeberg.cjs +0 -19
  161. package/dist/favicons/platform/handlers/deviantart.cjs +0 -29
  162. package/dist/favicons/platform/handlers/devto.cjs +0 -31
  163. package/dist/favicons/platform/handlers/github.cjs +0 -19
  164. package/dist/favicons/platform/handlers/githubGist.cjs +0 -19
  165. package/dist/favicons/platform/handlers/gitlab.cjs +0 -38
  166. package/dist/favicons/platform/handlers/lobsters.cjs +0 -21
  167. package/dist/favicons/platform/handlers/mastodon.cjs +0 -40
  168. package/dist/favicons/platform/handlers/reddit.cjs +0 -42
  169. package/dist/favicons/platform/handlers/sourceforge.cjs +0 -21
  170. package/dist/favicons/platform/handlers/tumblr.cjs +0 -16
  171. package/dist/favicons/types.d.cts +0 -4
  172. package/dist/favicons/utils.cjs +0 -10
  173. package/dist/favicons.cjs +0 -12
  174. package/dist/favicons.d.cts +0 -4
  175. package/dist/feeds/defaults.cjs +0 -178
  176. package/dist/feeds/defaults.d.cts +0 -20
  177. package/dist/feeds/extractors.cjs +0 -46
  178. package/dist/feeds/extractors.d.cts +0 -7
  179. package/dist/feeds/index.cjs +0 -27
  180. package/dist/feeds/index.d.cts +0 -7
  181. package/dist/feeds/platform/handlers/applePodcasts.cjs +0 -26
  182. package/dist/feeds/platform/handlers/behance.cjs +0 -49
  183. package/dist/feeds/platform/handlers/blogspot.cjs +0 -36
  184. package/dist/feeds/platform/handlers/bluesky.cjs +0 -20
  185. package/dist/feeds/platform/handlers/codeberg.cjs +0 -69
  186. package/dist/feeds/platform/handlers/csdn.cjs +0 -20
  187. package/dist/feeds/platform/handlers/dailymotion.cjs +0 -70
  188. package/dist/feeds/platform/handlers/deviantart.cjs +0 -66
  189. package/dist/feeds/platform/handlers/devto.cjs +0 -50
  190. package/dist/feeds/platform/handlers/douban.cjs +0 -56
  191. package/dist/feeds/platform/handlers/github.cjs +0 -116
  192. package/dist/feeds/platform/handlers/githubGist.cjs +0 -45
  193. package/dist/feeds/platform/handlers/gitlab.cjs +0 -80
  194. package/dist/feeds/platform/handlers/goodreads.cjs +0 -39
  195. package/dist/feeds/platform/handlers/hashnode.cjs +0 -16
  196. package/dist/feeds/platform/handlers/hatenablog.cjs +0 -53
  197. package/dist/feeds/platform/handlers/itchio.cjs +0 -88
  198. package/dist/feeds/platform/handlers/kickstarter.cjs +0 -22
  199. package/dist/feeds/platform/handlers/lemmy.cjs +0 -46
  200. package/dist/feeds/platform/handlers/letterboxd.cjs +0 -42
  201. package/dist/feeds/platform/handlers/lobsters.cjs +0 -57
  202. package/dist/feeds/platform/handlers/mastodon.cjs +0 -42
  203. package/dist/feeds/platform/handlers/medium.cjs +0 -68
  204. package/dist/feeds/platform/handlers/paragraph.cjs +0 -21
  205. package/dist/feeds/platform/handlers/pinterest.cjs +0 -44
  206. package/dist/feeds/platform/handlers/producthunt.cjs +0 -29
  207. package/dist/feeds/platform/handlers/reddit.cjs +0 -75
  208. package/dist/feeds/platform/handlers/soundcloud.cjs +0 -39
  209. package/dist/feeds/platform/handlers/sourceforge.cjs +0 -20
  210. package/dist/feeds/platform/handlers/stackExchange.cjs +0 -40
  211. package/dist/feeds/platform/handlers/steam.cjs +0 -28
  212. package/dist/feeds/platform/handlers/substack.cjs +0 -23
  213. package/dist/feeds/platform/handlers/tumblr.cjs +0 -24
  214. package/dist/feeds/platform/handlers/v2ex.cjs +0 -35
  215. package/dist/feeds/platform/handlers/vimeo.cjs +0 -69
  216. package/dist/feeds/platform/handlers/wordpress.cjs +0 -64
  217. package/dist/feeds/platform/handlers/wpengine.cjs +0 -10
  218. package/dist/feeds/platform/handlers/ximalaya.cjs +0 -20
  219. package/dist/feeds/platform/handlers/youtube.cjs +0 -94
  220. package/dist/feeds/types.d.cts +0 -9
  221. package/dist/feeds.cjs +0 -15
  222. package/dist/feeds.d.cts +0 -4
  223. package/dist/hubs/discover/index.cjs +0 -30
  224. package/dist/hubs/discover/index.d.cts +0 -7
  225. package/dist/hubs/discover/types.d.cts +0 -15
  226. package/dist/hubs/feed/index.cjs +0 -32
  227. package/dist/hubs/headers/index.cjs +0 -17
  228. package/dist/hubs/html/index.cjs +0 -28
  229. package/dist/hubs.cjs +0 -0
  230. package/dist/hubs.d.cts +0 -2
  231. package/dist/index.cjs +0 -12
  232. package/dist/index.d.cts +0 -7
  233. package/dist/methods.cjs +0 -15
  234. package/dist/methods.d.cts +0 -7
  235. package/dist/utils.cjs +0 -9
  236. package/dist/utils.d.cts +0 -2
@@ -0,0 +1,54 @@
1
+ import { composeHint, isAnyOf, isHostOf } from "../../../common/utils.js";
2
+ //#region src/feeds/platform/handlers/zenn.ts
3
+ const hosts = ["zenn.dev", "www.zenn.dev"];
4
+ const excludedPaths = [
5
+ "about",
6
+ "api",
7
+ "articles",
8
+ "books",
9
+ "login",
10
+ "notifications",
11
+ "p",
12
+ "privacy",
13
+ "publications",
14
+ "scraps",
15
+ "search",
16
+ "settings",
17
+ "signup",
18
+ "terms",
19
+ "topics"
20
+ ];
21
+ const topicRegex = /^\/topics\/([^/]+)/;
22
+ const publicationShortRegex = /^\/p\/([^/]+)/;
23
+ const publicationLongRegex = /^\/publications\/([^/]+)/;
24
+ const zennHandler = {
25
+ match: (url) => {
26
+ return isHostOf(url, hosts);
27
+ },
28
+ resolve: (url) => {
29
+ const { pathname } = new URL(url);
30
+ const topicMatch = pathname.match(topicRegex);
31
+ if (topicMatch?.[1]) return [{
32
+ uri: `https://zenn.dev/topics/${topicMatch[1]}/feed`,
33
+ hint: composeHint("zenn:topic")
34
+ }];
35
+ const pubMatch = pathname.match(publicationShortRegex) ?? pathname.match(publicationLongRegex);
36
+ if (pubMatch?.[1]) return [{
37
+ uri: `https://zenn.dev/p/${pubMatch[1]}/feed`,
38
+ hint: composeHint("zenn:publication")
39
+ }];
40
+ const pathSegments = pathname.split("/").filter(Boolean);
41
+ if (pathSegments.length === 0) return [{
42
+ uri: "https://zenn.dev/feed",
43
+ hint: composeHint("zenn:trending")
44
+ }];
45
+ const username = pathSegments[0];
46
+ if (isAnyOf(username, excludedPaths)) return [];
47
+ return [{
48
+ uri: `https://zenn.dev/${username}/feed`,
49
+ hint: composeHint("zenn:posts")
50
+ }];
51
+ }
52
+ };
53
+ //#endregion
54
+ export { zennHandler };
package/dist/feeds.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- import { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal } from "./feeds/defaults.js";
1
+ import { anchorLabels, anchorPathSegments, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal } from "./feeds/defaults.js";
2
2
  import { FeedResult } from "./feeds/types.js";
3
- import { defaultExtractor } from "./feeds/extractors.js";
4
- export { FeedResult, anchorLabels, defaultExtractor, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
3
+ import { defaultExtractFn } from "./feeds/extractors.js";
4
+ export { FeedResult, anchorLabels, anchorPathSegments, defaultExtractFn, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
package/dist/feeds.js CHANGED
@@ -1,3 +1,3 @@
1
- import { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal } from "./feeds/defaults.js";
2
- import { defaultExtractor } from "./feeds/extractors.js";
3
- export { anchorLabels, defaultExtractor, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
1
+ import { anchorLabels, anchorPathSegments, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal } from "./feeds/defaults.js";
2
+ import { defaultExtractFn } from "./feeds/extractors.js";
3
+ export { anchorLabels, anchorPathSegments, defaultExtractFn, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
@@ -1,5 +1,5 @@
1
- import { resolveUrl } from "../../common/utils.js";
2
- import { defaultFetchFn, normalizeInput } from "../../common/discover/utils.js";
1
+ import { normalizeInput } from "../../common/discover/utils.js";
2
+ import { defaultFetchFn, defaultResolveUrlFn } from "../../common/discover/defaults.js";
3
3
  import { discoverHubsFromFeed } from "../feed/index.js";
4
4
  import { discoverHubsFromHeaders } from "../headers/index.js";
5
5
  import { discoverHubsFromHtml } from "../html/index.js";
@@ -9,7 +9,7 @@ const discoverHubs = async (input, options = {}) => {
9
9
  "headers",
10
10
  "feed",
11
11
  "html"
12
- ], fetchFn = defaultFetchFn, resolveUrlFn = resolveUrl } = options;
12
+ ], fetchFn = defaultFetchFn, resolveUrlFn = defaultResolveUrlFn } = options;
13
13
  const normalizedInput = await normalizeInput(input, fetchFn);
14
14
  const results = [];
15
15
  if (methods.includes("headers") && normalizedInput.headers) {
@@ -1,10 +1,10 @@
1
- import { resolveUrl } from "../../common/utils.js";
1
+ import { defaultResolveUrlFn } from "../../common/discover/defaults.js";
2
2
  import { parseFeed } from "feedsmith";
3
3
  //#region src/hubs/feed/index.ts
4
4
  const getLinksWithRel = (links, rel) => {
5
5
  return links?.filter((link) => link.rel === rel && link.href).map((link) => link.href) ?? [];
6
6
  };
7
- const discoverHubsFromFeed = (content, baseUrl, resolveUrlFn = resolveUrl) => {
7
+ const discoverHubsFromFeed = (content, baseUrl, resolveUrlFn = defaultResolveUrlFn) => {
8
8
  try {
9
9
  const { format, feed } = parseFeed(content);
10
10
  if (format === "json") {
@@ -1,9 +1,9 @@
1
- import { resolveUrl } from "../../common/utils.js";
1
+ import { defaultResolveUrlFn } from "../../common/discover/defaults.js";
2
2
  import { discoverUrisFromHeaders } from "../../common/uris/headers/index.js";
3
3
  //#region src/hubs/headers/index.ts
4
4
  const hubSelector = [{ rel: "hub" }];
5
5
  const selfSelector = [{ rel: "self" }];
6
- const discoverHubsFromHeaders = (headers, baseUrl, resolveUrlFn = resolveUrl) => {
6
+ const discoverHubsFromHeaders = (headers, baseUrl, resolveUrlFn = defaultResolveUrlFn) => {
7
7
  const hubUris = discoverUrisFromHeaders(headers, { linkSelectors: hubSelector });
8
8
  if (hubUris.length === 0) return [];
9
9
  const selfUris = discoverUrisFromHeaders(headers, { linkSelectors: selfSelector });
@@ -1,4 +1,4 @@
1
- import { resolveUrl } from "../../common/utils.js";
1
+ import { defaultResolveUrlFn } from "../../common/discover/defaults.js";
2
2
  import { discoverUrisFromHtml } from "../../common/uris/html/index.js";
3
3
  //#region src/hubs/html/index.ts
4
4
  const hubSelector = [{ rel: "hub" }];
@@ -8,7 +8,7 @@ const htmlOptions = {
8
8
  anchorIgnoredUris: [],
9
9
  anchorLabels: []
10
10
  };
11
- const discoverHubsFromHtml = (content, baseUrl, resolveUrlFn = resolveUrl) => {
11
+ const discoverHubsFromHtml = (content, baseUrl, resolveUrlFn = defaultResolveUrlFn) => {
12
12
  const hubUris = discoverUrisFromHtml(content, {
13
13
  ...htmlOptions,
14
14
  linkSelectors: hubSelector
package/dist/index.d.ts CHANGED
@@ -1,6 +1,7 @@
1
1
  import { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethod, DiscoverMethodsConfig, DiscoverOnProgressFn, DiscoverOptions, DiscoverProgress, DiscoverResolveSiteUrlFn, DiscoverResolveUrlFn, DiscoverResult, DiscoverUriEntry, DiscoverUriHint, UriEntry } from "./common/types.js";
2
2
  import { discoverBlogrolls } from "./blogrolls/index.js";
3
- import { defaultResolveSiteUrlFn, getFeedSiteUrl } from "./common/discover/utils.js";
3
+ import { defaultResolveSiteUrlFn } from "./common/discover/defaults.js";
4
+ import { getFeedSiteUrl } from "./common/discover/utils.js";
4
5
  import { discoverFavicons } from "./favicons/index.js";
5
6
  import { discoverFeeds } from "./feeds/index.js";
6
7
  import { discoverHubs } from "./hubs/discover/index.js";
package/dist/index.js CHANGED
@@ -1,4 +1,5 @@
1
- import { defaultResolveSiteUrlFn, getFeedSiteUrl } from "./common/discover/utils.js";
1
+ import { getFeedSiteUrl } from "./common/discover/utils.js";
2
+ import { defaultResolveSiteUrlFn } from "./common/discover/defaults.js";
2
3
  import { discoverBlogrolls } from "./blogrolls/index.js";
3
4
  import { discoverFavicons } from "./favicons/index.js";
4
5
  import { discoverFeeds } from "./feeds/index.js";
package/dist/utils.d.ts CHANGED
@@ -1,2 +1,3 @@
1
+ import { Pattern } from "./common/types.js";
1
2
  import { anyWordMatchesAnyOf, endsWithAnyOf, includesAnyOf, isAnyOf, isHostOf, isSubdomainOf, omitEmpty } from "./common/utils.js";
2
- export { anyWordMatchesAnyOf, endsWithAnyOf, includesAnyOf, isAnyOf, isHostOf, isSubdomainOf, omitEmpty };
3
+ export { type Pattern, anyWordMatchesAnyOf, endsWithAnyOf, includesAnyOf, isAnyOf, isHostOf, isSubdomainOf, omitEmpty };
package/package.json CHANGED
@@ -26,74 +26,32 @@
26
26
  "type": "module",
27
27
  "exports": {
28
28
  ".": {
29
- "import": {
30
- "types": "./dist/index.d.ts",
31
- "default": "./dist/index.js"
32
- },
33
- "require": {
34
- "types": "./dist/index.d.cts",
35
- "default": "./dist/index.cjs"
36
- }
29
+ "types": "./dist/index.d.ts",
30
+ "default": "./dist/index.js"
37
31
  },
38
32
  "./feeds": {
39
- "import": {
40
- "types": "./dist/feeds.d.ts",
41
- "default": "./dist/feeds.js"
42
- },
43
- "require": {
44
- "types": "./dist/feeds.d.cts",
45
- "default": "./dist/feeds.cjs"
46
- }
33
+ "types": "./dist/feeds.d.ts",
34
+ "default": "./dist/feeds.js"
47
35
  },
48
36
  "./blogrolls": {
49
- "import": {
50
- "types": "./dist/blogrolls.d.ts",
51
- "default": "./dist/blogrolls.js"
52
- },
53
- "require": {
54
- "types": "./dist/blogrolls.d.cts",
55
- "default": "./dist/blogrolls.cjs"
56
- }
37
+ "types": "./dist/blogrolls.d.ts",
38
+ "default": "./dist/blogrolls.js"
57
39
  },
58
40
  "./favicons": {
59
- "import": {
60
- "types": "./dist/favicons.d.ts",
61
- "default": "./dist/favicons.js"
62
- },
63
- "require": {
64
- "types": "./dist/favicons.d.cts",
65
- "default": "./dist/favicons.cjs"
66
- }
41
+ "types": "./dist/favicons.d.ts",
42
+ "default": "./dist/favicons.js"
67
43
  },
68
44
  "./hubs": {
69
- "import": {
70
- "types": "./dist/hubs.d.ts",
71
- "default": "./dist/hubs.js"
72
- },
73
- "require": {
74
- "types": "./dist/hubs.d.cts",
75
- "default": "./dist/hubs.cjs"
76
- }
45
+ "types": "./dist/hubs.d.ts",
46
+ "default": "./dist/hubs.js"
77
47
  },
78
48
  "./methods": {
79
- "import": {
80
- "types": "./dist/methods.d.ts",
81
- "default": "./dist/methods.js"
82
- },
83
- "require": {
84
- "types": "./dist/methods.d.cts",
85
- "default": "./dist/methods.cjs"
86
- }
49
+ "types": "./dist/methods.d.ts",
50
+ "default": "./dist/methods.js"
87
51
  },
88
52
  "./utils": {
89
- "import": {
90
- "types": "./dist/utils.d.ts",
91
- "default": "./dist/utils.js"
92
- },
93
- "require": {
94
- "types": "./dist/utils.d.cts",
95
- "default": "./dist/utils.cjs"
96
- }
53
+ "types": "./dist/utils.d.ts",
54
+ "default": "./dist/utils.js"
97
55
  },
98
56
  "./package.json": "./package.json"
99
57
  },
@@ -102,20 +60,22 @@
102
60
  ],
103
61
  "scripts": {
104
62
  "prepare": "lefthook install",
105
- "build": "tsdown src/exports/index.ts src/exports/feeds.ts src/exports/blogrolls.ts src/exports/favicons.ts src/exports/hubs.ts src/exports/methods.ts src/exports/utils.ts --format cjs,esm --dts --clean --unbundle --no-fixed-extension",
63
+ "build": "tsdown src/exports/index.ts src/exports/feeds.ts src/exports/blogrolls.ts src/exports/favicons.ts src/exports/hubs.ts src/exports/methods.ts src/exports/utils.ts --format esm --dts --clean --unbundle --no-fixed-extension",
106
64
  "docs:dev": "vitepress dev docs",
107
65
  "docs:build": "vitepress build docs"
108
66
  },
109
67
  "dependencies": {
110
- "feedsmith": "^2.9.1",
111
- "htmlparser2": "^10.1.0",
112
- "typescript": "^6.0.2"
68
+ "htmlparser2": "^12.0.0"
69
+ },
70
+ "peerDependencies": {
71
+ "feedsmith": "^3.0.0-beta.4"
113
72
  },
114
73
  "devDependencies": {
115
- "@types/bun": "^1.3.11",
116
- "kvalita": "1.12.4",
117
- "tsdown": "^0.21.7",
74
+ "@types/bun": "^1.3.13",
75
+ "feedsmith": "^3.0.0-beta.4",
76
+ "kvalita": "^1.13.0",
77
+ "tsdown": "^0.22.2",
118
78
  "vitepress": "^2.0.0-alpha.17"
119
79
  },
120
- "version": "1.8.0"
80
+ "version": "2.0.0-beta.1"
121
81
  }
@@ -1,51 +0,0 @@
1
- //#region src/blogrolls/defaults.ts
2
- const mimeTypes = [
3
- "text/x-opml",
4
- "application/xml",
5
- "text/xml"
6
- ];
7
- const urisMinimal = [
8
- "/.well-known/recommendations.opml",
9
- "/blogroll.opml",
10
- "/opml.xml"
11
- ];
12
- const urisBalanced = [
13
- ...urisMinimal,
14
- "/blogroll.xml",
15
- "/subscriptions.opml",
16
- "/recommendations.opml"
17
- ];
18
- const urisComprehensive = [
19
- ...urisBalanced,
20
- "/links.opml",
21
- "/feeds.opml",
22
- "/subscriptions.xml"
23
- ];
24
- const anchorLabels = [
25
- "blogroll",
26
- "opml",
27
- "subscriptions",
28
- "reading list"
29
- ];
30
- const linkSelectors = [{ rel: "blogroll" }, {
31
- rel: "outline",
32
- types: mimeTypes
33
- }];
34
- const defaultHtmlOptions = {
35
- linkSelectors,
36
- anchorUris: urisComprehensive,
37
- anchorIgnoredUris: [],
38
- anchorLabels
39
- };
40
- const defaultHeadersOptions = { linkSelectors };
41
- const defaultGuessOptions = { uris: urisBalanced };
42
- //#endregion
43
- exports.anchorLabels = anchorLabels;
44
- exports.defaultGuessOptions = defaultGuessOptions;
45
- exports.defaultHeadersOptions = defaultHeadersOptions;
46
- exports.defaultHtmlOptions = defaultHtmlOptions;
47
- exports.linkSelectors = linkSelectors;
48
- exports.mimeTypes = mimeTypes;
49
- exports.urisBalanced = urisBalanced;
50
- exports.urisComprehensive = urisComprehensive;
51
- exports.urisMinimal = urisMinimal;
@@ -1,17 +0,0 @@
1
- import { GuessMethodOptions } from "../common/uris/guess/types.cjs";
2
- import { HeadersMethodOptions } from "../common/uris/headers/types.cjs";
3
- import { HtmlMethodOptions } from "../common/uris/html/types.cjs";
4
- import { LinkSelector } from "../common/types.cjs";
5
-
6
- //#region src/blogrolls/defaults.d.ts
7
- declare const mimeTypes: string[];
8
- declare const urisMinimal: string[];
9
- declare const urisBalanced: string[];
10
- declare const urisComprehensive: string[];
11
- declare const anchorLabels: string[];
12
- declare const linkSelectors: Array<LinkSelector>;
13
- declare const defaultHtmlOptions: Omit<HtmlMethodOptions, 'baseUrl'>;
14
- declare const defaultHeadersOptions: Omit<HeadersMethodOptions, 'baseUrl'>;
15
- declare const defaultGuessOptions: Omit<GuessMethodOptions, 'baseUrl'>;
16
- //#endregion
17
- export { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
@@ -1,21 +0,0 @@
1
- let feedsmith = require("feedsmith");
2
- //#region src/blogrolls/extractors.ts
3
- const defaultExtractor = ({ content, url }) => {
4
- if (!content) return {
5
- url,
6
- isValid: false
7
- };
8
- try {
9
- return {
10
- url,
11
- isValid: true,
12
- title: (0, feedsmith.parseOpml)(content).head?.title
13
- };
14
- } catch {}
15
- return {
16
- url,
17
- isValid: false
18
- };
19
- };
20
- //#endregion
21
- exports.defaultExtractor = defaultExtractor;
@@ -1,7 +0,0 @@
1
- import { DiscoverExtractFn } from "../common/types.cjs";
2
- import { BlogrollResult } from "./types.cjs";
3
-
4
- //#region src/blogrolls/extractors.d.ts
5
- declare const defaultExtractor: DiscoverExtractFn<BlogrollResult>;
6
- //#endregion
7
- export { defaultExtractor };
@@ -1,26 +0,0 @@
1
- const require_utils = require("../common/utils.cjs");
2
- const require_utils$1 = require("../common/discover/utils.cjs");
3
- const require_index = require("../common/discover/index.cjs");
4
- const require_defaults = require("./defaults.cjs");
5
- const require_extractors = require("./extractors.cjs");
6
- //#region src/blogrolls/index.ts
7
- const discoverBlogrolls = (input, options = {}) => {
8
- return require_index.discover(input, {
9
- ...options,
10
- methods: options.methods ?? [
11
- "html",
12
- "headers",
13
- "guess"
14
- ],
15
- fetchFn: options.fetchFn ?? require_utils$1.defaultFetchFn,
16
- extractFn: options.extractFn ?? require_extractors.defaultExtractor,
17
- resolveUrlFn: options.resolveUrlFn ?? require_utils.resolveUrl,
18
- resolveSiteUrlFn: options.resolveSiteUrlFn ?? require_utils$1.defaultResolveSiteUrlFn
19
- }, {
20
- html: require_defaults.defaultHtmlOptions,
21
- headers: require_defaults.defaultHeadersOptions,
22
- guess: require_defaults.defaultGuessOptions
23
- });
24
- };
25
- //#endregion
26
- exports.discoverBlogrolls = discoverBlogrolls;
@@ -1,7 +0,0 @@
1
- import { DiscoverInput, DiscoverOptions, DiscoverResult } from "../common/types.cjs";
2
- import { BlogrollResult } from "./types.cjs";
3
-
4
- //#region src/blogrolls/index.d.ts
5
- declare const discoverBlogrolls: <TValid extends BlogrollResult = BlogrollResult>(input: DiscoverInput, options?: DiscoverOptions<TValid, "html" | "headers" | "guess">) => Promise<Array<DiscoverResult<TValid>>>;
6
- //#endregion
7
- export { discoverBlogrolls };
@@ -1,6 +0,0 @@
1
- //#region src/blogrolls/types.d.ts
2
- type BlogrollResult = {
3
- title?: string;
4
- };
5
- //#endregion
6
- export { BlogrollResult };
@@ -1,13 +0,0 @@
1
- Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
2
- const require_defaults = require("./blogrolls/defaults.cjs");
3
- const require_extractors = require("./blogrolls/extractors.cjs");
4
- exports.anchorLabels = require_defaults.anchorLabels;
5
- exports.defaultExtractor = require_extractors.defaultExtractor;
6
- exports.defaultGuessOptions = require_defaults.defaultGuessOptions;
7
- exports.defaultHeadersOptions = require_defaults.defaultHeadersOptions;
8
- exports.defaultHtmlOptions = require_defaults.defaultHtmlOptions;
9
- exports.linkSelectors = require_defaults.linkSelectors;
10
- exports.mimeTypes = require_defaults.mimeTypes;
11
- exports.urisBalanced = require_defaults.urisBalanced;
12
- exports.urisComprehensive = require_defaults.urisComprehensive;
13
- exports.urisMinimal = require_defaults.urisMinimal;
@@ -1,4 +0,0 @@
1
- import { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal } from "./blogrolls/defaults.cjs";
2
- import { BlogrollResult } from "./blogrolls/types.cjs";
3
- import { defaultExtractor } from "./blogrolls/extractors.cjs";
4
- export { BlogrollResult, anchorLabels, defaultExtractor, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
@@ -1,105 +0,0 @@
1
- const require_types = require("../types.cjs");
2
- const require_utils = require("../utils.cjs");
3
- const require_index = require("../uris/index.cjs");
4
- const require_utils$1 = require("./utils.cjs");
5
- //#region src/common/discover/index.ts
6
- const discover = async (input, options, defaults) => {
7
- const { methods, fetchFn, extractFn, resolveUrlFn, resolveSiteUrlFn, stopOnFirstMethod = false, stopOnFirstResult = false, concurrency = 3, includeInvalid = false, onProgress } = options;
8
- const sourceInput = await require_utils$1.normalizeInput(input, fetchFn);
9
- if (sourceInput.content) {
10
- const result = await extractFn({
11
- url: sourceInput.url,
12
- content: sourceInput.content,
13
- headers: sourceInput.headers
14
- });
15
- if (result.isValid) return [result];
16
- }
17
- let siteInput;
18
- if (resolveSiteUrlFn) {
19
- const siteUrl = resolveSiteUrlFn(sourceInput);
20
- if (siteUrl) try {
21
- const response = await fetchFn(siteUrl);
22
- siteInput = {
23
- url: response.url,
24
- content: typeof response.body === "string" ? response.body : "",
25
- headers: response.headers
26
- };
27
- } catch {}
28
- }
29
- const urisByMethod = await require_index.discoverUris(require_utils$1.normalizeMethodsConfig(sourceInput, siteInput, methods, defaults), fetchFn);
30
- const seen = /* @__PURE__ */ new Set();
31
- const methodGroups = [];
32
- for (const method of require_types.discoverMethodOrder) {
33
- const rawUris = urisByMethod[method];
34
- if (!rawUris || rawUris.length === 0) continue;
35
- const unique = rawUris.map((entry) => {
36
- return require_utils$1.normalizeUriEntry(entry, resolveUrlFn, sourceInput.url);
37
- }).filter((entry) => {
38
- const key = typeof entry.uri === "string" ? entry.uri : entry.uri.join("\0");
39
- if (seen.has(key)) return false;
40
- seen.add(key);
41
- return true;
42
- });
43
- if (unique.length > 0) methodGroups.push({
44
- method,
45
- entries: unique
46
- });
47
- }
48
- const total = methodGroups.reduce((sum, group) => sum + group.entries.length, 0);
49
- const results = [];
50
- let tested = 0;
51
- let found = 0;
52
- const fetchAndExtract = async (url) => {
53
- try {
54
- const fetchResult = await fetchFn(url);
55
- return await extractFn({
56
- url: fetchResult.url,
57
- content: typeof fetchResult.body === "string" ? fetchResult.body : "",
58
- headers: fetchResult.headers,
59
- status: fetchResult.status
60
- });
61
- } catch (error) {
62
- return {
63
- url,
64
- isValid: false,
65
- error
66
- };
67
- }
68
- };
69
- const processUri = async (entry, method) => {
70
- const alternatives = typeof entry.uri === "string" ? [entry.uri] : entry.uri;
71
- for (const url of alternatives) {
72
- const result = await fetchAndExtract(url);
73
- results.push(entry.hint ? {
74
- ...result,
75
- method,
76
- hint: entry.hint
77
- } : {
78
- ...result,
79
- method
80
- });
81
- tested += 1;
82
- if (result.isValid) found += 1;
83
- onProgress?.({
84
- tested,
85
- total,
86
- found,
87
- current: url
88
- });
89
- if (result.isValid) break;
90
- }
91
- };
92
- for (const { method, entries } of methodGroups) {
93
- const foundBefore = found;
94
- await require_utils.processConcurrently(entries, (entry) => processUri(entry, method), {
95
- concurrency,
96
- shouldStop: () => {
97
- return stopOnFirstResult && found > 0;
98
- }
99
- });
100
- if (stopOnFirstMethod && found > foundBefore) break;
101
- }
102
- return includeInvalid ? results : results.filter((result) => result.isValid);
103
- };
104
- //#endregion
105
- exports.discover = discover;