feedscout 1.8.1 → 2.0.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. package/README.md +1 -1
  2. package/dist/common/discover/utils.d.ts +2 -3
  3. package/dist/common/locales.js +214 -10
  4. package/dist/common/types.d.ts +2 -1
  5. package/dist/common/uris/feed/types.d.ts +2 -2
  6. package/dist/common/uris/html/types.d.ts +4 -4
  7. package/dist/common/utils.d.ts +6 -4
  8. package/dist/common/utils.js +12 -3
  9. package/dist/feeds/defaults.d.ts +2 -1
  10. package/dist/feeds/defaults.js +116 -9
  11. package/dist/feeds/extractors.js +2 -2
  12. package/dist/feeds/platform/handlers/acast.js +26 -0
  13. package/dist/feeds/platform/handlers/ameblo.js +36 -0
  14. package/dist/feeds/platform/handlers/applePodcasts.js +1 -1
  15. package/dist/feeds/platform/handlers/arena.js +42 -0
  16. package/dist/feeds/platform/handlers/artstation.js +52 -0
  17. package/dist/feeds/platform/handlers/audioboom.js +23 -0
  18. package/dist/feeds/platform/handlers/bearblog.js +45 -0
  19. package/dist/feeds/platform/handlers/behance.js +7 -0
  20. package/dist/feeds/platform/handlers/blogspot.js +38 -2
  21. package/dist/feeds/platform/handlers/bookwyrm.js +48 -0
  22. package/dist/feeds/platform/handlers/buttondown.js +43 -0
  23. package/dist/feeds/platform/handlers/buzzsprout.js +22 -0
  24. package/dist/feeds/platform/handlers/codeberg.js +5 -5
  25. package/dist/feeds/platform/handlers/dailymotion.js +16 -0
  26. package/dist/feeds/platform/handlers/deviantart.js +25 -6
  27. package/dist/feeds/platform/handlers/devto.js +8 -0
  28. package/dist/feeds/platform/handlers/discourse.js +70 -0
  29. package/dist/feeds/platform/handlers/dreamwidth.js +48 -0
  30. package/dist/feeds/platform/handlers/exblog.js +35 -0
  31. package/dist/feeds/platform/handlers/fireside.js +24 -0
  32. package/dist/feeds/platform/handlers/friendica.js +44 -0
  33. package/dist/feeds/platform/handlers/ghost.js +30 -0
  34. package/dist/feeds/platform/handlers/github.js +6 -0
  35. package/dist/feeds/platform/handlers/githubGist.js +16 -5
  36. package/dist/feeds/platform/handlers/gitlab.js +31 -13
  37. package/dist/feeds/platform/handlers/goodreads.js +18 -8
  38. package/dist/feeds/platform/handlers/hackernews.js +21 -0
  39. package/dist/feeds/platform/handlers/hashnode.js +1 -1
  40. package/dist/feeds/platform/handlers/hatenablog.js +4 -1
  41. package/dist/feeds/platform/handlers/hearthis.js +32 -0
  42. package/dist/feeds/platform/handlers/heyWorld.js +18 -0
  43. package/dist/feeds/platform/handlers/insanejournal.js +69 -0
  44. package/dist/feeds/platform/handlers/itchio.js +24 -1
  45. package/dist/feeds/platform/handlers/lemmy.js +46 -4
  46. package/dist/feeds/platform/handlers/letterboxd.js +4 -0
  47. package/dist/feeds/platform/handlers/libsyn.js +25 -0
  48. package/dist/feeds/platform/handlers/listed.js +20 -0
  49. package/dist/feeds/platform/handlers/livejournal.js +68 -0
  50. package/dist/feeds/platform/handlers/mastodon.js +32 -0
  51. package/dist/feeds/platform/handlers/mataroa.js +16 -0
  52. package/dist/feeds/platform/handlers/medium.js +2 -2
  53. package/dist/feeds/platform/handlers/microblog.js +55 -0
  54. package/dist/feeds/platform/handlers/misskey.js +40 -0
  55. package/dist/feeds/platform/handlers/myanimelist.js +43 -0
  56. package/dist/feeds/platform/handlers/naverBlog.js +21 -0
  57. package/dist/feeds/platform/handlers/nebula.js +68 -0
  58. package/dist/feeds/platform/handlers/note.js +53 -0
  59. package/dist/feeds/platform/handlers/observable.js +34 -0
  60. package/dist/feeds/platform/handlers/odysee.js +20 -0
  61. package/dist/feeds/platform/handlers/pagecord.js +16 -0
  62. package/dist/feeds/platform/handlers/paragraph.js +1 -2
  63. package/dist/feeds/platform/handlers/pika.js +35 -0
  64. package/dist/feeds/platform/handlers/pinterest.js +13 -0
  65. package/dist/feeds/platform/handlers/pixelfed.js +46 -0
  66. package/dist/feeds/platform/handlers/pleroma.js +34 -0
  67. package/dist/feeds/platform/handlers/podbean.js +29 -0
  68. package/dist/feeds/platform/handlers/podigee.js +29 -0
  69. package/dist/feeds/platform/handlers/posthaven.js +24 -0
  70. package/dist/feeds/platform/handlers/prose.js +26 -0
  71. package/dist/feeds/platform/handlers/qiita.js +58 -0
  72. package/dist/feeds/platform/handlers/reddit.js +83 -9
  73. package/dist/feeds/platform/handlers/rssCom.js +20 -0
  74. package/dist/feeds/platform/handlers/seesaa.js +22 -0
  75. package/dist/feeds/platform/handlers/sourceforge.js +37 -4
  76. package/dist/feeds/platform/handlers/spreaker.js +21 -0
  77. package/dist/feeds/platform/handlers/stackExchange.js +26 -2
  78. package/dist/feeds/platform/handlers/steam.js +7 -0
  79. package/dist/feeds/platform/handlers/tildes.js +41 -0
  80. package/dist/feeds/platform/handlers/tistory.js +16 -0
  81. package/dist/feeds/platform/handlers/transistor.js +29 -0
  82. package/dist/feeds/platform/handlers/velog.js +24 -0
  83. package/dist/feeds/platform/handlers/vimeo.js +6 -0
  84. package/dist/feeds/platform/handlers/weblogLol.js +26 -0
  85. package/dist/feeds/platform/handlers/weebly.js +25 -0
  86. package/dist/feeds/platform/handlers/wordpress.js +173 -28
  87. package/dist/feeds/platform/handlers/writeas.js +51 -0
  88. package/dist/feeds/platform/handlers/ximalaya.js +1 -1
  89. package/dist/feeds/platform/handlers/youtube.js +4 -1
  90. package/dist/feeds/platform/handlers/zenn.js +54 -0
  91. package/dist/feeds.d.ts +2 -2
  92. package/dist/feeds.js +2 -2
  93. package/dist/utils.d.ts +2 -1
  94. package/package.json +24 -64
  95. package/dist/blogrolls/defaults.cjs +0 -51
  96. package/dist/blogrolls/defaults.d.cts +0 -17
  97. package/dist/blogrolls/extractors.cjs +0 -21
  98. package/dist/blogrolls/extractors.d.cts +0 -7
  99. package/dist/blogrolls/index.cjs +0 -25
  100. package/dist/blogrolls/index.d.cts +0 -7
  101. package/dist/blogrolls/types.d.cts +0 -6
  102. package/dist/blogrolls.cjs +0 -13
  103. package/dist/blogrolls.d.cts +0 -4
  104. package/dist/common/discover/defaults.cjs +0 -46
  105. package/dist/common/discover/defaults.d.cts +0 -6
  106. package/dist/common/discover/index.cjs +0 -105
  107. package/dist/common/discover/utils.cjs +0 -101
  108. package/dist/common/discover/utils.d.cts +0 -6
  109. package/dist/common/locales.cjs +0 -144
  110. package/dist/common/types.cjs +0 -10
  111. package/dist/common/types.d.cts +0 -89
  112. package/dist/common/uris/feed/index.cjs +0 -12
  113. package/dist/common/uris/feed/index.d.cts +0 -6
  114. package/dist/common/uris/feed/types.d.cts +0 -9
  115. package/dist/common/uris/guess/index.cjs +0 -8
  116. package/dist/common/uris/guess/index.d.cts +0 -7
  117. package/dist/common/uris/guess/types.d.cts +0 -10
  118. package/dist/common/uris/guess/utils.cjs +0 -42
  119. package/dist/common/uris/guess/utils.d.cts +0 -8
  120. package/dist/common/uris/headers/index.cjs +0 -26
  121. package/dist/common/uris/headers/index.d.cts +0 -6
  122. package/dist/common/uris/headers/types.d.cts +0 -9
  123. package/dist/common/uris/html/handlers.cjs +0 -45
  124. package/dist/common/uris/html/index.cjs +0 -19
  125. package/dist/common/uris/html/index.d.cts +0 -6
  126. package/dist/common/uris/html/types.d.cts +0 -12
  127. package/dist/common/uris/index.cjs +0 -32
  128. package/dist/common/uris/platform/index.cjs +0 -10
  129. package/dist/common/uris/platform/index.d.cts +0 -7
  130. package/dist/common/uris/platform/types.d.cts +0 -13
  131. package/dist/common/utils.cjs +0 -89
  132. package/dist/common/utils.d.cts +0 -10
  133. package/dist/favicons/defaults.cjs +0 -65
  134. package/dist/favicons/defaults.d.cts +0 -18
  135. package/dist/favicons/extractors.cjs +0 -25
  136. package/dist/favicons/extractors.d.cts +0 -7
  137. package/dist/favicons/index.cjs +0 -29
  138. package/dist/favicons/index.d.cts +0 -7
  139. package/dist/favicons/platform/handlers/bluesky.cjs +0 -27
  140. package/dist/favicons/platform/handlers/codeberg.cjs +0 -19
  141. package/dist/favicons/platform/handlers/deviantart.cjs +0 -29
  142. package/dist/favicons/platform/handlers/devto.cjs +0 -31
  143. package/dist/favicons/platform/handlers/github.cjs +0 -19
  144. package/dist/favicons/platform/handlers/githubGist.cjs +0 -19
  145. package/dist/favicons/platform/handlers/gitlab.cjs +0 -38
  146. package/dist/favicons/platform/handlers/lobsters.cjs +0 -21
  147. package/dist/favicons/platform/handlers/mastodon.cjs +0 -40
  148. package/dist/favicons/platform/handlers/reddit.cjs +0 -42
  149. package/dist/favicons/platform/handlers/sourceforge.cjs +0 -21
  150. package/dist/favicons/platform/handlers/tumblr.cjs +0 -16
  151. package/dist/favicons/types.d.cts +0 -4
  152. package/dist/favicons/utils.cjs +0 -10
  153. package/dist/favicons.cjs +0 -12
  154. package/dist/favicons.d.cts +0 -4
  155. package/dist/feeds/defaults.cjs +0 -178
  156. package/dist/feeds/defaults.d.cts +0 -20
  157. package/dist/feeds/extractors.cjs +0 -46
  158. package/dist/feeds/extractors.d.cts +0 -7
  159. package/dist/feeds/index.cjs +0 -26
  160. package/dist/feeds/index.d.cts +0 -7
  161. package/dist/feeds/platform/handlers/applePodcasts.cjs +0 -26
  162. package/dist/feeds/platform/handlers/behance.cjs +0 -49
  163. package/dist/feeds/platform/handlers/blogspot.cjs +0 -36
  164. package/dist/feeds/platform/handlers/bluesky.cjs +0 -20
  165. package/dist/feeds/platform/handlers/codeberg.cjs +0 -69
  166. package/dist/feeds/platform/handlers/csdn.cjs +0 -20
  167. package/dist/feeds/platform/handlers/dailymotion.cjs +0 -70
  168. package/dist/feeds/platform/handlers/deviantart.cjs +0 -66
  169. package/dist/feeds/platform/handlers/devto.cjs +0 -50
  170. package/dist/feeds/platform/handlers/douban.cjs +0 -56
  171. package/dist/feeds/platform/handlers/github.cjs +0 -116
  172. package/dist/feeds/platform/handlers/githubGist.cjs +0 -45
  173. package/dist/feeds/platform/handlers/gitlab.cjs +0 -80
  174. package/dist/feeds/platform/handlers/goodreads.cjs +0 -39
  175. package/dist/feeds/platform/handlers/hashnode.cjs +0 -16
  176. package/dist/feeds/platform/handlers/hatenablog.cjs +0 -53
  177. package/dist/feeds/platform/handlers/itchio.cjs +0 -88
  178. package/dist/feeds/platform/handlers/kickstarter.cjs +0 -22
  179. package/dist/feeds/platform/handlers/lemmy.cjs +0 -46
  180. package/dist/feeds/platform/handlers/letterboxd.cjs +0 -42
  181. package/dist/feeds/platform/handlers/lobsters.cjs +0 -57
  182. package/dist/feeds/platform/handlers/mastodon.cjs +0 -42
  183. package/dist/feeds/platform/handlers/medium.cjs +0 -68
  184. package/dist/feeds/platform/handlers/paragraph.cjs +0 -21
  185. package/dist/feeds/platform/handlers/pinterest.cjs +0 -44
  186. package/dist/feeds/platform/handlers/producthunt.cjs +0 -29
  187. package/dist/feeds/platform/handlers/reddit.cjs +0 -75
  188. package/dist/feeds/platform/handlers/soundcloud.cjs +0 -39
  189. package/dist/feeds/platform/handlers/sourceforge.cjs +0 -20
  190. package/dist/feeds/platform/handlers/stackExchange.cjs +0 -40
  191. package/dist/feeds/platform/handlers/steam.cjs +0 -28
  192. package/dist/feeds/platform/handlers/substack.cjs +0 -23
  193. package/dist/feeds/platform/handlers/tumblr.cjs +0 -24
  194. package/dist/feeds/platform/handlers/v2ex.cjs +0 -35
  195. package/dist/feeds/platform/handlers/vimeo.cjs +0 -69
  196. package/dist/feeds/platform/handlers/wordpress.cjs +0 -64
  197. package/dist/feeds/platform/handlers/wpengine.cjs +0 -10
  198. package/dist/feeds/platform/handlers/ximalaya.cjs +0 -20
  199. package/dist/feeds/platform/handlers/youtube.cjs +0 -94
  200. package/dist/feeds/types.d.cts +0 -9
  201. package/dist/feeds.cjs +0 -15
  202. package/dist/feeds.d.cts +0 -4
  203. package/dist/hubs/discover/index.cjs +0 -30
  204. package/dist/hubs/discover/index.d.cts +0 -7
  205. package/dist/hubs/discover/types.d.cts +0 -15
  206. package/dist/hubs/feed/index.cjs +0 -32
  207. package/dist/hubs/headers/index.cjs +0 -17
  208. package/dist/hubs/html/index.cjs +0 -28
  209. package/dist/hubs.cjs +0 -0
  210. package/dist/hubs.d.cts +0 -2
  211. package/dist/index.cjs +0 -13
  212. package/dist/index.d.cts +0 -8
  213. package/dist/methods.cjs +0 -15
  214. package/dist/methods.d.cts +0 -7
  215. package/dist/utils.cjs +0 -9
  216. package/dist/utils.d.cts +0 -2
@@ -1,25 +0,0 @@
1
- const require_defaults = require("../common/discover/defaults.cjs");
2
- const require_index = require("../common/discover/index.cjs");
3
- const require_defaults$1 = require("./defaults.cjs");
4
- const require_extractors = require("./extractors.cjs");
5
- //#region src/blogrolls/index.ts
6
- const discoverBlogrolls = (input, options = {}) => {
7
- return require_index.discover(input, {
8
- ...options,
9
- methods: options.methods ?? [
10
- "html",
11
- "headers",
12
- "guess"
13
- ],
14
- fetchFn: options.fetchFn ?? require_defaults.defaultFetchFn,
15
- extractFn: options.extractFn ?? require_extractors.defaultExtractFn,
16
- resolveUrlFn: options.resolveUrlFn ?? require_defaults.defaultResolveUrlFn,
17
- resolveSiteUrlFn: options.resolveSiteUrlFn ?? require_defaults.defaultResolveSiteUrlFn
18
- }, {
19
- html: require_defaults$1.defaultHtmlOptions,
20
- headers: require_defaults$1.defaultHeadersOptions,
21
- guess: require_defaults$1.defaultGuessOptions
22
- });
23
- };
24
- //#endregion
25
- exports.discoverBlogrolls = discoverBlogrolls;
@@ -1,7 +0,0 @@
1
- import { DiscoverInput, DiscoverOptions, DiscoverResult } from "../common/types.cjs";
2
- import { BlogrollResult } from "./types.cjs";
3
-
4
- //#region src/blogrolls/index.d.ts
5
- declare const discoverBlogrolls: <TValid extends BlogrollResult = BlogrollResult>(input: DiscoverInput, options?: DiscoverOptions<TValid, "html" | "headers" | "guess">) => Promise<Array<DiscoverResult<TValid>>>;
6
- //#endregion
7
- export { discoverBlogrolls };
@@ -1,6 +0,0 @@
1
- //#region src/blogrolls/types.d.ts
2
- type BlogrollResult = {
3
- title?: string;
4
- };
5
- //#endregion
6
- export { BlogrollResult };
@@ -1,13 +0,0 @@
1
- Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
2
- const require_defaults = require("./blogrolls/defaults.cjs");
3
- const require_extractors = require("./blogrolls/extractors.cjs");
4
- exports.anchorLabels = require_defaults.anchorLabels;
5
- exports.defaultExtractFn = require_extractors.defaultExtractFn;
6
- exports.defaultGuessOptions = require_defaults.defaultGuessOptions;
7
- exports.defaultHeadersOptions = require_defaults.defaultHeadersOptions;
8
- exports.defaultHtmlOptions = require_defaults.defaultHtmlOptions;
9
- exports.linkSelectors = require_defaults.linkSelectors;
10
- exports.mimeTypes = require_defaults.mimeTypes;
11
- exports.urisBalanced = require_defaults.urisBalanced;
12
- exports.urisComprehensive = require_defaults.urisComprehensive;
13
- exports.urisMinimal = require_defaults.urisMinimal;
@@ -1,4 +0,0 @@
1
- import { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal } from "./blogrolls/defaults.cjs";
2
- import { BlogrollResult } from "./blogrolls/types.cjs";
3
- import { defaultExtractFn } from "./blogrolls/extractors.cjs";
4
- export { BlogrollResult, anchorLabels, defaultExtractFn, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
@@ -1,46 +0,0 @@
1
- const require_utils = require("./utils.cjs");
2
- let feedsmith = require("feedsmith");
3
- //#region src/common/discover/defaults.ts
4
- const defaultFetchFn = async (url, options) => {
5
- const response = await fetch(url, {
6
- method: options?.method ?? "GET",
7
- headers: options?.headers
8
- });
9
- return {
10
- headers: response.headers,
11
- body: await response.text(),
12
- url: response.url,
13
- status: response.status,
14
- statusText: response.statusText
15
- };
16
- };
17
- const defaultResolveUrlFn = (url, baseUrl) => {
18
- try {
19
- return new URL(url, baseUrl).href;
20
- } catch {
21
- return;
22
- }
23
- };
24
- const defaultResolveSiteUrlFn = (input, resolveUrlFn) => {
25
- if (!input.content) return;
26
- try {
27
- let siteUrl = require_utils.getFeedSiteUrl((0, feedsmith.parseFeed)(input.content));
28
- if (siteUrl) {
29
- siteUrl = resolveUrlFn(siteUrl, input.url);
30
- if (siteUrl) {
31
- const parsed = new URL(siteUrl);
32
- parsed.hash = "";
33
- siteUrl = parsed.href;
34
- }
35
- }
36
- if (!siteUrl || siteUrl === new URL(input.url).href) try {
37
- siteUrl = new URL(input.url).origin;
38
- } catch {}
39
- if (siteUrl && new URL(siteUrl).href === new URL(input.url).href) return;
40
- return siteUrl;
41
- } catch {}
42
- };
43
- //#endregion
44
- exports.defaultFetchFn = defaultFetchFn;
45
- exports.defaultResolveSiteUrlFn = defaultResolveSiteUrlFn;
46
- exports.defaultResolveUrlFn = defaultResolveUrlFn;
@@ -1,6 +0,0 @@
1
- import { DiscoverResolveSiteUrlFn } from "../types.cjs";
2
-
3
- //#region src/common/discover/defaults.d.ts
4
- declare const defaultResolveSiteUrlFn: DiscoverResolveSiteUrlFn;
5
- //#endregion
6
- export { defaultResolveSiteUrlFn };
@@ -1,105 +0,0 @@
1
- const require_utils = require("./utils.cjs");
2
- const require_types = require("../types.cjs");
3
- const require_utils$1 = require("../utils.cjs");
4
- const require_index = require("../uris/index.cjs");
5
- //#region src/common/discover/index.ts
6
- const discover = async (input, options, defaults) => {
7
- const { methods, fetchFn, extractFn, resolveUrlFn, resolveSiteUrlFn, stopOnFirstMethod = false, stopOnFirstResult = false, concurrency = 3, includeInvalid = false, onProgress } = options;
8
- const sourceInput = await require_utils.normalizeInput(input, fetchFn);
9
- if (sourceInput.content) {
10
- const result = await extractFn({
11
- url: sourceInput.url,
12
- content: sourceInput.content,
13
- headers: sourceInput.headers
14
- });
15
- if (result.isValid) return [result];
16
- }
17
- let siteInput;
18
- if (resolveSiteUrlFn) {
19
- const siteUrl = resolveSiteUrlFn(sourceInput, resolveUrlFn);
20
- if (siteUrl) try {
21
- const response = await fetchFn(siteUrl);
22
- siteInput = {
23
- url: response.url,
24
- content: typeof response.body === "string" ? response.body : "",
25
- headers: response.headers
26
- };
27
- } catch {}
28
- }
29
- const urisByMethod = await require_index.discoverUris(require_utils.normalizeMethodsConfig(sourceInput, siteInput, methods, defaults), fetchFn);
30
- const seen = /* @__PURE__ */ new Set();
31
- const methodGroups = [];
32
- for (const method of require_types.discoverMethodOrder) {
33
- const rawUris = urisByMethod[method];
34
- if (!rawUris || rawUris.length === 0) continue;
35
- const unique = rawUris.map((entry) => {
36
- return require_utils.normalizeUriEntry(entry, resolveUrlFn, sourceInput.url);
37
- }).filter((entry) => {
38
- const key = typeof entry.uri === "string" ? entry.uri : entry.uri.join("\0");
39
- if (seen.has(key)) return false;
40
- seen.add(key);
41
- return true;
42
- });
43
- if (unique.length > 0) methodGroups.push({
44
- method,
45
- entries: unique
46
- });
47
- }
48
- const total = methodGroups.reduce((sum, group) => sum + group.entries.length, 0);
49
- const results = [];
50
- let tested = 0;
51
- let found = 0;
52
- const fetchAndExtract = async (url) => {
53
- try {
54
- const fetchResult = await fetchFn(url);
55
- return await extractFn({
56
- url: fetchResult.url,
57
- content: typeof fetchResult.body === "string" ? fetchResult.body : "",
58
- headers: fetchResult.headers,
59
- status: fetchResult.status
60
- });
61
- } catch (error) {
62
- return {
63
- url,
64
- isValid: false,
65
- error
66
- };
67
- }
68
- };
69
- const processUri = async (entry, method) => {
70
- const alternatives = typeof entry.uri === "string" ? [entry.uri] : entry.uri;
71
- for (const url of alternatives) {
72
- const result = await fetchAndExtract(url);
73
- results.push(entry.hint ? {
74
- ...result,
75
- method,
76
- hint: entry.hint
77
- } : {
78
- ...result,
79
- method
80
- });
81
- tested += 1;
82
- if (result.isValid) found += 1;
83
- onProgress?.({
84
- tested,
85
- total,
86
- found,
87
- current: url
88
- });
89
- if (result.isValid) break;
90
- }
91
- };
92
- for (const { method, entries } of methodGroups) {
93
- const foundBefore = found;
94
- await require_utils$1.processConcurrently(entries, (entry) => processUri(entry, method), {
95
- concurrency,
96
- shouldStop: () => {
97
- return stopOnFirstResult && found > 0;
98
- }
99
- });
100
- if (stopOnFirstMethod && found > foundBefore) break;
101
- }
102
- return includeInvalid ? results : results.filter((result) => result.isValid);
103
- };
104
- //#endregion
105
- exports.discover = discover;
@@ -1,101 +0,0 @@
1
- const require_locales = require("../locales.cjs");
2
- //#region src/common/discover/utils.ts
3
- const normalizeInput = async (input, fetchFn) => {
4
- if (typeof input === "object") return input;
5
- try {
6
- const response = await fetchFn(input);
7
- return {
8
- url: response.url,
9
- content: typeof response.body === "string" ? response.body : void 0,
10
- headers: response.headers
11
- };
12
- } catch {}
13
- return { url: input };
14
- };
15
- const getLinkOfType = (links, rel) => {
16
- return links?.find((link) => link.rel === rel);
17
- };
18
- const getFeedSiteUrl = (parsed) => {
19
- const { format, feed } = parsed;
20
- if (format === "rss" || format === "rdf") return getLinkOfType(feed.atom?.links, "alternate")?.href ?? feed.link;
21
- if (format === "atom") return getLinkOfType(feed.links, "alternate")?.href;
22
- if (format === "json") return feed.home_page_url;
23
- };
24
- const normalizeUriEntry = (entry, resolveUrlFn, baseUrl) => {
25
- if (typeof entry.uri === "string") return {
26
- ...entry,
27
- uri: resolveUrlFn(entry.uri, baseUrl) ?? entry.uri
28
- };
29
- return {
30
- ...entry,
31
- uri: entry.uri.map((uri) => resolveUrlFn(uri, baseUrl) ?? uri)
32
- };
33
- };
34
- const normalizeMethodsConfig = (sourceInput, siteInput, methods, defaults) => {
35
- const resolvedInput = siteInput ?? sourceInput;
36
- const methodsObj = Array.isArray(methods) ? Object.fromEntries(methods.map((method) => [method, true])) : methods;
37
- const methodsConfig = {};
38
- if (methodsObj.platform && defaults.platform) {
39
- if (!resolvedInput.url || resolvedInput.url === "") throw new Error(require_locales.errors.platformMethodRequiresUrl);
40
- const platformOptions = methodsObj.platform === true ? {} : methodsObj.platform;
41
- methodsConfig.platform = {
42
- content: resolvedInput.content,
43
- headers: resolvedInput.headers,
44
- options: {
45
- ...defaults.platform,
46
- ...platformOptions,
47
- baseUrl: resolvedInput.url
48
- }
49
- };
50
- }
51
- if (methodsObj.feed && defaults.feed) {
52
- if (sourceInput.content === void 0) throw new Error(require_locales.errors.feedMethodRequiresContent);
53
- const feedOptions = methodsObj.feed === true ? {} : methodsObj.feed;
54
- methodsConfig.feed = {
55
- content: sourceInput.content,
56
- options: {
57
- ...defaults.feed,
58
- ...feedOptions
59
- }
60
- };
61
- }
62
- if (methodsObj.html && defaults.html) {
63
- if (resolvedInput.content === void 0) throw new Error(require_locales.errors.htmlMethodRequiresContent);
64
- const htmlOptions = methodsObj.html === true ? {} : methodsObj.html;
65
- methodsConfig.html = {
66
- html: resolvedInput.content,
67
- options: {
68
- ...defaults.html,
69
- ...htmlOptions,
70
- baseUrl: resolvedInput.url
71
- }
72
- };
73
- }
74
- if (methodsObj.headers && defaults.headers) {
75
- if (resolvedInput.headers === void 0) throw new Error(require_locales.errors.headersMethodRequiresHeaders);
76
- const headersOptions = methodsObj.headers === true ? {} : methodsObj.headers;
77
- methodsConfig.headers = {
78
- headers: resolvedInput.headers,
79
- options: {
80
- ...defaults.headers,
81
- ...headersOptions,
82
- baseUrl: resolvedInput.url
83
- }
84
- };
85
- }
86
- if (methodsObj.guess && defaults.guess) {
87
- if (!resolvedInput.url || resolvedInput.url === "") throw new Error(require_locales.errors.guessMethodRequiresUrl);
88
- const guessOptions = methodsObj.guess === true ? {} : methodsObj.guess;
89
- methodsConfig.guess = { options: {
90
- ...defaults.guess,
91
- ...guessOptions,
92
- baseUrl: resolvedInput.url
93
- } };
94
- }
95
- return methodsConfig;
96
- };
97
- //#endregion
98
- exports.getFeedSiteUrl = getFeedSiteUrl;
99
- exports.normalizeInput = normalizeInput;
100
- exports.normalizeMethodsConfig = normalizeMethodsConfig;
101
- exports.normalizeUriEntry = normalizeUriEntry;
@@ -1,6 +0,0 @@
1
- import { parseFeed } from "feedsmith";
2
-
3
- //#region src/common/discover/utils.d.ts
4
- declare const getFeedSiteUrl: (parsed: ReturnType<typeof parseFeed>) => string | undefined;
5
- //#endregion
6
- export { getFeedSiteUrl };
@@ -1,144 +0,0 @@
1
- //#region src/common/locales.json
2
- var errors = {
3
- "platformMethodRequiresUrl": "Platform method requires url to be provided in input",
4
- "feedMethodRequiresContent": "Feed method requires content to be provided in input",
5
- "htmlMethodRequiresContent": "HTML method requires content to be provided in input",
6
- "headersMethodRequiresHeaders": "Headers method requires headers to be provided in input",
7
- "guessMethodRequiresUrl": "Guess method requires url to be provided in input"
8
- };
9
- var hints = {
10
- "youtube:all": "All uploads",
11
- "youtube:videos": "Videos",
12
- "youtube:shorts": "Shorts",
13
- "youtube:live": "Live streams",
14
- "youtube:popular-videos": "Popular videos",
15
- "youtube:popular-shorts": "Popular shorts",
16
- "youtube:popular-live": "Popular live streams",
17
- "youtube:member-videos": "Member videos",
18
- "youtube:member-shorts": "Member shorts",
19
- "youtube:member-live": "Member live streams",
20
- "youtube:playlist": "Playlist",
21
- "github:activity": "Activity",
22
- "github:releases": "Releases",
23
- "github:commits": "Commits",
24
- "github:tags": "Tags",
25
- "github:wiki": "Wiki",
26
- "github:discussions": "Discussions",
27
- "github:branch-commits": "Branch commits",
28
- "github:file-history": "File history",
29
- "reddit:posts": "Posts",
30
- "reddit:post-comments": "Post comments",
31
- "reddit:comments": "Comments",
32
- "reddit:multireddit": "Multireddit",
33
- "gitlab:activity": "Activity",
34
- "gitlab:releases": "Releases",
35
- "gitlab:tags": "Tags",
36
- "codeberg:activity": "Activity",
37
- "codeberg:releases": "Releases",
38
- "codeberg:tags": "Tags",
39
- "codeberg:branch-commits": "Branch commits",
40
- "codeberg:file-history": "File history",
41
- "wordpress:category": "Category",
42
- "wordpress:tag": "Tag",
43
- "wordpress:author": "Author",
44
- "wordpress:posts-rss2": "Posts (RSS 2.0)",
45
- "wordpress:posts-rss2-alt": "Posts (RSS 2.0)",
46
- "wordpress:posts-rdf": "Posts (RDF)",
47
- "wordpress:posts-atom": "Posts (Atom)",
48
- "wordpress:comments": "Comments",
49
- "wordpress:comments-rss2": "Comments (RSS 2.0)",
50
- "wordpress:comments-rdf": "Comments (RDF)",
51
- "wordpress:comments-atom": "Comments (Atom)",
52
- "blogspot:label": "Label",
53
- "blogspot:posts-atom": "Posts (Atom)",
54
- "blogspot:posts-rss": "Posts (RSS)",
55
- "behance:portfolio": "Portfolio",
56
- "behance:appreciated": "Appreciated",
57
- "bluesky:posts": "Posts",
58
- "dailymotion:videos": "Videos",
59
- "dailymotion:playlist": "Playlist",
60
- "deviantart:tag": "Tag",
61
- "deviantart:favorites": "Favorites",
62
- "deviantart:gallery": "Gallery",
63
- "deviantart:deviations": "Deviations",
64
- "devto:posts": "Posts",
65
- "devto:tag": "Tag",
66
- "github-gist:gists": "Gists",
67
- "github-gist:starred": "Starred",
68
- "kickstarter:updates": "Updates",
69
- "kickstarter:projects": "Projects",
70
- "letterboxd:diary": "Diary",
71
- "lobsters:tag": "Tag",
72
- "lobsters:domain": "Domain",
73
- "lobsters:stories": "Stories",
74
- "lobsters:top": "Top stories",
75
- "lobsters:newest": "Newest",
76
- "lobsters:comments": "Comments",
77
- "mastodon:posts": "Posts",
78
- "mastodon:tag": "Tag",
79
- "medium:posts": "Posts",
80
- "medium:tag": "Tag",
81
- "medium:tagged": "Tagged",
82
- "medium:publication": "Publication",
83
- "pinterest:pins": "Pins",
84
- "producthunt:topic": "Topic",
85
- "producthunt:category": "Category",
86
- "producthunt:products": "Products",
87
- "soundcloud:tracks": "Tracks",
88
- "stackexchange:tag": "Tag",
89
- "stackexchange:question": "Question",
90
- "stackexchange:user": "User",
91
- "steam:news": "News",
92
- "steam:group": "Group",
93
- "substack:newsletter": "Newsletter",
94
- "tumblr:posts": "Posts",
95
- "tumblr:tag": "Tag",
96
- "goodreads:updates": "Updates",
97
- "goodreads:reviews": "Reviews",
98
- "sourceforge:activity": "Activity",
99
- "vimeo:videos": "Videos",
100
- "vimeo:likes": "Likes",
101
- "vimeo:channel": "Channel",
102
- "vimeo:group": "Group",
103
- "hashnode:blog": "Blog",
104
- "paragraph:blog": "Blog",
105
- "hatenablog:posts-rss": "Posts (RSS)",
106
- "hatenablog:posts-atom": "Posts (Atom)",
107
- "hatenablog:category-rss": "Category (RSS)",
108
- "hatenablog:category-atom": "Category (Atom)",
109
- "hatenablog:author-rss": "Author (RSS)",
110
- "hatenablog:author-atom": "Author (Atom)",
111
- "itchio:devlog": "Devlog",
112
- "itchio:games": "Games",
113
- "itchio:tag": "Tag",
114
- "itchio:section": "Section",
115
- "itchio:featured": "Featured",
116
- "itchio:new": "New",
117
- "itchio:sales": "Sales",
118
- "csdn:blog": "Blog",
119
- "douban:interests": "Interests",
120
- "douban:reviews": "Reviews",
121
- "douban:notes": "Notes",
122
- "douban:subjectReviews": "Subject reviews",
123
- "v2ex:index": "Index",
124
- "v2ex:node": "Node",
125
- "v2ex:member": "Member",
126
- "v2ex:tab": "Tab",
127
- "ximalaya:album": "Album",
128
- "lemmy:community": "Community",
129
- "lemmy:user": "User",
130
- "apple-podcasts:podcast": "Podcast"
131
- };
132
- //#endregion
133
- Object.defineProperty(exports, "errors", {
134
- enumerable: true,
135
- get: function() {
136
- return errors;
137
- }
138
- });
139
- Object.defineProperty(exports, "hints", {
140
- enumerable: true,
141
- get: function() {
142
- return hints;
143
- }
144
- });
@@ -1,10 +0,0 @@
1
- //#region src/common/types.ts
2
- const discoverMethodOrder = [
3
- "platform",
4
- "feed",
5
- "html",
6
- "headers",
7
- "guess"
8
- ];
9
- //#endregion
10
- exports.discoverMethodOrder = discoverMethodOrder;
@@ -1,89 +0,0 @@
1
- import { FeedMethodOptions } from "./uris/feed/types.cjs";
2
- import { GuessMethodOptions } from "./uris/guess/types.cjs";
3
- import { HeadersMethodOptions } from "./uris/headers/types.cjs";
4
- import { HtmlMethodOptions } from "./uris/html/types.cjs";
5
- import { PlatformMethodOptions } from "./uris/platform/types.cjs";
6
-
7
- //#region src/common/types.d.ts
8
- type MaybePromise<T> = T | Promise<T>;
9
- type UriEntry = string | Array<string>;
10
- type DiscoverUriHint = {
11
- key: string;
12
- label: string;
13
- };
14
- type DiscoverUriEntry = {
15
- uri: UriEntry;
16
- hint?: DiscoverUriHint;
17
- };
18
- declare const discoverMethodOrder: readonly ["platform", "feed", "html", "headers", "guess"];
19
- type DiscoverMethod = (typeof discoverMethodOrder)[number];
20
- type LinkSelector = {
21
- rel: string;
22
- types?: Array<string>;
23
- };
24
- type DiscoverResolveUrlFn = (url: string, baseUrl: string | undefined) => string | undefined;
25
- type DiscoverResolveSiteUrlFn = (input: DiscoverInputObject, resolveUrlFn: DiscoverResolveUrlFn) => string | undefined;
26
- type DiscoverFetchFnOptions = {
27
- method?: 'GET' | 'HEAD';
28
- headers?: Record<string, string>;
29
- };
30
- type DiscoverFetchFnResponse = {
31
- headers: Headers;
32
- body: string | ReadableStream<Uint8Array>;
33
- url: string;
34
- status: number;
35
- statusText: string;
36
- };
37
- type DiscoverFetchFn = (url: string, options?: DiscoverFetchFnOptions) => MaybePromise<DiscoverFetchFnResponse>;
38
- type DiscoverProgress = {
39
- tested: number;
40
- total: number;
41
- found: number;
42
- current: string;
43
- };
44
- type DiscoverOnProgressFn = (progress: DiscoverProgress) => void;
45
- type DiscoverResult<TValid = object> = ({
46
- url: string;
47
- isValid: true;
48
- method?: DiscoverMethod;
49
- hint?: DiscoverUriHint;
50
- } & TValid) | {
51
- url: string;
52
- isValid: false;
53
- method?: DiscoverMethod;
54
- hint?: DiscoverUriHint;
55
- error?: unknown;
56
- };
57
- type DiscoverExtractFn<TValid> = (input: {
58
- url: string;
59
- content: string;
60
- headers?: Headers;
61
- status?: number;
62
- }) => MaybePromise<DiscoverResult<TValid>>;
63
- type DiscoverInputObject = {
64
- url: string;
65
- content?: string;
66
- headers?: Headers;
67
- };
68
- type DiscoverInput = string | DiscoverInputObject;
69
- type DiscoverMethodsConfig<TMethods extends DiscoverMethod = DiscoverMethod> = Array<TMethods> | Pick<{
70
- platform?: true | Partial<PlatformMethodOptions>;
71
- feed?: true | Partial<FeedMethodOptions>;
72
- html?: true | Partial<Omit<HtmlMethodOptions, 'baseUrl'>>;
73
- headers?: true | Partial<Omit<HeadersMethodOptions, 'baseUrl'>>;
74
- guess?: true | Partial<Omit<GuessMethodOptions, 'baseUrl'>>;
75
- }, TMethods>;
76
- type DiscoverOptions<TValid, TMethods extends DiscoverMethod = DiscoverMethod> = {
77
- methods?: DiscoverMethodsConfig<TMethods>;
78
- fetchFn?: DiscoverFetchFn;
79
- extractFn?: DiscoverExtractFn<TValid>;
80
- resolveUrlFn?: DiscoverResolveUrlFn;
81
- resolveSiteUrlFn?: DiscoverResolveSiteUrlFn;
82
- stopOnFirstMethod?: boolean;
83
- stopOnFirstResult?: boolean;
84
- concurrency?: number;
85
- onProgress?: DiscoverOnProgressFn;
86
- includeInvalid?: boolean;
87
- };
88
- //#endregion
89
- export { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethod, DiscoverMethodsConfig, DiscoverOnProgressFn, DiscoverOptions, DiscoverProgress, DiscoverResolveSiteUrlFn, DiscoverResolveUrlFn, DiscoverResult, DiscoverUriEntry, DiscoverUriHint, LinkSelector, MaybePromise, UriEntry };
@@ -1,12 +0,0 @@
1
- const require_utils = require("../../utils.cjs");
2
- let feedsmith = require("feedsmith");
3
- //#region src/common/uris/feed/index.ts
4
- const discoverUrisFromFeed = (content, options) => {
5
- try {
6
- const result = (0, feedsmith.parseFeed)(content);
7
- return require_utils.omitEmpty(options.extractUrls(result));
8
- } catch {}
9
- return [];
10
- };
11
- //#endregion
12
- exports.discoverUrisFromFeed = discoverUrisFromFeed;
@@ -1,6 +0,0 @@
1
- import { FeedMethodOptions } from "./types.cjs";
2
-
3
- //#region src/common/uris/feed/index.d.ts
4
- declare const discoverUrisFromFeed: (content: string, options: FeedMethodOptions) => Array<string>;
5
- //#endregion
6
- export { discoverUrisFromFeed };
@@ -1,9 +0,0 @@
1
- import { parseFeed } from "feedsmith";
2
-
3
- //#region src/common/uris/feed/types.d.ts
4
- type FeedMethodData = ReturnType<typeof parseFeed>;
5
- type FeedMethodOptions = {
6
- extractUrls: (params: FeedMethodData) => Array<string>;
7
- };
8
- //#endregion
9
- export { FeedMethodOptions };
@@ -1,8 +0,0 @@
1
- const require_utils = require("./utils.cjs");
2
- //#region src/common/uris/guess/index.ts
3
- const discoverUrisFromGuess = (options) => {
4
- const { baseUrl, uris, additionalBaseUrls = [] } = options;
5
- return require_utils.generateUrlCombinations([baseUrl, ...additionalBaseUrls], uris);
6
- };
7
- //#endregion
8
- exports.discoverUrisFromGuess = discoverUrisFromGuess;
@@ -1,7 +0,0 @@
1
- import { GuessMethodOptions } from "./types.cjs";
2
- import { UriEntry } from "../../types.cjs";
3
-
4
- //#region src/common/uris/guess/index.d.ts
5
- declare const discoverUrisFromGuess: (options: GuessMethodOptions) => Array<UriEntry>;
6
- //#endregion
7
- export { discoverUrisFromGuess };
@@ -1,10 +0,0 @@
1
- import { UriEntry } from "../../types.cjs";
2
-
3
- //#region src/common/uris/guess/types.d.ts
4
- type GuessMethodOptions = {
5
- baseUrl: string;
6
- uris: Array<UriEntry>;
7
- additionalBaseUrls?: Array<string>;
8
- };
9
- //#endregion
10
- export { GuessMethodOptions };