feedscout 1.7.1 → 1.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/dist/blogrolls/extractors.cjs +2 -2
  2. package/dist/blogrolls/extractors.d.cts +2 -2
  3. package/dist/blogrolls/extractors.d.ts +2 -2
  4. package/dist/blogrolls/extractors.js +2 -2
  5. package/dist/blogrolls/index.cjs +9 -10
  6. package/dist/blogrolls/index.js +4 -5
  7. package/dist/blogrolls.cjs +1 -1
  8. package/dist/blogrolls.d.cts +2 -2
  9. package/dist/blogrolls.d.ts +2 -2
  10. package/dist/blogrolls.js +2 -2
  11. package/dist/common/discover/defaults.cjs +46 -0
  12. package/dist/common/discover/defaults.d.cts +6 -0
  13. package/dist/common/discover/defaults.d.ts +6 -0
  14. package/dist/common/discover/defaults.js +44 -0
  15. package/dist/common/discover/index.cjs +7 -7
  16. package/dist/common/discover/index.js +2 -2
  17. package/dist/common/discover/utils.cjs +9 -35
  18. package/dist/common/discover/utils.d.cts +1 -3
  19. package/dist/common/discover/utils.d.ts +1 -3
  20. package/dist/common/discover/utils.js +10 -34
  21. package/dist/common/locales.cjs +11 -4
  22. package/dist/common/locales.js +11 -4
  23. package/dist/common/types.d.cts +5 -4
  24. package/dist/common/types.d.ts +5 -4
  25. package/dist/common/uris/headers/index.cjs +1 -1
  26. package/dist/common/uris/headers/index.js +1 -1
  27. package/dist/common/uris/platform/types.d.cts +2 -2
  28. package/dist/common/uris/platform/types.d.ts +2 -2
  29. package/dist/common/utils.cjs +0 -8
  30. package/dist/common/utils.js +1 -8
  31. package/dist/favicons/defaults.cjs +6 -1
  32. package/dist/favicons/defaults.js +6 -1
  33. package/dist/favicons/extractors.cjs +2 -2
  34. package/dist/favicons/extractors.d.cts +2 -2
  35. package/dist/favicons/extractors.d.ts +2 -2
  36. package/dist/favicons/extractors.js +2 -2
  37. package/dist/favicons/index.cjs +11 -12
  38. package/dist/favicons/index.js +4 -5
  39. package/dist/favicons/platform/handlers/bluesky.cjs +1 -1
  40. package/dist/favicons/platform/handlers/bluesky.js +1 -1
  41. package/dist/favicons/platform/handlers/codeberg.cjs +4 -3
  42. package/dist/favicons/platform/handlers/codeberg.js +4 -3
  43. package/dist/favicons/platform/handlers/devto.cjs +31 -0
  44. package/dist/favicons/platform/handlers/devto.js +31 -0
  45. package/dist/favicons/platform/handlers/github.cjs +4 -3
  46. package/dist/favicons/platform/handlers/github.js +4 -3
  47. package/dist/favicons/platform/handlers/githubGist.cjs +4 -3
  48. package/dist/favicons/platform/handlers/githubGist.js +4 -3
  49. package/dist/favicons/platform/handlers/gitlab.cjs +38 -0
  50. package/dist/favicons/platform/handlers/gitlab.js +38 -0
  51. package/dist/favicons/platform/handlers/lobsters.cjs +3 -3
  52. package/dist/favicons/platform/handlers/lobsters.js +3 -3
  53. package/dist/favicons/platform/handlers/mastodon.cjs +5 -3
  54. package/dist/favicons/platform/handlers/mastodon.js +5 -3
  55. package/dist/favicons/platform/handlers/reddit.cjs +10 -8
  56. package/dist/favicons/platform/handlers/reddit.js +10 -8
  57. package/dist/favicons.cjs +1 -1
  58. package/dist/favicons.d.cts +2 -2
  59. package/dist/favicons.d.ts +2 -2
  60. package/dist/favicons.js +2 -2
  61. package/dist/feeds/defaults.cjs +4 -2
  62. package/dist/feeds/defaults.js +4 -2
  63. package/dist/feeds/extractors.cjs +6 -6
  64. package/dist/feeds/extractors.d.cts +2 -2
  65. package/dist/feeds/extractors.d.ts +2 -2
  66. package/dist/feeds/extractors.js +4 -4
  67. package/dist/feeds/index.cjs +9 -10
  68. package/dist/feeds/index.js +4 -5
  69. package/dist/feeds/platform/handlers/applePodcasts.cjs +26 -0
  70. package/dist/feeds/platform/handlers/applePodcasts.js +26 -0
  71. package/dist/feeds/platform/handlers/behance.cjs +2 -2
  72. package/dist/feeds/platform/handlers/behance.js +2 -2
  73. package/dist/feeds/platform/handlers/dailymotion.cjs +4 -4
  74. package/dist/feeds/platform/handlers/dailymotion.js +4 -4
  75. package/dist/feeds/platform/handlers/devto.cjs +6 -4
  76. package/dist/feeds/platform/handlers/devto.js +5 -5
  77. package/dist/feeds/platform/handlers/gitlab.cjs +4 -0
  78. package/dist/feeds/platform/handlers/gitlab.js +1 -1
  79. package/dist/feeds/platform/handlers/hatenablog.cjs +4 -4
  80. package/dist/feeds/platform/handlers/hatenablog.js +4 -4
  81. package/dist/feeds/platform/handlers/itchio.cjs +10 -10
  82. package/dist/feeds/platform/handlers/itchio.js +10 -10
  83. package/dist/feeds/platform/handlers/lobsters.cjs +8 -8
  84. package/dist/feeds/platform/handlers/lobsters.js +8 -8
  85. package/dist/feeds/platform/handlers/mastodon.cjs +4 -4
  86. package/dist/feeds/platform/handlers/mastodon.js +4 -4
  87. package/dist/feeds/platform/handlers/medium.cjs +10 -10
  88. package/dist/feeds/platform/handlers/medium.js +10 -10
  89. package/dist/feeds/platform/handlers/paragraph.cjs +2 -2
  90. package/dist/feeds/platform/handlers/paragraph.js +2 -2
  91. package/dist/feeds/platform/handlers/producthunt.cjs +4 -4
  92. package/dist/feeds/platform/handlers/producthunt.js +4 -4
  93. package/dist/feeds/platform/handlers/reddit.cjs +10 -10
  94. package/dist/feeds/platform/handlers/reddit.js +10 -10
  95. package/dist/feeds/platform/handlers/stackExchange.cjs +6 -6
  96. package/dist/feeds/platform/handlers/stackExchange.js +6 -6
  97. package/dist/feeds/platform/handlers/steam.cjs +4 -4
  98. package/dist/feeds/platform/handlers/steam.js +4 -4
  99. package/dist/feeds/platform/handlers/substack.cjs +10 -3
  100. package/dist/feeds/platform/handlers/substack.js +11 -4
  101. package/dist/feeds/platform/handlers/tumblr.cjs +2 -2
  102. package/dist/feeds/platform/handlers/tumblr.js +2 -2
  103. package/dist/feeds/platform/handlers/v2ex.cjs +4 -4
  104. package/dist/feeds/platform/handlers/v2ex.js +4 -4
  105. package/dist/feeds/platform/handlers/vimeo.cjs +2 -2
  106. package/dist/feeds/platform/handlers/vimeo.js +2 -2
  107. package/dist/feeds/platform/handlers/wordpress.cjs +6 -6
  108. package/dist/feeds/platform/handlers/wordpress.js +6 -6
  109. package/dist/feeds/platform/handlers/ximalaya.cjs +2 -2
  110. package/dist/feeds/platform/handlers/ximalaya.js +2 -2
  111. package/dist/feeds/platform/handlers/youtube.cjs +36 -21
  112. package/dist/feeds/platform/handlers/youtube.js +36 -21
  113. package/dist/feeds.cjs +1 -1
  114. package/dist/feeds.d.cts +2 -2
  115. package/dist/feeds.d.ts +2 -2
  116. package/dist/feeds.js +2 -2
  117. package/dist/hubs/discover/index.cjs +4 -5
  118. package/dist/hubs/discover/index.js +3 -4
  119. package/dist/hubs/feed/index.cjs +2 -2
  120. package/dist/hubs/feed/index.js +2 -2
  121. package/dist/hubs/headers/index.cjs +2 -2
  122. package/dist/hubs/headers/index.js +2 -2
  123. package/dist/hubs/html/index.cjs +2 -2
  124. package/dist/hubs/html/index.js +2 -2
  125. package/dist/index.cjs +2 -1
  126. package/dist/index.d.cts +2 -1
  127. package/dist/index.d.ts +2 -1
  128. package/dist/index.js +2 -1
  129. package/package.json +3 -2
  130. package/dist/hubs/discover/utils.cjs +0 -18
  131. package/dist/hubs/discover/utils.js +0 -18
@@ -1,6 +1,6 @@
1
1
  let feedsmith = require("feedsmith");
2
2
  //#region src/blogrolls/extractors.ts
3
- const defaultExtractor = ({ content, url }) => {
3
+ const defaultExtractFn = ({ content, url }) => {
4
4
  if (!content) return {
5
5
  url,
6
6
  isValid: false
@@ -18,4 +18,4 @@ const defaultExtractor = ({ content, url }) => {
18
18
  };
19
19
  };
20
20
  //#endregion
21
- exports.defaultExtractor = defaultExtractor;
21
+ exports.defaultExtractFn = defaultExtractFn;
@@ -2,6 +2,6 @@ import { DiscoverExtractFn } from "../common/types.cjs";
2
2
  import { BlogrollResult } from "./types.cjs";
3
3
 
4
4
  //#region src/blogrolls/extractors.d.ts
5
- declare const defaultExtractor: DiscoverExtractFn<BlogrollResult>;
5
+ declare const defaultExtractFn: DiscoverExtractFn<BlogrollResult>;
6
6
  //#endregion
7
- export { defaultExtractor };
7
+ export { defaultExtractFn };
@@ -2,6 +2,6 @@ import { DiscoverExtractFn } from "../common/types.js";
2
2
  import { BlogrollResult } from "./types.js";
3
3
 
4
4
  //#region src/blogrolls/extractors.d.ts
5
- declare const defaultExtractor: DiscoverExtractFn<BlogrollResult>;
5
+ declare const defaultExtractFn: DiscoverExtractFn<BlogrollResult>;
6
6
  //#endregion
7
- export { defaultExtractor };
7
+ export { defaultExtractFn };
@@ -1,6 +1,6 @@
1
1
  import { parseOpml } from "feedsmith";
2
2
  //#region src/blogrolls/extractors.ts
3
- const defaultExtractor = ({ content, url }) => {
3
+ const defaultExtractFn = ({ content, url }) => {
4
4
  if (!content) return {
5
5
  url,
6
6
  isValid: false
@@ -18,4 +18,4 @@ const defaultExtractor = ({ content, url }) => {
18
18
  };
19
19
  };
20
20
  //#endregion
21
- export { defaultExtractor };
21
+ export { defaultExtractFn };
@@ -1,7 +1,6 @@
1
- const require_utils = require("../common/utils.cjs");
2
- const require_utils$1 = require("../common/discover/utils.cjs");
1
+ const require_defaults = require("../common/discover/defaults.cjs");
3
2
  const require_index = require("../common/discover/index.cjs");
4
- const require_defaults = require("./defaults.cjs");
3
+ const require_defaults$1 = require("./defaults.cjs");
5
4
  const require_extractors = require("./extractors.cjs");
6
5
  //#region src/blogrolls/index.ts
7
6
  const discoverBlogrolls = (input, options = {}) => {
@@ -12,14 +11,14 @@ const discoverBlogrolls = (input, options = {}) => {
12
11
  "headers",
13
12
  "guess"
14
13
  ],
15
- fetchFn: options.fetchFn ?? require_utils$1.defaultFetchFn,
16
- extractFn: options.extractFn ?? require_extractors.defaultExtractor,
17
- resolveUrlFn: options.resolveUrlFn ?? require_utils.resolveUrl,
18
- resolveSiteUrlFn: options.resolveSiteUrlFn ?? require_utils$1.defaultResolveSiteUrlFn
14
+ fetchFn: options.fetchFn ?? require_defaults.defaultFetchFn,
15
+ extractFn: options.extractFn ?? require_extractors.defaultExtractFn,
16
+ resolveUrlFn: options.resolveUrlFn ?? require_defaults.defaultResolveUrlFn,
17
+ resolveSiteUrlFn: options.resolveSiteUrlFn ?? require_defaults.defaultResolveSiteUrlFn
19
18
  }, {
20
- html: require_defaults.defaultHtmlOptions,
21
- headers: require_defaults.defaultHeadersOptions,
22
- guess: require_defaults.defaultGuessOptions
19
+ html: require_defaults$1.defaultHtmlOptions,
20
+ headers: require_defaults$1.defaultHeadersOptions,
21
+ guess: require_defaults$1.defaultGuessOptions
23
22
  });
24
23
  };
25
24
  //#endregion
@@ -1,8 +1,7 @@
1
- import { resolveUrl } from "../common/utils.js";
2
- import { defaultFetchFn, defaultResolveSiteUrlFn } from "../common/discover/utils.js";
1
+ import { defaultFetchFn, defaultResolveSiteUrlFn, defaultResolveUrlFn } from "../common/discover/defaults.js";
3
2
  import { discover } from "../common/discover/index.js";
4
3
  import { defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions } from "./defaults.js";
5
- import { defaultExtractor } from "./extractors.js";
4
+ import { defaultExtractFn } from "./extractors.js";
6
5
  //#region src/blogrolls/index.ts
7
6
  const discoverBlogrolls = (input, options = {}) => {
8
7
  return discover(input, {
@@ -13,8 +12,8 @@ const discoverBlogrolls = (input, options = {}) => {
13
12
  "guess"
14
13
  ],
15
14
  fetchFn: options.fetchFn ?? defaultFetchFn,
16
- extractFn: options.extractFn ?? defaultExtractor,
17
- resolveUrlFn: options.resolveUrlFn ?? resolveUrl,
15
+ extractFn: options.extractFn ?? defaultExtractFn,
16
+ resolveUrlFn: options.resolveUrlFn ?? defaultResolveUrlFn,
18
17
  resolveSiteUrlFn: options.resolveSiteUrlFn ?? defaultResolveSiteUrlFn
19
18
  }, {
20
19
  html: defaultHtmlOptions,
@@ -2,7 +2,7 @@ Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
2
2
  const require_defaults = require("./blogrolls/defaults.cjs");
3
3
  const require_extractors = require("./blogrolls/extractors.cjs");
4
4
  exports.anchorLabels = require_defaults.anchorLabels;
5
- exports.defaultExtractor = require_extractors.defaultExtractor;
5
+ exports.defaultExtractFn = require_extractors.defaultExtractFn;
6
6
  exports.defaultGuessOptions = require_defaults.defaultGuessOptions;
7
7
  exports.defaultHeadersOptions = require_defaults.defaultHeadersOptions;
8
8
  exports.defaultHtmlOptions = require_defaults.defaultHtmlOptions;
@@ -1,4 +1,4 @@
1
1
  import { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal } from "./blogrolls/defaults.cjs";
2
2
  import { BlogrollResult } from "./blogrolls/types.cjs";
3
- import { defaultExtractor } from "./blogrolls/extractors.cjs";
4
- export { BlogrollResult, anchorLabels, defaultExtractor, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
3
+ import { defaultExtractFn } from "./blogrolls/extractors.cjs";
4
+ export { BlogrollResult, anchorLabels, defaultExtractFn, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
@@ -1,4 +1,4 @@
1
1
  import { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal } from "./blogrolls/defaults.js";
2
2
  import { BlogrollResult } from "./blogrolls/types.js";
3
- import { defaultExtractor } from "./blogrolls/extractors.js";
4
- export { BlogrollResult, anchorLabels, defaultExtractor, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
3
+ import { defaultExtractFn } from "./blogrolls/extractors.js";
4
+ export { BlogrollResult, anchorLabels, defaultExtractFn, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
package/dist/blogrolls.js CHANGED
@@ -1,3 +1,3 @@
1
1
  import { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal } from "./blogrolls/defaults.js";
2
- import { defaultExtractor } from "./blogrolls/extractors.js";
3
- export { anchorLabels, defaultExtractor, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
2
+ import { defaultExtractFn } from "./blogrolls/extractors.js";
3
+ export { anchorLabels, defaultExtractFn, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
@@ -0,0 +1,46 @@
1
+ const require_utils = require("./utils.cjs");
2
+ let feedsmith = require("feedsmith");
3
+ //#region src/common/discover/defaults.ts
4
+ const defaultFetchFn = async (url, options) => {
5
+ const response = await fetch(url, {
6
+ method: options?.method ?? "GET",
7
+ headers: options?.headers
8
+ });
9
+ return {
10
+ headers: response.headers,
11
+ body: await response.text(),
12
+ url: response.url,
13
+ status: response.status,
14
+ statusText: response.statusText
15
+ };
16
+ };
17
+ const defaultResolveUrlFn = (url, baseUrl) => {
18
+ try {
19
+ return new URL(url, baseUrl).href;
20
+ } catch {
21
+ return;
22
+ }
23
+ };
24
+ const defaultResolveSiteUrlFn = (input, resolveUrlFn) => {
25
+ if (!input.content) return;
26
+ try {
27
+ let siteUrl = require_utils.getFeedSiteUrl((0, feedsmith.parseFeed)(input.content));
28
+ if (siteUrl) {
29
+ siteUrl = resolveUrlFn(siteUrl, input.url);
30
+ if (siteUrl) {
31
+ const parsed = new URL(siteUrl);
32
+ parsed.hash = "";
33
+ siteUrl = parsed.href;
34
+ }
35
+ }
36
+ if (!siteUrl || siteUrl === new URL(input.url).href) try {
37
+ siteUrl = new URL(input.url).origin;
38
+ } catch {}
39
+ if (siteUrl && new URL(siteUrl).href === new URL(input.url).href) return;
40
+ return siteUrl;
41
+ } catch {}
42
+ };
43
+ //#endregion
44
+ exports.defaultFetchFn = defaultFetchFn;
45
+ exports.defaultResolveSiteUrlFn = defaultResolveSiteUrlFn;
46
+ exports.defaultResolveUrlFn = defaultResolveUrlFn;
@@ -0,0 +1,6 @@
1
+ import { DiscoverResolveSiteUrlFn } from "../types.cjs";
2
+
3
+ //#region src/common/discover/defaults.d.ts
4
+ declare const defaultResolveSiteUrlFn: DiscoverResolveSiteUrlFn;
5
+ //#endregion
6
+ export { defaultResolveSiteUrlFn };
@@ -0,0 +1,6 @@
1
+ import { DiscoverResolveSiteUrlFn } from "../types.js";
2
+
3
+ //#region src/common/discover/defaults.d.ts
4
+ declare const defaultResolveSiteUrlFn: DiscoverResolveSiteUrlFn;
5
+ //#endregion
6
+ export { defaultResolveSiteUrlFn };
@@ -0,0 +1,44 @@
1
+ import { getFeedSiteUrl } from "./utils.js";
2
+ import { parseFeed } from "feedsmith";
3
+ //#region src/common/discover/defaults.ts
4
+ const defaultFetchFn = async (url, options) => {
5
+ const response = await fetch(url, {
6
+ method: options?.method ?? "GET",
7
+ headers: options?.headers
8
+ });
9
+ return {
10
+ headers: response.headers,
11
+ body: await response.text(),
12
+ url: response.url,
13
+ status: response.status,
14
+ statusText: response.statusText
15
+ };
16
+ };
17
+ const defaultResolveUrlFn = (url, baseUrl) => {
18
+ try {
19
+ return new URL(url, baseUrl).href;
20
+ } catch {
21
+ return;
22
+ }
23
+ };
24
+ const defaultResolveSiteUrlFn = (input, resolveUrlFn) => {
25
+ if (!input.content) return;
26
+ try {
27
+ let siteUrl = getFeedSiteUrl(parseFeed(input.content));
28
+ if (siteUrl) {
29
+ siteUrl = resolveUrlFn(siteUrl, input.url);
30
+ if (siteUrl) {
31
+ const parsed = new URL(siteUrl);
32
+ parsed.hash = "";
33
+ siteUrl = parsed.href;
34
+ }
35
+ }
36
+ if (!siteUrl || siteUrl === new URL(input.url).href) try {
37
+ siteUrl = new URL(input.url).origin;
38
+ } catch {}
39
+ if (siteUrl && new URL(siteUrl).href === new URL(input.url).href) return;
40
+ return siteUrl;
41
+ } catch {}
42
+ };
43
+ //#endregion
44
+ export { defaultFetchFn, defaultResolveSiteUrlFn, defaultResolveUrlFn };
@@ -1,11 +1,11 @@
1
+ const require_utils = require("./utils.cjs");
1
2
  const require_types = require("../types.cjs");
2
- const require_utils = require("../utils.cjs");
3
+ const require_utils$1 = require("../utils.cjs");
3
4
  const require_index = require("../uris/index.cjs");
4
- const require_utils$1 = require("./utils.cjs");
5
5
  //#region src/common/discover/index.ts
6
6
  const discover = async (input, options, defaults) => {
7
7
  const { methods, fetchFn, extractFn, resolveUrlFn, resolveSiteUrlFn, stopOnFirstMethod = false, stopOnFirstResult = false, concurrency = 3, includeInvalid = false, onProgress } = options;
8
- const sourceInput = await require_utils$1.normalizeInput(input, fetchFn);
8
+ const sourceInput = await require_utils.normalizeInput(input, fetchFn);
9
9
  if (sourceInput.content) {
10
10
  const result = await extractFn({
11
11
  url: sourceInput.url,
@@ -16,7 +16,7 @@ const discover = async (input, options, defaults) => {
16
16
  }
17
17
  let siteInput;
18
18
  if (resolveSiteUrlFn) {
19
- const siteUrl = resolveSiteUrlFn(sourceInput);
19
+ const siteUrl = resolveSiteUrlFn(sourceInput, resolveUrlFn);
20
20
  if (siteUrl) try {
21
21
  const response = await fetchFn(siteUrl);
22
22
  siteInput = {
@@ -26,14 +26,14 @@ const discover = async (input, options, defaults) => {
26
26
  };
27
27
  } catch {}
28
28
  }
29
- const urisByMethod = await require_index.discoverUris(require_utils$1.normalizeMethodsConfig(sourceInput, siteInput, methods, defaults), fetchFn);
29
+ const urisByMethod = await require_index.discoverUris(require_utils.normalizeMethodsConfig(sourceInput, siteInput, methods, defaults), fetchFn);
30
30
  const seen = /* @__PURE__ */ new Set();
31
31
  const methodGroups = [];
32
32
  for (const method of require_types.discoverMethodOrder) {
33
33
  const rawUris = urisByMethod[method];
34
34
  if (!rawUris || rawUris.length === 0) continue;
35
35
  const unique = rawUris.map((entry) => {
36
- return require_utils$1.normalizeUriEntry(entry, resolveUrlFn, sourceInput.url);
36
+ return require_utils.normalizeUriEntry(entry, resolveUrlFn, sourceInput.url);
37
37
  }).filter((entry) => {
38
38
  const key = typeof entry.uri === "string" ? entry.uri : entry.uri.join("\0");
39
39
  if (seen.has(key)) return false;
@@ -91,7 +91,7 @@ const discover = async (input, options, defaults) => {
91
91
  };
92
92
  for (const { method, entries } of methodGroups) {
93
93
  const foundBefore = found;
94
- await require_utils.processConcurrently(entries, (entry) => processUri(entry, method), {
94
+ await require_utils$1.processConcurrently(entries, (entry) => processUri(entry, method), {
95
95
  concurrency,
96
96
  shouldStop: () => {
97
97
  return stopOnFirstResult && found > 0;
@@ -1,7 +1,7 @@
1
+ import { normalizeInput, normalizeMethodsConfig, normalizeUriEntry } from "./utils.js";
1
2
  import { discoverMethodOrder } from "../types.js";
2
3
  import { processConcurrently } from "../utils.js";
3
4
  import { discoverUris } from "../uris/index.js";
4
- import { normalizeInput, normalizeMethodsConfig, normalizeUriEntry } from "./utils.js";
5
5
  //#region src/common/discover/index.ts
6
6
  const discover = async (input, options, defaults) => {
7
7
  const { methods, fetchFn, extractFn, resolveUrlFn, resolveSiteUrlFn, stopOnFirstMethod = false, stopOnFirstResult = false, concurrency = 3, includeInvalid = false, onProgress } = options;
@@ -16,7 +16,7 @@ const discover = async (input, options, defaults) => {
16
16
  }
17
17
  let siteInput;
18
18
  if (resolveSiteUrlFn) {
19
- const siteUrl = resolveSiteUrlFn(sourceInput);
19
+ const siteUrl = resolveSiteUrlFn(sourceInput, resolveUrlFn);
20
20
  if (siteUrl) try {
21
21
  const response = await fetchFn(siteUrl);
22
22
  siteInput = {
@@ -1,28 +1,16 @@
1
1
  const require_locales = require("../locales.cjs");
2
- const require_utils = require("../utils.cjs");
3
- let feedsmith = require("feedsmith");
4
2
  //#region src/common/discover/utils.ts
5
- const defaultFetchFn = async (url, options) => {
6
- const response = await fetch(url, {
7
- method: options?.method ?? "GET",
8
- headers: options?.headers
9
- });
10
- return {
11
- headers: response.headers,
12
- body: await response.text(),
13
- url: response.url,
14
- status: response.status,
15
- statusText: response.statusText
16
- };
17
- };
18
3
  const normalizeInput = async (input, fetchFn) => {
19
4
  if (typeof input === "object") return input;
20
- const response = await fetchFn(input);
21
- return {
22
- url: response.url,
23
- content: typeof response.body === "string" ? response.body : "",
24
- headers: response.headers
25
- };
5
+ try {
6
+ const response = await fetchFn(input);
7
+ return {
8
+ url: response.url,
9
+ content: typeof response.body === "string" ? response.body : void 0,
10
+ headers: response.headers
11
+ };
12
+ } catch {}
13
+ return { url: input };
26
14
  };
27
15
  const getLinkOfType = (links, rel) => {
28
16
  return links?.find((link) => link.rel === rel);
@@ -33,18 +21,6 @@ const getFeedSiteUrl = (parsed) => {
33
21
  if (format === "atom") return getLinkOfType(feed.links, "alternate")?.href;
34
22
  if (format === "json") return feed.home_page_url;
35
23
  };
36
- const defaultResolveSiteUrlFn = (input) => {
37
- if (!input.content) return;
38
- try {
39
- let siteUrl = getFeedSiteUrl((0, feedsmith.parseFeed)(input.content));
40
- if (!siteUrl) try {
41
- siteUrl = new URL(input.url).origin;
42
- } catch {}
43
- else siteUrl = require_utils.resolveUrl(siteUrl, input.url);
44
- if (siteUrl && new URL(siteUrl).href === new URL(input.url).href) return;
45
- return siteUrl;
46
- } catch {}
47
- };
48
24
  const normalizeUriEntry = (entry, resolveUrlFn, baseUrl) => {
49
25
  if (typeof entry.uri === "string") return {
50
26
  ...entry,
@@ -119,8 +95,6 @@ const normalizeMethodsConfig = (sourceInput, siteInput, methods, defaults) => {
119
95
  return methodsConfig;
120
96
  };
121
97
  //#endregion
122
- exports.defaultFetchFn = defaultFetchFn;
123
- exports.defaultResolveSiteUrlFn = defaultResolveSiteUrlFn;
124
98
  exports.getFeedSiteUrl = getFeedSiteUrl;
125
99
  exports.normalizeInput = normalizeInput;
126
100
  exports.normalizeMethodsConfig = normalizeMethodsConfig;
@@ -1,8 +1,6 @@
1
- import { DiscoverResolveSiteUrlFn } from "../types.cjs";
2
1
  import { parseFeed } from "feedsmith";
3
2
 
4
3
  //#region src/common/discover/utils.d.ts
5
4
  declare const getFeedSiteUrl: (parsed: ReturnType<typeof parseFeed>) => string | undefined;
6
- declare const defaultResolveSiteUrlFn: DiscoverResolveSiteUrlFn;
7
5
  //#endregion
8
- export { defaultResolveSiteUrlFn, getFeedSiteUrl };
6
+ export { getFeedSiteUrl };
@@ -1,8 +1,6 @@
1
- import { DiscoverResolveSiteUrlFn } from "../types.js";
2
1
  import { parseFeed } from "feedsmith";
3
2
 
4
3
  //#region src/common/discover/utils.d.ts
5
4
  declare const getFeedSiteUrl: (parsed: ReturnType<typeof parseFeed>) => string | undefined;
6
- declare const defaultResolveSiteUrlFn: DiscoverResolveSiteUrlFn;
7
5
  //#endregion
8
- export { defaultResolveSiteUrlFn, getFeedSiteUrl };
6
+ export { getFeedSiteUrl };
@@ -1,28 +1,16 @@
1
1
  import { errors } from "../locales.js";
2
- import { resolveUrl } from "../utils.js";
3
- import { parseFeed } from "feedsmith";
4
2
  //#region src/common/discover/utils.ts
5
- const defaultFetchFn = async (url, options) => {
6
- const response = await fetch(url, {
7
- method: options?.method ?? "GET",
8
- headers: options?.headers
9
- });
10
- return {
11
- headers: response.headers,
12
- body: await response.text(),
13
- url: response.url,
14
- status: response.status,
15
- statusText: response.statusText
16
- };
17
- };
18
3
  const normalizeInput = async (input, fetchFn) => {
19
4
  if (typeof input === "object") return input;
20
- const response = await fetchFn(input);
21
- return {
22
- url: response.url,
23
- content: typeof response.body === "string" ? response.body : "",
24
- headers: response.headers
25
- };
5
+ try {
6
+ const response = await fetchFn(input);
7
+ return {
8
+ url: response.url,
9
+ content: typeof response.body === "string" ? response.body : void 0,
10
+ headers: response.headers
11
+ };
12
+ } catch {}
13
+ return { url: input };
26
14
  };
27
15
  const getLinkOfType = (links, rel) => {
28
16
  return links?.find((link) => link.rel === rel);
@@ -33,18 +21,6 @@ const getFeedSiteUrl = (parsed) => {
33
21
  if (format === "atom") return getLinkOfType(feed.links, "alternate")?.href;
34
22
  if (format === "json") return feed.home_page_url;
35
23
  };
36
- const defaultResolveSiteUrlFn = (input) => {
37
- if (!input.content) return;
38
- try {
39
- let siteUrl = getFeedSiteUrl(parseFeed(input.content));
40
- if (!siteUrl) try {
41
- siteUrl = new URL(input.url).origin;
42
- } catch {}
43
- else siteUrl = resolveUrl(siteUrl, input.url);
44
- if (siteUrl && new URL(siteUrl).href === new URL(input.url).href) return;
45
- return siteUrl;
46
- } catch {}
47
- };
48
24
  const normalizeUriEntry = (entry, resolveUrlFn, baseUrl) => {
49
25
  if (typeof entry.uri === "string") return {
50
26
  ...entry,
@@ -119,4 +95,4 @@ const normalizeMethodsConfig = (sourceInput, siteInput, methods, defaults) => {
119
95
  return methodsConfig;
120
96
  };
121
97
  //#endregion
122
- export { defaultFetchFn, defaultResolveSiteUrlFn, getFeedSiteUrl, normalizeInput, normalizeMethodsConfig, normalizeUriEntry };
98
+ export { getFeedSiteUrl, normalizeInput, normalizeMethodsConfig, normalizeUriEntry };
@@ -8,9 +8,15 @@ var errors = {
8
8
  };
9
9
  var hints = {
10
10
  "youtube:all": "All uploads",
11
- "youtube:videos": "Videos only",
12
- "youtube:shorts": "Shorts only",
13
- "youtube:live": "Live streams only",
11
+ "youtube:videos": "Videos",
12
+ "youtube:shorts": "Shorts",
13
+ "youtube:live": "Live streams",
14
+ "youtube:popular-videos": "Popular videos",
15
+ "youtube:popular-shorts": "Popular shorts",
16
+ "youtube:popular-live": "Popular live streams",
17
+ "youtube:member-videos": "Member videos",
18
+ "youtube:member-shorts": "Member shorts",
19
+ "youtube:member-live": "Member live streams",
14
20
  "youtube:playlist": "Playlist",
15
21
  "github:activity": "Activity",
16
22
  "github:releases": "Releases",
@@ -120,7 +126,8 @@ var hints = {
120
126
  "v2ex:tab": "Tab",
121
127
  "ximalaya:album": "Album",
122
128
  "lemmy:community": "Community",
123
- "lemmy:user": "User"
129
+ "lemmy:user": "User",
130
+ "apple-podcasts:podcast": "Podcast"
124
131
  };
125
132
  //#endregion
126
133
  Object.defineProperty(exports, "errors", {
@@ -8,9 +8,15 @@ var errors = {
8
8
  };
9
9
  var hints = {
10
10
  "youtube:all": "All uploads",
11
- "youtube:videos": "Videos only",
12
- "youtube:shorts": "Shorts only",
13
- "youtube:live": "Live streams only",
11
+ "youtube:videos": "Videos",
12
+ "youtube:shorts": "Shorts",
13
+ "youtube:live": "Live streams",
14
+ "youtube:popular-videos": "Popular videos",
15
+ "youtube:popular-shorts": "Popular shorts",
16
+ "youtube:popular-live": "Popular live streams",
17
+ "youtube:member-videos": "Member videos",
18
+ "youtube:member-shorts": "Member shorts",
19
+ "youtube:member-live": "Member live streams",
14
20
  "youtube:playlist": "Playlist",
15
21
  "github:activity": "Activity",
16
22
  "github:releases": "Releases",
@@ -120,7 +126,8 @@ var hints = {
120
126
  "v2ex:tab": "Tab",
121
127
  "ximalaya:album": "Album",
122
128
  "lemmy:community": "Community",
123
- "lemmy:user": "User"
129
+ "lemmy:user": "User",
130
+ "apple-podcasts:podcast": "Podcast"
124
131
  };
125
132
  //#endregion
126
133
  export { errors, hints };
@@ -5,6 +5,7 @@ import { HtmlMethodOptions } from "./uris/html/types.cjs";
5
5
  import { PlatformMethodOptions } from "./uris/platform/types.cjs";
6
6
 
7
7
  //#region src/common/types.d.ts
8
+ type MaybePromise<T> = T | Promise<T>;
8
9
  type UriEntry = string | Array<string>;
9
10
  type DiscoverUriHint = {
10
11
  key: string;
@@ -21,7 +22,7 @@ type LinkSelector = {
21
22
  types?: Array<string>;
22
23
  };
23
24
  type DiscoverResolveUrlFn = (url: string, baseUrl: string | undefined) => string | undefined;
24
- type DiscoverResolveSiteUrlFn = (input: DiscoverInputObject) => string | undefined;
25
+ type DiscoverResolveSiteUrlFn = (input: DiscoverInputObject, resolveUrlFn: DiscoverResolveUrlFn) => string | undefined;
25
26
  type DiscoverFetchFnOptions = {
26
27
  method?: 'GET' | 'HEAD';
27
28
  headers?: Record<string, string>;
@@ -33,7 +34,7 @@ type DiscoverFetchFnResponse = {
33
34
  status: number;
34
35
  statusText: string;
35
36
  };
36
- type DiscoverFetchFn = (url: string, options?: DiscoverFetchFnOptions) => Promise<DiscoverFetchFnResponse>;
37
+ type DiscoverFetchFn = (url: string, options?: DiscoverFetchFnOptions) => MaybePromise<DiscoverFetchFnResponse>;
37
38
  type DiscoverProgress = {
38
39
  tested: number;
39
40
  total: number;
@@ -58,7 +59,7 @@ type DiscoverExtractFn<TValid> = (input: {
58
59
  content: string;
59
60
  headers?: Headers;
60
61
  status?: number;
61
- }) => Promise<DiscoverResult<TValid>> | DiscoverResult<TValid>;
62
+ }) => MaybePromise<DiscoverResult<TValid>>;
62
63
  type DiscoverInputObject = {
63
64
  url: string;
64
65
  content?: string;
@@ -85,4 +86,4 @@ type DiscoverOptions<TValid, TMethods extends DiscoverMethod = DiscoverMethod> =
85
86
  includeInvalid?: boolean;
86
87
  };
87
88
  //#endregion
88
- export { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethod, DiscoverMethodsConfig, DiscoverOnProgressFn, DiscoverOptions, DiscoverProgress, DiscoverResolveSiteUrlFn, DiscoverResolveUrlFn, DiscoverResult, DiscoverUriEntry, DiscoverUriHint, LinkSelector, UriEntry };
89
+ export { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethod, DiscoverMethodsConfig, DiscoverOnProgressFn, DiscoverOptions, DiscoverProgress, DiscoverResolveSiteUrlFn, DiscoverResolveUrlFn, DiscoverResult, DiscoverUriEntry, DiscoverUriHint, LinkSelector, MaybePromise, UriEntry };
@@ -5,6 +5,7 @@ import { HtmlMethodOptions } from "./uris/html/types.js";
5
5
  import { PlatformMethodOptions } from "./uris/platform/types.js";
6
6
 
7
7
  //#region src/common/types.d.ts
8
+ type MaybePromise<T> = T | Promise<T>;
8
9
  type UriEntry = string | Array<string>;
9
10
  type DiscoverUriHint = {
10
11
  key: string;
@@ -21,7 +22,7 @@ type LinkSelector = {
21
22
  types?: Array<string>;
22
23
  };
23
24
  type DiscoverResolveUrlFn = (url: string, baseUrl: string | undefined) => string | undefined;
24
- type DiscoverResolveSiteUrlFn = (input: DiscoverInputObject) => string | undefined;
25
+ type DiscoverResolveSiteUrlFn = (input: DiscoverInputObject, resolveUrlFn: DiscoverResolveUrlFn) => string | undefined;
25
26
  type DiscoverFetchFnOptions = {
26
27
  method?: 'GET' | 'HEAD';
27
28
  headers?: Record<string, string>;
@@ -33,7 +34,7 @@ type DiscoverFetchFnResponse = {
33
34
  status: number;
34
35
  statusText: string;
35
36
  };
36
- type DiscoverFetchFn = (url: string, options?: DiscoverFetchFnOptions) => Promise<DiscoverFetchFnResponse>;
37
+ type DiscoverFetchFn = (url: string, options?: DiscoverFetchFnOptions) => MaybePromise<DiscoverFetchFnResponse>;
37
38
  type DiscoverProgress = {
38
39
  tested: number;
39
40
  total: number;
@@ -58,7 +59,7 @@ type DiscoverExtractFn<TValid> = (input: {
58
59
  content: string;
59
60
  headers?: Headers;
60
61
  status?: number;
61
- }) => Promise<DiscoverResult<TValid>> | DiscoverResult<TValid>;
62
+ }) => MaybePromise<DiscoverResult<TValid>>;
62
63
  type DiscoverInputObject = {
63
64
  url: string;
64
65
  content?: string;
@@ -85,4 +86,4 @@ type DiscoverOptions<TValid, TMethods extends DiscoverMethod = DiscoverMethod> =
85
86
  includeInvalid?: boolean;
86
87
  };
87
88
  //#endregion
88
- export { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethod, DiscoverMethodsConfig, DiscoverOnProgressFn, DiscoverOptions, DiscoverProgress, DiscoverResolveSiteUrlFn, DiscoverResolveUrlFn, DiscoverResult, DiscoverUriEntry, DiscoverUriHint, LinkSelector, UriEntry };
89
+ export { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethod, DiscoverMethodsConfig, DiscoverOnProgressFn, DiscoverOptions, DiscoverProgress, DiscoverResolveSiteUrlFn, DiscoverResolveUrlFn, DiscoverResult, DiscoverUriEntry, DiscoverUriHint, LinkSelector, MaybePromise, UriEntry };
@@ -1,9 +1,9 @@
1
1
  const require_utils = require("../../utils.cjs");
2
2
  //#region src/common/uris/headers/index.ts
3
+ const linkSplitRegex = /,(?=\s*<)/;
3
4
  const urlRegex = /<([^<>]+)>/;
4
5
  const relRegex = /rel\s*=\s*["']?([^"';,]+)["']?/i;
5
6
  const typeRegex = /type\s*=\s*["']?([^"';,]+)["']?/i;
6
- const linkSplitRegex = /,(?=\s*<)/;
7
7
  const discoverUrisFromHeaders = (headers, options) => {
8
8
  const uris = /* @__PURE__ */ new Set();
9
9
  const linkHeader = headers.get("link");
@@ -1,9 +1,9 @@
1
1
  import { matchesAnyOfLinkSelectors } from "../../utils.js";
2
2
  //#region src/common/uris/headers/index.ts
3
+ const linkSplitRegex = /,(?=\s*<)/;
3
4
  const urlRegex = /<([^<>]+)>/;
4
5
  const relRegex = /rel\s*=\s*["']?([^"';,]+)["']?/i;
5
6
  const typeRegex = /type\s*=\s*["']?([^"';,]+)["']?/i;
6
- const linkSplitRegex = /,(?=\s*<)/;
7
7
  const discoverUrisFromHeaders = (headers, options) => {
8
8
  const uris = /* @__PURE__ */ new Set();
9
9
  const linkHeader = headers.get("link");
@@ -1,9 +1,9 @@
1
- import { DiscoverFetchFn, DiscoverUriEntry } from "../../types.cjs";
1
+ import { DiscoverFetchFn, DiscoverUriEntry, MaybePromise } from "../../types.cjs";
2
2
 
3
3
  //#region src/common/uris/platform/types.d.ts
4
4
  type PlatformHandler = {
5
5
  match: (url: string, content?: string, headers?: Headers) => boolean;
6
- resolve: (url: string, content?: string, headers?: Headers, fetchFn?: DiscoverFetchFn) => Array<DiscoverUriEntry> | Promise<Array<DiscoverUriEntry>>;
6
+ resolve: (url: string, content?: string, headers?: Headers, fetchFn?: DiscoverFetchFn) => MaybePromise<Array<DiscoverUriEntry>>;
7
7
  };
8
8
  type PlatformMethodOptions = {
9
9
  baseUrl: string;