feedscout 1.0.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/README.md +10 -7
  2. package/dist/blogrolls/index.cjs +9 -3
  3. package/dist/blogrolls/index.d.cts +1 -1
  4. package/dist/blogrolls/index.d.ts +1 -1
  5. package/dist/blogrolls/index.js +9 -3
  6. package/dist/common/discover/utils.cjs +26 -0
  7. package/dist/common/discover/utils.js +26 -1
  8. package/dist/common/locales.cjs +1 -0
  9. package/dist/common/locales.js +1 -0
  10. package/dist/common/types.d.cts +7 -5
  11. package/dist/common/types.d.ts +7 -5
  12. package/dist/common/uris/headers/index.cjs +6 -3
  13. package/dist/common/uris/headers/index.js +6 -3
  14. package/dist/common/uris/index.cjs +2 -0
  15. package/dist/common/uris/index.js +2 -0
  16. package/dist/common/uris/platform/index.cjs +12 -0
  17. package/dist/common/uris/platform/index.js +11 -0
  18. package/dist/common/uris/platform/types.d.cts +11 -0
  19. package/dist/common/uris/platform/types.d.ts +11 -0
  20. package/dist/common/utils.cjs +10 -0
  21. package/dist/common/utils.d.cts +9 -0
  22. package/dist/common/utils.d.ts +9 -0
  23. package/dist/common/utils.js +7 -1
  24. package/dist/feeds/defaults.cjs +43 -0
  25. package/dist/feeds/defaults.d.cts +3 -1
  26. package/dist/feeds/defaults.d.ts +3 -1
  27. package/dist/feeds/defaults.js +44 -1
  28. package/dist/feeds/index.cjs +10 -3
  29. package/dist/feeds/index.d.cts +1 -1
  30. package/dist/feeds/index.d.ts +1 -1
  31. package/dist/feeds/index.js +11 -4
  32. package/dist/feeds/platform/handlers/behance.cjs +45 -0
  33. package/dist/feeds/platform/handlers/behance.js +45 -0
  34. package/dist/feeds/platform/handlers/blogspot.cjs +24 -0
  35. package/dist/feeds/platform/handlers/blogspot.js +23 -0
  36. package/dist/feeds/platform/handlers/bluesky.cjs +18 -0
  37. package/dist/feeds/platform/handlers/bluesky.js +18 -0
  38. package/dist/feeds/platform/handlers/dailymotion.cjs +66 -0
  39. package/dist/feeds/platform/handlers/dailymotion.js +66 -0
  40. package/dist/feeds/platform/handlers/deviantart.cjs +50 -0
  41. package/dist/feeds/platform/handlers/deviantart.js +50 -0
  42. package/dist/feeds/platform/handlers/devto.cjs +44 -0
  43. package/dist/feeds/platform/handlers/devto.js +44 -0
  44. package/dist/feeds/platform/handlers/github.cjs +89 -0
  45. package/dist/feeds/platform/handlers/github.js +89 -0
  46. package/dist/feeds/platform/handlers/githubGist.cjs +42 -0
  47. package/dist/feeds/platform/handlers/githubGist.js +42 -0
  48. package/dist/feeds/platform/handlers/gitlab.cjs +53 -0
  49. package/dist/feeds/platform/handlers/gitlab.js +53 -0
  50. package/dist/feeds/platform/handlers/kickstarter.cjs +18 -0
  51. package/dist/feeds/platform/handlers/kickstarter.js +18 -0
  52. package/dist/feeds/platform/handlers/lobsters.cjs +34 -0
  53. package/dist/feeds/platform/handlers/lobsters.js +34 -0
  54. package/dist/feeds/platform/handlers/medium.cjs +47 -0
  55. package/dist/feeds/platform/handlers/medium.js +47 -0
  56. package/dist/feeds/platform/handlers/pinterest.cjs +48 -0
  57. package/dist/feeds/platform/handlers/pinterest.js +48 -0
  58. package/dist/feeds/platform/handlers/producthunt.cjs +22 -0
  59. package/dist/feeds/platform/handlers/producthunt.js +22 -0
  60. package/dist/feeds/platform/handlers/reddit.cjs +47 -0
  61. package/dist/feeds/platform/handlers/reddit.js +47 -0
  62. package/dist/feeds/platform/handlers/soundcloud.cjs +37 -0
  63. package/dist/feeds/platform/handlers/soundcloud.js +37 -0
  64. package/dist/feeds/platform/handlers/substack.cjs +15 -0
  65. package/dist/feeds/platform/handlers/substack.js +15 -0
  66. package/dist/feeds/platform/handlers/tumblr.cjs +18 -0
  67. package/dist/feeds/platform/handlers/tumblr.js +18 -0
  68. package/dist/feeds/platform/handlers/wordpress.cjs +30 -0
  69. package/dist/feeds/platform/handlers/wordpress.js +30 -0
  70. package/dist/feeds/platform/handlers/youtube.cjs +56 -0
  71. package/dist/feeds/platform/handlers/youtube.js +56 -0
  72. package/dist/feeds.cjs +1 -0
  73. package/dist/feeds.d.cts +2 -2
  74. package/dist/feeds.d.ts +2 -2
  75. package/dist/feeds.js +2 -2
  76. package/dist/hubs/discover/index.cjs +7 -6
  77. package/dist/hubs/discover/index.js +5 -4
  78. package/dist/hubs/discover/types.d.cts +2 -1
  79. package/dist/hubs/discover/types.d.ts +2 -1
  80. package/dist/hubs/headers/index.cjs +3 -3
  81. package/dist/hubs/headers/index.js +3 -3
  82. package/dist/hubs/html/index.cjs +3 -3
  83. package/dist/hubs/html/index.js +3 -3
  84. package/dist/index.d.cts +2 -2
  85. package/dist/index.d.ts +2 -2
  86. package/dist/utils.cjs +8 -0
  87. package/dist/utils.d.cts +2 -0
  88. package/dist/utils.d.ts +2 -0
  89. package/dist/utils.js +3 -0
  90. package/package.json +32 -32
  91. package/dist/adapters.cjs +0 -6
  92. package/dist/adapters.d.cts +0 -2
  93. package/dist/adapters.d.ts +0 -2
  94. package/dist/adapters.js +0 -3
  95. package/dist/common/discover/adapters.cjs +0 -76
  96. package/dist/common/discover/adapters.d.cts +0 -10
  97. package/dist/common/discover/adapters.d.ts +0 -10
  98. package/dist/common/discover/adapters.js +0 -72
package/README.md CHANGED
@@ -24,6 +24,7 @@ Finds feeds by scanning links and anchors in HTML content, parsing HTTP headers,
24
24
 
25
25
  ### Discovery Methods
26
26
 
27
+ - **Platform** — Generates feed URLs for YouTube, GitHub, WordPress, and other popular platforms using URL pattern matching.
27
28
  - **HTML** — Scans `<link>` elements with feed MIME types and `<a>` elements matching feed patterns or labels like "RSS", "Subscribe".
28
29
  - **Headers** — Parses HTTP `Link` headers for `rel="alternate"` with feed MIME types per RFC 8288.
29
30
  - **Guess** — Tests common paths (e.g. `/feed`, `/rss.xml`, `/atom.xml`) against the base URL as a fallback.
@@ -32,7 +33,7 @@ Finds feeds by scanning links and anchors in HTML content, parsing HTTP headers,
32
33
 
33
34
  - **Custom extractors** — Override the default parser to extract additional metadata from feeds and blogrolls.
34
35
  - **Configurable methods** — Enable/disable discovery methods or customize their options.
35
- - **Adapter system** — Use native fetch or integrate with Axios, Got, or Ky.
36
+ - **Adapter system** — Use native fetch or easily integrate with Axios, Got, or Ky.
36
37
  - **Concurrency control** — Limit parallel requests during validation.
37
38
  - **Progress tracking** — Monitor discovery progress with callbacks.
38
39
  - **Type-safe** — Full TypeScript support with exported types.
@@ -55,9 +56,7 @@ npm install feedscout
55
56
  ```typescript
56
57
  import { discoverFeeds } from 'feedscout'
57
58
 
58
- const feeds = await discoverFeeds('https://example.com', {
59
- methods: ['html', 'headers'],
60
- })
59
+ const feeds = await discoverFeeds('https://example.com')
61
60
 
62
61
  // [{
63
62
  // url: 'https://example.com/feed.xml',
@@ -69,6 +68,12 @@ const feeds = await discoverFeeds('https://example.com', {
69
68
  // }]
70
69
  ```
71
70
 
71
+ ```typescript
72
+ const feeds = await discoverFeeds('https://example.com', {
73
+ methods: ['html', 'headers'],
74
+ })
75
+ ```
76
+
72
77
  Or with existing HTML content:
73
78
 
74
79
  ```html
@@ -135,9 +140,7 @@ const feeds = await discoverFeeds(
135
140
  ```typescript
136
141
  import { discoverBlogrolls } from 'feedscout'
137
142
 
138
- const blogrolls = await discoverBlogrolls('https://example.com', {
139
- methods: ['html'],
140
- })
143
+ const blogrolls = await discoverBlogrolls('https://example.com')
141
144
 
142
145
  // [{
143
146
  // url: 'https://example.com/blogroll.opml',
@@ -1,17 +1,23 @@
1
- const require_adapters = require('../common/discover/adapters.cjs');
2
1
  const require_utils = require('../common/utils.cjs');
2
+ const require_utils$1 = require('../common/discover/utils.cjs');
3
3
  const require_index = require('../common/discover/index.cjs');
4
4
  const require_defaults = require('./defaults.cjs');
5
5
  const require_extractors = require('./extractors.cjs');
6
6
 
7
7
  //#region src/blogrolls/index.ts
8
- const discoverBlogrolls = async (input, options) => {
8
+ const discoverBlogrolls = async (input, options = {}) => {
9
9
  return require_index.discover(input, {
10
10
  ...options,
11
- fetchFn: options.fetchFn ?? require_adapters.createNativeFetchAdapter(),
11
+ methods: options.methods ?? [
12
+ "html",
13
+ "headers",
14
+ "guess"
15
+ ],
16
+ fetchFn: options.fetchFn ?? require_utils$1.defaultFetchFn,
12
17
  extractFn: options.extractFn ?? require_extractors.defaultExtractor,
13
18
  normalizeUrlFn: options.normalizeUrlFn ?? require_utils.normalizeUrl
14
19
  }, {
20
+ platform: { handlers: [] },
15
21
  html: require_defaults.defaultHtmlOptions,
16
22
  headers: require_defaults.defaultHeadersOptions,
17
23
  guess: require_defaults.defaultGuessOptions
@@ -2,6 +2,6 @@ import { DiscoverInput, DiscoverOptions, DiscoverResult } from "../common/types.
2
2
  import { BlogrollResult } from "./types.cjs";
3
3
 
4
4
  //#region src/blogrolls/index.d.ts
5
- declare const discoverBlogrolls: <TValid extends BlogrollResult = BlogrollResult>(input: DiscoverInput, options: DiscoverOptions<TValid>) => Promise<Array<DiscoverResult<TValid>>>;
5
+ declare const discoverBlogrolls: <TValid extends BlogrollResult = BlogrollResult>(input: DiscoverInput, options?: DiscoverOptions<TValid>) => Promise<Array<DiscoverResult<TValid>>>;
6
6
  //#endregion
7
7
  export { discoverBlogrolls };
@@ -2,6 +2,6 @@ import { DiscoverInput, DiscoverOptions, DiscoverResult } from "../common/types.
2
2
  import { BlogrollResult } from "./types.js";
3
3
 
4
4
  //#region src/blogrolls/index.d.ts
5
- declare const discoverBlogrolls: <TValid extends BlogrollResult = BlogrollResult>(input: DiscoverInput, options: DiscoverOptions<TValid>) => Promise<Array<DiscoverResult<TValid>>>;
5
+ declare const discoverBlogrolls: <TValid extends BlogrollResult = BlogrollResult>(input: DiscoverInput, options?: DiscoverOptions<TValid>) => Promise<Array<DiscoverResult<TValid>>>;
6
6
  //#endregion
7
7
  export { discoverBlogrolls };
@@ -1,17 +1,23 @@
1
- import { createNativeFetchAdapter } from "../common/discover/adapters.js";
2
1
  import { normalizeUrl } from "../common/utils.js";
2
+ import { defaultFetchFn } from "../common/discover/utils.js";
3
3
  import { discover } from "../common/discover/index.js";
4
4
  import { defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions } from "./defaults.js";
5
5
  import { defaultExtractor } from "./extractors.js";
6
6
 
7
7
  //#region src/blogrolls/index.ts
8
- const discoverBlogrolls = async (input, options) => {
8
+ const discoverBlogrolls = async (input, options = {}) => {
9
9
  return discover(input, {
10
10
  ...options,
11
- fetchFn: options.fetchFn ?? createNativeFetchAdapter(),
11
+ methods: options.methods ?? [
12
+ "html",
13
+ "headers",
14
+ "guess"
15
+ ],
16
+ fetchFn: options.fetchFn ?? defaultFetchFn,
12
17
  extractFn: options.extractFn ?? defaultExtractor,
13
18
  normalizeUrlFn: options.normalizeUrlFn ?? normalizeUrl
14
19
  }, {
20
+ platform: { handlers: [] },
15
21
  html: defaultHtmlOptions,
16
22
  headers: defaultHeadersOptions,
17
23
  guess: defaultGuessOptions
@@ -1,6 +1,19 @@
1
1
  const require_locales = require('../locales.cjs');
2
2
 
3
3
  //#region src/common/discover/utils.ts
4
+ const defaultFetchFn = async (url, options) => {
5
+ const response = await fetch(url, {
6
+ method: options?.method ?? "GET",
7
+ headers: options?.headers
8
+ });
9
+ return {
10
+ headers: response.headers,
11
+ body: await response.text(),
12
+ url: response.url,
13
+ status: response.status,
14
+ statusText: response.statusText
15
+ };
16
+ };
4
17
  const normalizeInput = async (input, fetchFn) => {
5
18
  if (typeof input === "object") return input;
6
19
  const response = await fetchFn(input);
@@ -13,6 +26,18 @@ const normalizeInput = async (input, fetchFn) => {
13
26
  const normalizeMethodsConfig = (input, methods, defaults) => {
14
27
  const methodsObj = Array.isArray(methods) ? Object.fromEntries(methods.map((method) => [method, true])) : methods;
15
28
  const methodsConfig = {};
29
+ if (methodsObj.platform) {
30
+ if (!input.url || input.url === "") throw new Error(require_locales.errors.platformMethodRequiresUrl);
31
+ const platformOptions = methodsObj.platform === true ? {} : methodsObj.platform;
32
+ methodsConfig.platform = {
33
+ html: input.content ?? "",
34
+ options: {
35
+ ...defaults.platform,
36
+ ...platformOptions,
37
+ baseUrl: input.url
38
+ }
39
+ };
40
+ }
16
41
  if (methodsObj.html) {
17
42
  if (input.content === void 0) throw new Error(require_locales.errors.htmlMethodRequiresContent);
18
43
  const htmlOptions = methodsObj.html === true ? {} : methodsObj.html;
@@ -50,5 +75,6 @@ const normalizeMethodsConfig = (input, methods, defaults) => {
50
75
  };
51
76
 
52
77
  //#endregion
78
+ exports.defaultFetchFn = defaultFetchFn;
53
79
  exports.normalizeInput = normalizeInput;
54
80
  exports.normalizeMethodsConfig = normalizeMethodsConfig;
@@ -1,6 +1,19 @@
1
1
  import { errors } from "../locales.js";
2
2
 
3
3
  //#region src/common/discover/utils.ts
4
+ const defaultFetchFn = async (url, options) => {
5
+ const response = await fetch(url, {
6
+ method: options?.method ?? "GET",
7
+ headers: options?.headers
8
+ });
9
+ return {
10
+ headers: response.headers,
11
+ body: await response.text(),
12
+ url: response.url,
13
+ status: response.status,
14
+ statusText: response.statusText
15
+ };
16
+ };
4
17
  const normalizeInput = async (input, fetchFn) => {
5
18
  if (typeof input === "object") return input;
6
19
  const response = await fetchFn(input);
@@ -13,6 +26,18 @@ const normalizeInput = async (input, fetchFn) => {
13
26
  const normalizeMethodsConfig = (input, methods, defaults) => {
14
27
  const methodsObj = Array.isArray(methods) ? Object.fromEntries(methods.map((method) => [method, true])) : methods;
15
28
  const methodsConfig = {};
29
+ if (methodsObj.platform) {
30
+ if (!input.url || input.url === "") throw new Error(errors.platformMethodRequiresUrl);
31
+ const platformOptions = methodsObj.platform === true ? {} : methodsObj.platform;
32
+ methodsConfig.platform = {
33
+ html: input.content ?? "",
34
+ options: {
35
+ ...defaults.platform,
36
+ ...platformOptions,
37
+ baseUrl: input.url
38
+ }
39
+ };
40
+ }
16
41
  if (methodsObj.html) {
17
42
  if (input.content === void 0) throw new Error(errors.htmlMethodRequiresContent);
18
43
  const htmlOptions = methodsObj.html === true ? {} : methodsObj.html;
@@ -50,4 +75,4 @@ const normalizeMethodsConfig = (input, methods, defaults) => {
50
75
  };
51
76
 
52
77
  //#endregion
53
- export { normalizeInput, normalizeMethodsConfig };
78
+ export { defaultFetchFn, normalizeInput, normalizeMethodsConfig };
@@ -1,6 +1,7 @@
1
1
 
2
2
  //#region src/common/locales.json
3
3
  var errors = {
4
+ "platformMethodRequiresUrl": "Platform method requires url to be provided in input",
4
5
  "htmlMethodRequiresContent": "HTML method requires content to be provided in input",
5
6
  "headersMethodRequiresHeaders": "Headers method requires headers to be provided in input",
6
7
  "guessMethodRequiresUrl": "Guess method requires url to be provided in input"
@@ -1,5 +1,6 @@
1
1
  //#region src/common/locales.json
2
2
  var errors = {
3
+ "platformMethodRequiresUrl": "Platform method requires url to be provided in input",
3
4
  "htmlMethodRequiresContent": "HTML method requires content to be provided in input",
4
5
  "headersMethodRequiresHeaders": "Headers method requires headers to be provided in input",
5
6
  "guessMethodRequiresUrl": "Guess method requires url to be provided in input"
@@ -1,6 +1,7 @@
1
1
  import { GuessMethodOptions } from "./uris/guess/types.cjs";
2
2
  import { HeadersMethodOptions } from "./uris/headers/types.cjs";
3
3
  import { HtmlMethodOptions } from "./uris/html/types.cjs";
4
+ import { PlatformMethodOptions } from "./uris/platform/types.cjs";
4
5
 
5
6
  //#region src/common/types.d.ts
6
7
  type LinkSelector = {
@@ -26,7 +27,7 @@ type DiscoverProgress = {
26
27
  found: number;
27
28
  current: string;
28
29
  };
29
- type DiscoverProgressFn = (progress: DiscoverProgress) => void;
30
+ type DiscoverOnProgressFn = (progress: DiscoverProgress) => void;
30
31
  type DiscoverResult<TValid = object> = ({
31
32
  url: string;
32
33
  isValid: true;
@@ -46,20 +47,21 @@ type DiscoverInputObject = {
46
47
  headers?: Headers;
47
48
  };
48
49
  type DiscoverInput = string | DiscoverInputObject;
49
- type DiscoverMethodsConfig = Array<'html' | 'headers' | 'guess'> | {
50
+ type DiscoverMethodsConfig = Array<'platform' | 'html' | 'headers' | 'guess'> | {
51
+ platform?: true | Partial<PlatformMethodOptions>;
50
52
  html?: true | Partial<Omit<HtmlMethodOptions, 'baseUrl'>>;
51
53
  headers?: true | Partial<Omit<HeadersMethodOptions, 'baseUrl'>>;
52
54
  guess?: true | Partial<Omit<GuessMethodOptions, 'baseUrl'>>;
53
55
  };
54
56
  type DiscoverOptions<TValid> = {
55
- methods: DiscoverMethodsConfig;
57
+ methods?: DiscoverMethodsConfig;
56
58
  fetchFn?: DiscoverFetchFn;
57
59
  extractFn?: DiscoverExtractFn<TValid>;
58
60
  normalizeUrlFn?: DiscoverNormalizeUrlFn;
59
61
  concurrency?: number;
60
62
  stopOnFirstResult?: boolean;
61
63
  includeInvalid?: boolean;
62
- onProgress?: DiscoverProgressFn;
64
+ onProgress?: DiscoverOnProgressFn;
63
65
  };
64
66
  //#endregion
65
- export { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethodsConfig, DiscoverNormalizeUrlFn, DiscoverOptions, DiscoverProgress, DiscoverProgressFn, DiscoverResult, LinkSelector };
67
+ export { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethodsConfig, DiscoverNormalizeUrlFn, DiscoverOnProgressFn, DiscoverOptions, DiscoverProgress, DiscoverResult, LinkSelector };
@@ -1,6 +1,7 @@
1
1
  import { GuessMethodOptions } from "./uris/guess/types.js";
2
2
  import { HeadersMethodOptions } from "./uris/headers/types.js";
3
3
  import { HtmlMethodOptions } from "./uris/html/types.js";
4
+ import { PlatformMethodOptions } from "./uris/platform/types.js";
4
5
 
5
6
  //#region src/common/types.d.ts
6
7
  type LinkSelector = {
@@ -26,7 +27,7 @@ type DiscoverProgress = {
26
27
  found: number;
27
28
  current: string;
28
29
  };
29
- type DiscoverProgressFn = (progress: DiscoverProgress) => void;
30
+ type DiscoverOnProgressFn = (progress: DiscoverProgress) => void;
30
31
  type DiscoverResult<TValid = object> = ({
31
32
  url: string;
32
33
  isValid: true;
@@ -46,20 +47,21 @@ type DiscoverInputObject = {
46
47
  headers?: Headers;
47
48
  };
48
49
  type DiscoverInput = string | DiscoverInputObject;
49
- type DiscoverMethodsConfig = Array<'html' | 'headers' | 'guess'> | {
50
+ type DiscoverMethodsConfig = Array<'platform' | 'html' | 'headers' | 'guess'> | {
51
+ platform?: true | Partial<PlatformMethodOptions>;
50
52
  html?: true | Partial<Omit<HtmlMethodOptions, 'baseUrl'>>;
51
53
  headers?: true | Partial<Omit<HeadersMethodOptions, 'baseUrl'>>;
52
54
  guess?: true | Partial<Omit<GuessMethodOptions, 'baseUrl'>>;
53
55
  };
54
56
  type DiscoverOptions<TValid> = {
55
- methods: DiscoverMethodsConfig;
57
+ methods?: DiscoverMethodsConfig;
56
58
  fetchFn?: DiscoverFetchFn;
57
59
  extractFn?: DiscoverExtractFn<TValid>;
58
60
  normalizeUrlFn?: DiscoverNormalizeUrlFn;
59
61
  concurrency?: number;
60
62
  stopOnFirstResult?: boolean;
61
63
  includeInvalid?: boolean;
62
- onProgress?: DiscoverProgressFn;
64
+ onProgress?: DiscoverOnProgressFn;
63
65
  };
64
66
  //#endregion
65
- export { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethodsConfig, DiscoverNormalizeUrlFn, DiscoverOptions, DiscoverProgress, DiscoverProgressFn, DiscoverResult, LinkSelector };
67
+ export { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethodsConfig, DiscoverNormalizeUrlFn, DiscoverOnProgressFn, DiscoverOptions, DiscoverProgress, DiscoverResult, LinkSelector };
@@ -1,15 +1,18 @@
1
1
  const require_utils = require('../../utils.cjs');
2
2
 
3
3
  //#region src/common/uris/headers/index.ts
4
+ const urlRegex = /<([^<>]+)>/;
5
+ const relRegex = /rel\s*=\s*["']?([^"';,]+)["']?/i;
6
+ const typeRegex = /type\s*=\s*["']?([^"';,]+)["']?/i;
4
7
  const discoverUrisFromHeaders = (headers, options) => {
5
8
  const uris = /* @__PURE__ */ new Set();
6
9
  const linkHeader = headers.get("link");
7
10
  if (!linkHeader) return [];
8
11
  const links = linkHeader.split(/,(?=\s*<)/);
9
12
  for (const link of links) {
10
- const urlMatch = link.match(/<([^<>]+)>/);
11
- const relMatch = link.match(/rel\s*=\s*["']?([^"';,]+)["']?/i);
12
- const typeMatch = link.match(/type\s*=\s*["']?([^"';,]+)["']?/i);
13
+ const urlMatch = link.match(urlRegex);
14
+ const relMatch = link.match(relRegex);
15
+ const typeMatch = link.match(typeRegex);
13
16
  if (!urlMatch) continue;
14
17
  const url = urlMatch[1];
15
18
  const rel = relMatch?.[1]?.toLowerCase();
@@ -1,15 +1,18 @@
1
1
  import { matchesAnyOfLinkSelectors } from "../../utils.js";
2
2
 
3
3
  //#region src/common/uris/headers/index.ts
4
+ const urlRegex = /<([^<>]+)>/;
5
+ const relRegex = /rel\s*=\s*["']?([^"';,]+)["']?/i;
6
+ const typeRegex = /type\s*=\s*["']?([^"';,]+)["']?/i;
4
7
  const discoverUrisFromHeaders = (headers, options) => {
5
8
  const uris = /* @__PURE__ */ new Set();
6
9
  const linkHeader = headers.get("link");
7
10
  if (!linkHeader) return [];
8
11
  const links = linkHeader.split(/,(?=\s*<)/);
9
12
  for (const link of links) {
10
- const urlMatch = link.match(/<([^<>]+)>/);
11
- const relMatch = link.match(/rel\s*=\s*["']?([^"';,]+)["']?/i);
12
- const typeMatch = link.match(/type\s*=\s*["']?([^"';,]+)["']?/i);
13
+ const urlMatch = link.match(urlRegex);
14
+ const relMatch = link.match(relRegex);
15
+ const typeMatch = link.match(typeRegex);
13
16
  if (!urlMatch) continue;
14
17
  const url = urlMatch[1];
15
18
  const rel = relMatch?.[1]?.toLowerCase();
@@ -1,10 +1,12 @@
1
1
  const require_index = require('./guess/index.cjs');
2
2
  const require_index$1 = require('./headers/index.cjs');
3
3
  const require_index$2 = require('./html/index.cjs');
4
+ const require_index$3 = require('./platform/index.cjs');
4
5
 
5
6
  //#region src/common/uris/index.ts
6
7
  const discoverUris = (config) => {
7
8
  const uris = /* @__PURE__ */ new Set();
9
+ if (config.platform) for (const uri of require_index$3.discoverUrisFromPlatform(config.platform.html, config.platform.options)) uris.add(uri);
8
10
  if (config.html) for (const uri of require_index$2.discoverUrisFromHtml(config.html.html, config.html.options)) uris.add(uri);
9
11
  if (config.headers) for (const uri of require_index$1.discoverUrisFromHeaders(config.headers.headers, config.headers.options)) uris.add(uri);
10
12
  if (config.guess) for (const uri of require_index.discoverUrisFromGuess(config.guess.options)) uris.add(uri);
@@ -1,10 +1,12 @@
1
1
  import { discoverUrisFromGuess } from "./guess/index.js";
2
2
  import { discoverUrisFromHeaders } from "./headers/index.js";
3
3
  import { discoverUrisFromHtml } from "./html/index.js";
4
+ import { discoverUrisFromPlatform } from "./platform/index.js";
4
5
 
5
6
  //#region src/common/uris/index.ts
6
7
  const discoverUris = (config) => {
7
8
  const uris = /* @__PURE__ */ new Set();
9
+ if (config.platform) for (const uri of discoverUrisFromPlatform(config.platform.html, config.platform.options)) uris.add(uri);
8
10
  if (config.html) for (const uri of discoverUrisFromHtml(config.html.html, config.html.options)) uris.add(uri);
9
11
  if (config.headers) for (const uri of discoverUrisFromHeaders(config.headers.headers, config.headers.options)) uris.add(uri);
10
12
  if (config.guess) for (const uri of discoverUrisFromGuess(config.guess.options)) uris.add(uri);
@@ -0,0 +1,12 @@
1
+
2
+ //#region src/common/uris/platform/index.ts
3
+ const discoverUrisFromPlatform = (html, options) => {
4
+ const { baseUrl, handlers } = options;
5
+ for (const handler of handlers) try {
6
+ if (handler.match(baseUrl)) return handler.resolve(baseUrl, html);
7
+ } catch {}
8
+ return [];
9
+ };
10
+
11
+ //#endregion
12
+ exports.discoverUrisFromPlatform = discoverUrisFromPlatform;
@@ -0,0 +1,11 @@
1
+ //#region src/common/uris/platform/index.ts
2
+ const discoverUrisFromPlatform = (html, options) => {
3
+ const { baseUrl, handlers } = options;
4
+ for (const handler of handlers) try {
5
+ if (handler.match(baseUrl)) return handler.resolve(baseUrl, html);
6
+ } catch {}
7
+ return [];
8
+ };
9
+
10
+ //#endregion
11
+ export { discoverUrisFromPlatform };
@@ -0,0 +1,11 @@
1
+ //#region src/common/uris/platform/types.d.ts
2
+ type PlatformHandler = {
3
+ match: (url: string) => boolean;
4
+ resolve: (url: string, content?: string) => Array<string>;
5
+ };
6
+ type PlatformMethodOptions = {
7
+ baseUrl: string;
8
+ handlers: Array<PlatformHandler>;
9
+ };
10
+ //#endregion
11
+ export { PlatformMethodOptions };
@@ -0,0 +1,11 @@
1
+ //#region src/common/uris/platform/types.d.ts
2
+ type PlatformHandler = {
3
+ match: (url: string) => boolean;
4
+ resolve: (url: string, content?: string) => Array<string>;
5
+ };
6
+ type PlatformMethodOptions = {
7
+ baseUrl: string;
8
+ handlers: Array<PlatformHandler>;
9
+ };
10
+ //#endregion
11
+ export { PlatformMethodOptions };
@@ -3,6 +3,12 @@
3
3
  const normalizeMimeType = (type) => {
4
4
  return type.split(";")[0].trim().toLowerCase();
5
5
  };
6
+ const isSubdomainOf = (url, domain) => {
7
+ return new URL(url).hostname.toLowerCase().endsWith(`.${domain}`);
8
+ };
9
+ const isHostOf = (url, hosts) => {
10
+ return isAnyOf(new URL(url).hostname, hosts);
11
+ };
6
12
  const includesAnyOf = (value, patterns, parser) => {
7
13
  const parsedValue = parser ? parser(value) : value?.toLowerCase();
8
14
  return patterns.map((pattern) => pattern.toLowerCase()).some((pattern) => parsedValue?.includes(pattern));
@@ -50,8 +56,12 @@ const processConcurrently = async (items, processFn, options) => {
50
56
  };
51
57
 
52
58
  //#endregion
59
+ exports.anyWordMatchesAnyOf = anyWordMatchesAnyOf;
53
60
  exports.endsWithAnyOf = endsWithAnyOf;
54
61
  exports.includesAnyOf = includesAnyOf;
62
+ exports.isAnyOf = isAnyOf;
63
+ exports.isHostOf = isHostOf;
64
+ exports.isSubdomainOf = isSubdomainOf;
55
65
  exports.matchesAnyOfLinkSelectors = matchesAnyOfLinkSelectors;
56
66
  exports.normalizeUrl = normalizeUrl;
57
67
  exports.processConcurrently = processConcurrently;
@@ -0,0 +1,9 @@
1
+ //#region src/common/utils.d.ts
2
+ declare const isSubdomainOf: (url: string, domain: string) => boolean;
3
+ declare const isHostOf: (url: string, hosts: Array<string>) => boolean;
4
+ declare const includesAnyOf: (value: string, patterns: Array<string>, parser?: (value: string) => string) => boolean;
5
+ declare const isAnyOf: (value: string, patterns: Array<string>, parser?: (value: string) => string) => boolean;
6
+ declare const anyWordMatchesAnyOf: (value: string, patterns: Array<string>) => boolean;
7
+ declare const endsWithAnyOf: (value: string, patterns: Array<string>) => boolean;
8
+ //#endregion
9
+ export { anyWordMatchesAnyOf, endsWithAnyOf, includesAnyOf, isAnyOf, isHostOf, isSubdomainOf };
@@ -0,0 +1,9 @@
1
+ //#region src/common/utils.d.ts
2
+ declare const isSubdomainOf: (url: string, domain: string) => boolean;
3
+ declare const isHostOf: (url: string, hosts: Array<string>) => boolean;
4
+ declare const includesAnyOf: (value: string, patterns: Array<string>, parser?: (value: string) => string) => boolean;
5
+ declare const isAnyOf: (value: string, patterns: Array<string>, parser?: (value: string) => string) => boolean;
6
+ declare const anyWordMatchesAnyOf: (value: string, patterns: Array<string>) => boolean;
7
+ declare const endsWithAnyOf: (value: string, patterns: Array<string>) => boolean;
8
+ //#endregion
9
+ export { anyWordMatchesAnyOf, endsWithAnyOf, includesAnyOf, isAnyOf, isHostOf, isSubdomainOf };
@@ -2,6 +2,12 @@
2
2
  const normalizeMimeType = (type) => {
3
3
  return type.split(";")[0].trim().toLowerCase();
4
4
  };
5
+ const isSubdomainOf = (url, domain) => {
6
+ return new URL(url).hostname.toLowerCase().endsWith(`.${domain}`);
7
+ };
8
+ const isHostOf = (url, hosts) => {
9
+ return isAnyOf(new URL(url).hostname, hosts);
10
+ };
5
11
  const includesAnyOf = (value, patterns, parser) => {
6
12
  const parsedValue = parser ? parser(value) : value?.toLowerCase();
7
13
  return patterns.map((pattern) => pattern.toLowerCase()).some((pattern) => parsedValue?.includes(pattern));
@@ -49,4 +55,4 @@ const processConcurrently = async (items, processFn, options) => {
49
55
  };
50
56
 
51
57
  //#endregion
52
- export { endsWithAnyOf, includesAnyOf, matchesAnyOfLinkSelectors, normalizeUrl, processConcurrently };
58
+ export { anyWordMatchesAnyOf, endsWithAnyOf, includesAnyOf, isAnyOf, isHostOf, isSubdomainOf, matchesAnyOfLinkSelectors, normalizeUrl, processConcurrently };
@@ -1,3 +1,23 @@
1
+ const require_behance = require('./platform/handlers/behance.cjs');
2
+ const require_blogspot = require('./platform/handlers/blogspot.cjs');
3
+ const require_bluesky = require('./platform/handlers/bluesky.cjs');
4
+ const require_dailymotion = require('./platform/handlers/dailymotion.cjs');
5
+ const require_deviantart = require('./platform/handlers/deviantart.cjs');
6
+ const require_devto = require('./platform/handlers/devto.cjs');
7
+ const require_github = require('./platform/handlers/github.cjs');
8
+ const require_githubGist = require('./platform/handlers/githubGist.cjs');
9
+ const require_gitlab = require('./platform/handlers/gitlab.cjs');
10
+ const require_kickstarter = require('./platform/handlers/kickstarter.cjs');
11
+ const require_lobsters = require('./platform/handlers/lobsters.cjs');
12
+ const require_medium = require('./platform/handlers/medium.cjs');
13
+ const require_pinterest = require('./platform/handlers/pinterest.cjs');
14
+ const require_producthunt = require('./platform/handlers/producthunt.cjs');
15
+ const require_reddit = require('./platform/handlers/reddit.cjs');
16
+ const require_soundcloud = require('./platform/handlers/soundcloud.cjs');
17
+ const require_substack = require('./platform/handlers/substack.cjs');
18
+ const require_tumblr = require('./platform/handlers/tumblr.cjs');
19
+ const require_wordpress = require('./platform/handlers/wordpress.cjs');
20
+ const require_youtube = require('./platform/handlers/youtube.cjs');
1
21
 
2
22
  //#region src/feeds/defaults.ts
3
23
  const mimeTypes = [
@@ -75,12 +95,35 @@ const defaultHtmlOptions = {
75
95
  };
76
96
  const defaultHeadersOptions = { linkSelectors };
77
97
  const defaultGuessOptions = { uris: urisBalanced };
98
+ const defaultPlatformOptions = { handlers: [
99
+ require_behance.behanceHandler,
100
+ require_blogspot.blogspotHandler,
101
+ require_bluesky.blueskyHandler,
102
+ require_deviantart.deviantartHandler,
103
+ require_dailymotion.dailymotionHandler,
104
+ require_devto.devtoHandler,
105
+ require_github.githubHandler,
106
+ require_githubGist.githubGistHandler,
107
+ require_gitlab.gitlabHandler,
108
+ require_kickstarter.kickstarterHandler,
109
+ require_medium.mediumHandler,
110
+ require_pinterest.pinterestHandler,
111
+ require_producthunt.producthuntHandler,
112
+ require_lobsters.lobstersHandler,
113
+ require_reddit.redditHandler,
114
+ require_soundcloud.soundcloudHandler,
115
+ require_substack.substackHandler,
116
+ require_tumblr.tumblrHandler,
117
+ require_wordpress.wordpressHandler,
118
+ require_youtube.youtubeHandler
119
+ ] };
78
120
 
79
121
  //#endregion
80
122
  exports.anchorLabels = anchorLabels;
81
123
  exports.defaultGuessOptions = defaultGuessOptions;
82
124
  exports.defaultHeadersOptions = defaultHeadersOptions;
83
125
  exports.defaultHtmlOptions = defaultHtmlOptions;
126
+ exports.defaultPlatformOptions = defaultPlatformOptions;
84
127
  exports.ignoredUris = ignoredUris;
85
128
  exports.linkSelectors = linkSelectors;
86
129
  exports.mimeTypes = mimeTypes;
@@ -1,6 +1,7 @@
1
1
  import { GuessMethodOptions } from "../common/uris/guess/types.cjs";
2
2
  import { HeadersMethodOptions } from "../common/uris/headers/types.cjs";
3
3
  import { HtmlMethodOptions } from "../common/uris/html/types.cjs";
4
+ import { PlatformMethodOptions } from "../common/uris/platform/types.cjs";
4
5
  import { LinkSelector } from "../common/types.cjs";
5
6
 
6
7
  //#region src/feeds/defaults.d.ts
@@ -14,5 +15,6 @@ declare const linkSelectors: Array<LinkSelector>;
14
15
  declare const defaultHtmlOptions: Omit<HtmlMethodOptions, 'baseUrl'>;
15
16
  declare const defaultHeadersOptions: Omit<HeadersMethodOptions, 'baseUrl'>;
16
17
  declare const defaultGuessOptions: Omit<GuessMethodOptions, 'baseUrl'>;
18
+ declare const defaultPlatformOptions: Omit<PlatformMethodOptions, 'baseUrl'>;
17
19
  //#endregion
18
- export { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
20
+ export { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
@@ -1,6 +1,7 @@
1
1
  import { GuessMethodOptions } from "../common/uris/guess/types.js";
2
2
  import { HeadersMethodOptions } from "../common/uris/headers/types.js";
3
3
  import { HtmlMethodOptions } from "../common/uris/html/types.js";
4
+ import { PlatformMethodOptions } from "../common/uris/platform/types.js";
4
5
  import { LinkSelector } from "../common/types.js";
5
6
 
6
7
  //#region src/feeds/defaults.d.ts
@@ -14,5 +15,6 @@ declare const linkSelectors: Array<LinkSelector>;
14
15
  declare const defaultHtmlOptions: Omit<HtmlMethodOptions, 'baseUrl'>;
15
16
  declare const defaultHeadersOptions: Omit<HeadersMethodOptions, 'baseUrl'>;
16
17
  declare const defaultGuessOptions: Omit<GuessMethodOptions, 'baseUrl'>;
18
+ declare const defaultPlatformOptions: Omit<PlatformMethodOptions, 'baseUrl'>;
17
19
  //#endregion
18
- export { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
20
+ export { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };