feedscout 1.1.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +8 -6
  2. package/dist/blogrolls/index.cjs +8 -3
  3. package/dist/blogrolls/index.d.cts +1 -1
  4. package/dist/blogrolls/index.d.ts +1 -1
  5. package/dist/blogrolls/index.js +8 -3
  6. package/dist/common/discover/utils.cjs +14 -0
  7. package/dist/common/discover/utils.js +14 -1
  8. package/dist/common/types.d.cts +4 -4
  9. package/dist/common/types.d.ts +4 -4
  10. package/dist/common/uris/headers/index.cjs +6 -3
  11. package/dist/common/uris/headers/index.js +6 -3
  12. package/dist/common/utils.cjs +1 -0
  13. package/dist/common/utils.d.cts +9 -0
  14. package/dist/common/utils.d.ts +9 -0
  15. package/dist/common/utils.js +1 -1
  16. package/dist/feeds/defaults.cjs +18 -0
  17. package/dist/feeds/defaults.js +18 -0
  18. package/dist/feeds/index.cjs +9 -3
  19. package/dist/feeds/index.d.cts +1 -1
  20. package/dist/feeds/index.d.ts +1 -1
  21. package/dist/feeds/index.js +9 -3
  22. package/dist/feeds/platform/handlers/behance.cjs +45 -0
  23. package/dist/feeds/platform/handlers/behance.js +45 -0
  24. package/dist/feeds/platform/handlers/blogspot.cjs +13 -4
  25. package/dist/feeds/platform/handlers/blogspot.js +13 -5
  26. package/dist/feeds/platform/handlers/dailymotion.cjs +66 -0
  27. package/dist/feeds/platform/handlers/dailymotion.js +66 -0
  28. package/dist/feeds/platform/handlers/deviantart.cjs +50 -0
  29. package/dist/feeds/platform/handlers/deviantart.js +50 -0
  30. package/dist/feeds/platform/handlers/devto.cjs +44 -0
  31. package/dist/feeds/platform/handlers/devto.js +44 -0
  32. package/dist/feeds/platform/handlers/github.cjs +9 -3
  33. package/dist/feeds/platform/handlers/github.js +9 -3
  34. package/dist/feeds/platform/handlers/githubGist.cjs +42 -0
  35. package/dist/feeds/platform/handlers/githubGist.js +42 -0
  36. package/dist/feeds/platform/handlers/gitlab.cjs +36 -2
  37. package/dist/feeds/platform/handlers/gitlab.js +37 -3
  38. package/dist/feeds/platform/handlers/kickstarter.cjs +4 -7
  39. package/dist/feeds/platform/handlers/kickstarter.js +4 -7
  40. package/dist/feeds/platform/handlers/lobsters.cjs +34 -0
  41. package/dist/feeds/platform/handlers/lobsters.js +34 -0
  42. package/dist/feeds/platform/handlers/medium.cjs +47 -0
  43. package/dist/feeds/platform/handlers/medium.js +47 -0
  44. package/dist/feeds/platform/handlers/pinterest.cjs +48 -0
  45. package/dist/feeds/platform/handlers/pinterest.js +48 -0
  46. package/dist/feeds/platform/handlers/producthunt.cjs +22 -0
  47. package/dist/feeds/platform/handlers/producthunt.js +22 -0
  48. package/dist/feeds/platform/handlers/reddit.cjs +3 -0
  49. package/dist/feeds/platform/handlers/reddit.js +3 -0
  50. package/dist/feeds/platform/handlers/soundcloud.cjs +1 -1
  51. package/dist/feeds/platform/handlers/soundcloud.js +2 -2
  52. package/dist/feeds/platform/handlers/tumblr.cjs +4 -1
  53. package/dist/feeds/platform/handlers/tumblr.js +4 -1
  54. package/dist/feeds/platform/handlers/wordpress.cjs +17 -6
  55. package/dist/feeds/platform/handlers/wordpress.js +17 -6
  56. package/dist/feeds/platform/handlers/youtube.cjs +16 -4
  57. package/dist/feeds/platform/handlers/youtube.js +16 -4
  58. package/dist/hubs/discover/index.cjs +7 -6
  59. package/dist/hubs/discover/index.js +5 -4
  60. package/dist/hubs/discover/types.d.cts +2 -1
  61. package/dist/hubs/discover/types.d.ts +2 -1
  62. package/dist/hubs/headers/index.cjs +3 -3
  63. package/dist/hubs/headers/index.js +3 -3
  64. package/dist/hubs/html/index.cjs +3 -3
  65. package/dist/hubs/html/index.js +3 -3
  66. package/dist/index.d.cts +2 -2
  67. package/dist/index.d.ts +2 -2
  68. package/dist/utils.cjs +8 -0
  69. package/dist/utils.d.cts +2 -0
  70. package/dist/utils.d.ts +2 -0
  71. package/dist/utils.js +3 -0
  72. package/package.json +18 -19
  73. package/dist/adapters.cjs +0 -6
  74. package/dist/adapters.d.cts +0 -2
  75. package/dist/adapters.d.ts +0 -2
  76. package/dist/adapters.js +0 -3
  77. package/dist/common/discover/adapters.cjs +0 -76
  78. package/dist/common/discover/adapters.d.cts +0 -10
  79. package/dist/common/discover/adapters.d.ts +0 -10
  80. package/dist/common/discover/adapters.js +0 -72
package/README.md CHANGED
@@ -56,9 +56,7 @@ npm install feedscout
56
56
  ```typescript
57
57
  import { discoverFeeds } from 'feedscout'
58
58
 
59
- const feeds = await discoverFeeds('https://example.com', {
60
- methods: ['html', 'headers'],
61
- })
59
+ const feeds = await discoverFeeds('https://example.com')
62
60
 
63
61
  // [{
64
62
  // url: 'https://example.com/feed.xml',
@@ -70,6 +68,12 @@ const feeds = await discoverFeeds('https://example.com', {
70
68
  // }]
71
69
  ```
72
70
 
71
+ ```typescript
72
+ const feeds = await discoverFeeds('https://example.com', {
73
+ methods: ['html', 'headers'],
74
+ })
75
+ ```
76
+
73
77
  Or with existing HTML content:
74
78
 
75
79
  ```html
@@ -136,9 +140,7 @@ const feeds = await discoverFeeds(
136
140
  ```typescript
137
141
  import { discoverBlogrolls } from 'feedscout'
138
142
 
139
- const blogrolls = await discoverBlogrolls('https://example.com', {
140
- methods: ['html'],
141
- })
143
+ const blogrolls = await discoverBlogrolls('https://example.com')
142
144
 
143
145
  // [{
144
146
  // url: 'https://example.com/blogroll.opml',
@@ -1,14 +1,19 @@
1
- const require_adapters = require('../common/discover/adapters.cjs');
2
1
  const require_utils = require('../common/utils.cjs');
2
+ const require_utils$1 = require('../common/discover/utils.cjs');
3
3
  const require_index = require('../common/discover/index.cjs');
4
4
  const require_defaults = require('./defaults.cjs');
5
5
  const require_extractors = require('./extractors.cjs');
6
6
 
7
7
  //#region src/blogrolls/index.ts
8
- const discoverBlogrolls = async (input, options) => {
8
+ const discoverBlogrolls = async (input, options = {}) => {
9
9
  return require_index.discover(input, {
10
10
  ...options,
11
- fetchFn: options.fetchFn ?? require_adapters.createNativeFetchAdapter(),
11
+ methods: options.methods ?? [
12
+ "html",
13
+ "headers",
14
+ "guess"
15
+ ],
16
+ fetchFn: options.fetchFn ?? require_utils$1.defaultFetchFn,
12
17
  extractFn: options.extractFn ?? require_extractors.defaultExtractor,
13
18
  normalizeUrlFn: options.normalizeUrlFn ?? require_utils.normalizeUrl
14
19
  }, {
@@ -2,6 +2,6 @@ import { DiscoverInput, DiscoverOptions, DiscoverResult } from "../common/types.
2
2
  import { BlogrollResult } from "./types.cjs";
3
3
 
4
4
  //#region src/blogrolls/index.d.ts
5
- declare const discoverBlogrolls: <TValid extends BlogrollResult = BlogrollResult>(input: DiscoverInput, options: DiscoverOptions<TValid>) => Promise<Array<DiscoverResult<TValid>>>;
5
+ declare const discoverBlogrolls: <TValid extends BlogrollResult = BlogrollResult>(input: DiscoverInput, options?: DiscoverOptions<TValid>) => Promise<Array<DiscoverResult<TValid>>>;
6
6
  //#endregion
7
7
  export { discoverBlogrolls };
@@ -2,6 +2,6 @@ import { DiscoverInput, DiscoverOptions, DiscoverResult } from "../common/types.
2
2
  import { BlogrollResult } from "./types.js";
3
3
 
4
4
  //#region src/blogrolls/index.d.ts
5
- declare const discoverBlogrolls: <TValid extends BlogrollResult = BlogrollResult>(input: DiscoverInput, options: DiscoverOptions<TValid>) => Promise<Array<DiscoverResult<TValid>>>;
5
+ declare const discoverBlogrolls: <TValid extends BlogrollResult = BlogrollResult>(input: DiscoverInput, options?: DiscoverOptions<TValid>) => Promise<Array<DiscoverResult<TValid>>>;
6
6
  //#endregion
7
7
  export { discoverBlogrolls };
@@ -1,14 +1,19 @@
1
- import { createNativeFetchAdapter } from "../common/discover/adapters.js";
2
1
  import { normalizeUrl } from "../common/utils.js";
2
+ import { defaultFetchFn } from "../common/discover/utils.js";
3
3
  import { discover } from "../common/discover/index.js";
4
4
  import { defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions } from "./defaults.js";
5
5
  import { defaultExtractor } from "./extractors.js";
6
6
 
7
7
  //#region src/blogrolls/index.ts
8
- const discoverBlogrolls = async (input, options) => {
8
+ const discoverBlogrolls = async (input, options = {}) => {
9
9
  return discover(input, {
10
10
  ...options,
11
- fetchFn: options.fetchFn ?? createNativeFetchAdapter(),
11
+ methods: options.methods ?? [
12
+ "html",
13
+ "headers",
14
+ "guess"
15
+ ],
16
+ fetchFn: options.fetchFn ?? defaultFetchFn,
12
17
  extractFn: options.extractFn ?? defaultExtractor,
13
18
  normalizeUrlFn: options.normalizeUrlFn ?? normalizeUrl
14
19
  }, {
@@ -1,6 +1,19 @@
1
1
  const require_locales = require('../locales.cjs');
2
2
 
3
3
  //#region src/common/discover/utils.ts
4
+ const defaultFetchFn = async (url, options) => {
5
+ const response = await fetch(url, {
6
+ method: options?.method ?? "GET",
7
+ headers: options?.headers
8
+ });
9
+ return {
10
+ headers: response.headers,
11
+ body: await response.text(),
12
+ url: response.url,
13
+ status: response.status,
14
+ statusText: response.statusText
15
+ };
16
+ };
4
17
  const normalizeInput = async (input, fetchFn) => {
5
18
  if (typeof input === "object") return input;
6
19
  const response = await fetchFn(input);
@@ -62,5 +75,6 @@ const normalizeMethodsConfig = (input, methods, defaults) => {
62
75
  };
63
76
 
64
77
  //#endregion
78
+ exports.defaultFetchFn = defaultFetchFn;
65
79
  exports.normalizeInput = normalizeInput;
66
80
  exports.normalizeMethodsConfig = normalizeMethodsConfig;
@@ -1,6 +1,19 @@
1
1
  import { errors } from "../locales.js";
2
2
 
3
3
  //#region src/common/discover/utils.ts
4
+ const defaultFetchFn = async (url, options) => {
5
+ const response = await fetch(url, {
6
+ method: options?.method ?? "GET",
7
+ headers: options?.headers
8
+ });
9
+ return {
10
+ headers: response.headers,
11
+ body: await response.text(),
12
+ url: response.url,
13
+ status: response.status,
14
+ statusText: response.statusText
15
+ };
16
+ };
4
17
  const normalizeInput = async (input, fetchFn) => {
5
18
  if (typeof input === "object") return input;
6
19
  const response = await fetchFn(input);
@@ -62,4 +75,4 @@ const normalizeMethodsConfig = (input, methods, defaults) => {
62
75
  };
63
76
 
64
77
  //#endregion
65
- export { normalizeInput, normalizeMethodsConfig };
78
+ export { defaultFetchFn, normalizeInput, normalizeMethodsConfig };
@@ -27,7 +27,7 @@ type DiscoverProgress = {
27
27
  found: number;
28
28
  current: string;
29
29
  };
30
- type DiscoverProgressFn = (progress: DiscoverProgress) => void;
30
+ type DiscoverOnProgressFn = (progress: DiscoverProgress) => void;
31
31
  type DiscoverResult<TValid = object> = ({
32
32
  url: string;
33
33
  isValid: true;
@@ -54,14 +54,14 @@ type DiscoverMethodsConfig = Array<'platform' | 'html' | 'headers' | 'guess'> |
54
54
  guess?: true | Partial<Omit<GuessMethodOptions, 'baseUrl'>>;
55
55
  };
56
56
  type DiscoverOptions<TValid> = {
57
- methods: DiscoverMethodsConfig;
57
+ methods?: DiscoverMethodsConfig;
58
58
  fetchFn?: DiscoverFetchFn;
59
59
  extractFn?: DiscoverExtractFn<TValid>;
60
60
  normalizeUrlFn?: DiscoverNormalizeUrlFn;
61
61
  concurrency?: number;
62
62
  stopOnFirstResult?: boolean;
63
63
  includeInvalid?: boolean;
64
- onProgress?: DiscoverProgressFn;
64
+ onProgress?: DiscoverOnProgressFn;
65
65
  };
66
66
  //#endregion
67
- export { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethodsConfig, DiscoverNormalizeUrlFn, DiscoverOptions, DiscoverProgress, DiscoverProgressFn, DiscoverResult, LinkSelector };
67
+ export { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethodsConfig, DiscoverNormalizeUrlFn, DiscoverOnProgressFn, DiscoverOptions, DiscoverProgress, DiscoverResult, LinkSelector };
@@ -27,7 +27,7 @@ type DiscoverProgress = {
27
27
  found: number;
28
28
  current: string;
29
29
  };
30
- type DiscoverProgressFn = (progress: DiscoverProgress) => void;
30
+ type DiscoverOnProgressFn = (progress: DiscoverProgress) => void;
31
31
  type DiscoverResult<TValid = object> = ({
32
32
  url: string;
33
33
  isValid: true;
@@ -54,14 +54,14 @@ type DiscoverMethodsConfig = Array<'platform' | 'html' | 'headers' | 'guess'> |
54
54
  guess?: true | Partial<Omit<GuessMethodOptions, 'baseUrl'>>;
55
55
  };
56
56
  type DiscoverOptions<TValid> = {
57
- methods: DiscoverMethodsConfig;
57
+ methods?: DiscoverMethodsConfig;
58
58
  fetchFn?: DiscoverFetchFn;
59
59
  extractFn?: DiscoverExtractFn<TValid>;
60
60
  normalizeUrlFn?: DiscoverNormalizeUrlFn;
61
61
  concurrency?: number;
62
62
  stopOnFirstResult?: boolean;
63
63
  includeInvalid?: boolean;
64
- onProgress?: DiscoverProgressFn;
64
+ onProgress?: DiscoverOnProgressFn;
65
65
  };
66
66
  //#endregion
67
- export { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethodsConfig, DiscoverNormalizeUrlFn, DiscoverOptions, DiscoverProgress, DiscoverProgressFn, DiscoverResult, LinkSelector };
67
+ export { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethodsConfig, DiscoverNormalizeUrlFn, DiscoverOnProgressFn, DiscoverOptions, DiscoverProgress, DiscoverResult, LinkSelector };
@@ -1,15 +1,18 @@
1
1
  const require_utils = require('../../utils.cjs');
2
2
 
3
3
  //#region src/common/uris/headers/index.ts
4
+ const urlRegex = /<([^<>]+)>/;
5
+ const relRegex = /rel\s*=\s*["']?([^"';,]+)["']?/i;
6
+ const typeRegex = /type\s*=\s*["']?([^"';,]+)["']?/i;
4
7
  const discoverUrisFromHeaders = (headers, options) => {
5
8
  const uris = /* @__PURE__ */ new Set();
6
9
  const linkHeader = headers.get("link");
7
10
  if (!linkHeader) return [];
8
11
  const links = linkHeader.split(/,(?=\s*<)/);
9
12
  for (const link of links) {
10
- const urlMatch = link.match(/<([^<>]+)>/);
11
- const relMatch = link.match(/rel\s*=\s*["']?([^"';,]+)["']?/i);
12
- const typeMatch = link.match(/type\s*=\s*["']?([^"';,]+)["']?/i);
13
+ const urlMatch = link.match(urlRegex);
14
+ const relMatch = link.match(relRegex);
15
+ const typeMatch = link.match(typeRegex);
13
16
  if (!urlMatch) continue;
14
17
  const url = urlMatch[1];
15
18
  const rel = relMatch?.[1]?.toLowerCase();
@@ -1,15 +1,18 @@
1
1
  import { matchesAnyOfLinkSelectors } from "../../utils.js";
2
2
 
3
3
  //#region src/common/uris/headers/index.ts
4
+ const urlRegex = /<([^<>]+)>/;
5
+ const relRegex = /rel\s*=\s*["']?([^"';,]+)["']?/i;
6
+ const typeRegex = /type\s*=\s*["']?([^"';,]+)["']?/i;
4
7
  const discoverUrisFromHeaders = (headers, options) => {
5
8
  const uris = /* @__PURE__ */ new Set();
6
9
  const linkHeader = headers.get("link");
7
10
  if (!linkHeader) return [];
8
11
  const links = linkHeader.split(/,(?=\s*<)/);
9
12
  for (const link of links) {
10
- const urlMatch = link.match(/<([^<>]+)>/);
11
- const relMatch = link.match(/rel\s*=\s*["']?([^"';,]+)["']?/i);
12
- const typeMatch = link.match(/type\s*=\s*["']?([^"';,]+)["']?/i);
13
+ const urlMatch = link.match(urlRegex);
14
+ const relMatch = link.match(relRegex);
15
+ const typeMatch = link.match(typeRegex);
13
16
  if (!urlMatch) continue;
14
17
  const url = urlMatch[1];
15
18
  const rel = relMatch?.[1]?.toLowerCase();
@@ -56,6 +56,7 @@ const processConcurrently = async (items, processFn, options) => {
56
56
  };
57
57
 
58
58
  //#endregion
59
+ exports.anyWordMatchesAnyOf = anyWordMatchesAnyOf;
59
60
  exports.endsWithAnyOf = endsWithAnyOf;
60
61
  exports.includesAnyOf = includesAnyOf;
61
62
  exports.isAnyOf = isAnyOf;
@@ -0,0 +1,9 @@
1
+ //#region src/common/utils.d.ts
2
+ declare const isSubdomainOf: (url: string, domain: string) => boolean;
3
+ declare const isHostOf: (url: string, hosts: Array<string>) => boolean;
4
+ declare const includesAnyOf: (value: string, patterns: Array<string>, parser?: (value: string) => string) => boolean;
5
+ declare const isAnyOf: (value: string, patterns: Array<string>, parser?: (value: string) => string) => boolean;
6
+ declare const anyWordMatchesAnyOf: (value: string, patterns: Array<string>) => boolean;
7
+ declare const endsWithAnyOf: (value: string, patterns: Array<string>) => boolean;
8
+ //#endregion
9
+ export { anyWordMatchesAnyOf, endsWithAnyOf, includesAnyOf, isAnyOf, isHostOf, isSubdomainOf };
@@ -0,0 +1,9 @@
1
+ //#region src/common/utils.d.ts
2
+ declare const isSubdomainOf: (url: string, domain: string) => boolean;
3
+ declare const isHostOf: (url: string, hosts: Array<string>) => boolean;
4
+ declare const includesAnyOf: (value: string, patterns: Array<string>, parser?: (value: string) => string) => boolean;
5
+ declare const isAnyOf: (value: string, patterns: Array<string>, parser?: (value: string) => string) => boolean;
6
+ declare const anyWordMatchesAnyOf: (value: string, patterns: Array<string>) => boolean;
7
+ declare const endsWithAnyOf: (value: string, patterns: Array<string>) => boolean;
8
+ //#endregion
9
+ export { anyWordMatchesAnyOf, endsWithAnyOf, includesAnyOf, isAnyOf, isHostOf, isSubdomainOf };
@@ -55,4 +55,4 @@ const processConcurrently = async (items, processFn, options) => {
55
55
  };
56
56
 
57
57
  //#endregion
58
- export { endsWithAnyOf, includesAnyOf, isAnyOf, isHostOf, isSubdomainOf, matchesAnyOfLinkSelectors, normalizeUrl, processConcurrently };
58
+ export { anyWordMatchesAnyOf, endsWithAnyOf, includesAnyOf, isAnyOf, isHostOf, isSubdomainOf, matchesAnyOfLinkSelectors, normalizeUrl, processConcurrently };
@@ -1,8 +1,17 @@
1
+ const require_behance = require('./platform/handlers/behance.cjs');
1
2
  const require_blogspot = require('./platform/handlers/blogspot.cjs');
2
3
  const require_bluesky = require('./platform/handlers/bluesky.cjs');
4
+ const require_dailymotion = require('./platform/handlers/dailymotion.cjs');
5
+ const require_deviantart = require('./platform/handlers/deviantart.cjs');
6
+ const require_devto = require('./platform/handlers/devto.cjs');
3
7
  const require_github = require('./platform/handlers/github.cjs');
8
+ const require_githubGist = require('./platform/handlers/githubGist.cjs');
4
9
  const require_gitlab = require('./platform/handlers/gitlab.cjs');
5
10
  const require_kickstarter = require('./platform/handlers/kickstarter.cjs');
11
+ const require_lobsters = require('./platform/handlers/lobsters.cjs');
12
+ const require_medium = require('./platform/handlers/medium.cjs');
13
+ const require_pinterest = require('./platform/handlers/pinterest.cjs');
14
+ const require_producthunt = require('./platform/handlers/producthunt.cjs');
6
15
  const require_reddit = require('./platform/handlers/reddit.cjs');
7
16
  const require_soundcloud = require('./platform/handlers/soundcloud.cjs');
8
17
  const require_substack = require('./platform/handlers/substack.cjs');
@@ -87,11 +96,20 @@ const defaultHtmlOptions = {
87
96
  const defaultHeadersOptions = { linkSelectors };
88
97
  const defaultGuessOptions = { uris: urisBalanced };
89
98
  const defaultPlatformOptions = { handlers: [
99
+ require_behance.behanceHandler,
90
100
  require_blogspot.blogspotHandler,
91
101
  require_bluesky.blueskyHandler,
102
+ require_deviantart.deviantartHandler,
103
+ require_dailymotion.dailymotionHandler,
104
+ require_devto.devtoHandler,
92
105
  require_github.githubHandler,
106
+ require_githubGist.githubGistHandler,
93
107
  require_gitlab.gitlabHandler,
94
108
  require_kickstarter.kickstarterHandler,
109
+ require_medium.mediumHandler,
110
+ require_pinterest.pinterestHandler,
111
+ require_producthunt.producthuntHandler,
112
+ require_lobsters.lobstersHandler,
95
113
  require_reddit.redditHandler,
96
114
  require_soundcloud.soundcloudHandler,
97
115
  require_substack.substackHandler,
@@ -1,8 +1,17 @@
1
+ import { behanceHandler } from "./platform/handlers/behance.js";
1
2
  import { blogspotHandler } from "./platform/handlers/blogspot.js";
2
3
  import { blueskyHandler } from "./platform/handlers/bluesky.js";
4
+ import { dailymotionHandler } from "./platform/handlers/dailymotion.js";
5
+ import { deviantartHandler } from "./platform/handlers/deviantart.js";
6
+ import { devtoHandler } from "./platform/handlers/devto.js";
3
7
  import { githubHandler } from "./platform/handlers/github.js";
8
+ import { githubGistHandler } from "./platform/handlers/githubGist.js";
4
9
  import { gitlabHandler } from "./platform/handlers/gitlab.js";
5
10
  import { kickstarterHandler } from "./platform/handlers/kickstarter.js";
11
+ import { lobstersHandler } from "./platform/handlers/lobsters.js";
12
+ import { mediumHandler } from "./platform/handlers/medium.js";
13
+ import { pinterestHandler } from "./platform/handlers/pinterest.js";
14
+ import { producthuntHandler } from "./platform/handlers/producthunt.js";
6
15
  import { redditHandler } from "./platform/handlers/reddit.js";
7
16
  import { soundcloudHandler } from "./platform/handlers/soundcloud.js";
8
17
  import { substackHandler } from "./platform/handlers/substack.js";
@@ -87,11 +96,20 @@ const defaultHtmlOptions = {
87
96
  const defaultHeadersOptions = { linkSelectors };
88
97
  const defaultGuessOptions = { uris: urisBalanced };
89
98
  const defaultPlatformOptions = { handlers: [
99
+ behanceHandler,
90
100
  blogspotHandler,
91
101
  blueskyHandler,
102
+ deviantartHandler,
103
+ dailymotionHandler,
104
+ devtoHandler,
92
105
  githubHandler,
106
+ githubGistHandler,
93
107
  gitlabHandler,
94
108
  kickstarterHandler,
109
+ mediumHandler,
110
+ pinterestHandler,
111
+ producthuntHandler,
112
+ lobstersHandler,
95
113
  redditHandler,
96
114
  soundcloudHandler,
97
115
  substackHandler,
@@ -1,14 +1,20 @@
1
- const require_adapters = require('../common/discover/adapters.cjs');
2
1
  const require_utils = require('../common/utils.cjs');
2
+ const require_utils$1 = require('../common/discover/utils.cjs');
3
3
  const require_index = require('../common/discover/index.cjs');
4
4
  const require_defaults = require('./defaults.cjs');
5
5
  const require_extractors = require('./extractors.cjs');
6
6
 
7
7
  //#region src/feeds/index.ts
8
- const discoverFeeds = async (input, options) => {
8
+ const discoverFeeds = async (input, options = {}) => {
9
9
  return require_index.discover(input, {
10
10
  ...options,
11
- fetchFn: options.fetchFn ?? require_adapters.createNativeFetchAdapter(),
11
+ methods: options.methods ?? [
12
+ "platform",
13
+ "html",
14
+ "headers",
15
+ "guess"
16
+ ],
17
+ fetchFn: options.fetchFn ?? require_utils$1.defaultFetchFn,
12
18
  extractFn: options.extractFn ?? require_extractors.defaultExtractor,
13
19
  normalizeUrlFn: options.normalizeUrlFn ?? require_utils.normalizeUrl
14
20
  }, {
@@ -2,6 +2,6 @@ import { DiscoverInput, DiscoverOptions, DiscoverResult } from "../common/types.
2
2
  import { FeedResult } from "./types.cjs";
3
3
 
4
4
  //#region src/feeds/index.d.ts
5
- declare const discoverFeeds: <TValid extends FeedResult = FeedResult>(input: DiscoverInput, options: DiscoverOptions<TValid>) => Promise<Array<DiscoverResult<TValid>>>;
5
+ declare const discoverFeeds: <TValid extends FeedResult = FeedResult>(input: DiscoverInput, options?: DiscoverOptions<TValid>) => Promise<Array<DiscoverResult<TValid>>>;
6
6
  //#endregion
7
7
  export { discoverFeeds };
@@ -2,6 +2,6 @@ import { DiscoverInput, DiscoverOptions, DiscoverResult } from "../common/types.
2
2
  import { FeedResult } from "./types.js";
3
3
 
4
4
  //#region src/feeds/index.d.ts
5
- declare const discoverFeeds: <TValid extends FeedResult = FeedResult>(input: DiscoverInput, options: DiscoverOptions<TValid>) => Promise<Array<DiscoverResult<TValid>>>;
5
+ declare const discoverFeeds: <TValid extends FeedResult = FeedResult>(input: DiscoverInput, options?: DiscoverOptions<TValid>) => Promise<Array<DiscoverResult<TValid>>>;
6
6
  //#endregion
7
7
  export { discoverFeeds };
@@ -1,14 +1,20 @@
1
- import { createNativeFetchAdapter } from "../common/discover/adapters.js";
2
1
  import { normalizeUrl } from "../common/utils.js";
2
+ import { defaultFetchFn } from "../common/discover/utils.js";
3
3
  import { discover } from "../common/discover/index.js";
4
4
  import { defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions } from "./defaults.js";
5
5
  import { defaultExtractor } from "./extractors.js";
6
6
 
7
7
  //#region src/feeds/index.ts
8
- const discoverFeeds = async (input, options) => {
8
+ const discoverFeeds = async (input, options = {}) => {
9
9
  return discover(input, {
10
10
  ...options,
11
- fetchFn: options.fetchFn ?? createNativeFetchAdapter(),
11
+ methods: options.methods ?? [
12
+ "platform",
13
+ "html",
14
+ "headers",
15
+ "guess"
16
+ ],
17
+ fetchFn: options.fetchFn ?? defaultFetchFn,
12
18
  extractFn: options.extractFn ?? defaultExtractor,
13
19
  normalizeUrlFn: options.normalizeUrlFn ?? normalizeUrl
14
20
  }, {
@@ -0,0 +1,45 @@
1
+ const require_utils = require('../../../common/utils.cjs');
2
+
3
+ //#region src/feeds/platform/handlers/behance.ts
4
+ const hosts = ["behance.net", "www.behance.net"];
5
+ const userPathRegex = /^\/([a-zA-Z0-9_-]+)(?:\/(appreciated))?\/?$/;
6
+ const excludedPaths = [
7
+ "search",
8
+ "galleries",
9
+ "curated",
10
+ "features",
11
+ "live",
12
+ "joblist",
13
+ "hire",
14
+ "blog",
15
+ "about",
16
+ "privacy",
17
+ "tos",
18
+ "help",
19
+ "onboarding",
20
+ "settings",
21
+ "notifications",
22
+ "messages",
23
+ "adobe"
24
+ ];
25
+ const behanceHandler = {
26
+ match: (url) => {
27
+ return require_utils.isHostOf(url, hosts);
28
+ },
29
+ resolve: (url) => {
30
+ const { pathname } = new URL(url);
31
+ const userMatch = pathname.match(userPathRegex);
32
+ if (userMatch?.[1]) {
33
+ const username = userMatch[1];
34
+ const subpage = userMatch[2];
35
+ if (!require_utils.isAnyOf(username, excludedPaths)) {
36
+ if (subpage === "appreciated") return [`https://www.behance.net/feeds/user?username=${username}&content=appreciated`];
37
+ return [`https://www.behance.net/feeds/user?username=${username}`];
38
+ }
39
+ }
40
+ return [];
41
+ }
42
+ };
43
+
44
+ //#endregion
45
+ exports.behanceHandler = behanceHandler;
@@ -0,0 +1,45 @@
1
+ import { isAnyOf, isHostOf } from "../../../common/utils.js";
2
+
3
+ //#region src/feeds/platform/handlers/behance.ts
4
+ const hosts = ["behance.net", "www.behance.net"];
5
+ const userPathRegex = /^\/([a-zA-Z0-9_-]+)(?:\/(appreciated))?\/?$/;
6
+ const excludedPaths = [
7
+ "search",
8
+ "galleries",
9
+ "curated",
10
+ "features",
11
+ "live",
12
+ "joblist",
13
+ "hire",
14
+ "blog",
15
+ "about",
16
+ "privacy",
17
+ "tos",
18
+ "help",
19
+ "onboarding",
20
+ "settings",
21
+ "notifications",
22
+ "messages",
23
+ "adobe"
24
+ ];
25
+ const behanceHandler = {
26
+ match: (url) => {
27
+ return isHostOf(url, hosts);
28
+ },
29
+ resolve: (url) => {
30
+ const { pathname } = new URL(url);
31
+ const userMatch = pathname.match(userPathRegex);
32
+ if (userMatch?.[1]) {
33
+ const username = userMatch[1];
34
+ const subpage = userMatch[2];
35
+ if (!isAnyOf(username, excludedPaths)) {
36
+ if (subpage === "appreciated") return [`https://www.behance.net/feeds/user?username=${username}&content=appreciated`];
37
+ return [`https://www.behance.net/feeds/user?username=${username}`];
38
+ }
39
+ }
40
+ return [];
41
+ }
42
+ };
43
+
44
+ //#endregion
45
+ export { behanceHandler };
@@ -1,13 +1,22 @@
1
- const require_utils = require('../../../common/utils.cjs');
2
1
 
3
2
  //#region src/feeds/platform/handlers/blogspot.ts
3
+ const blogspotDomainRegex = /^.+\.blogspot\.(?:com|co\.[a-z]{2}|com\.[a-z]{2}|[a-z]{2,3})$/;
4
4
  const blogspotHandler = {
5
5
  match: (url) => {
6
- return require_utils.isSubdomainOf(url, "blogspot.com");
6
+ const hostname = new URL(url).hostname.toLowerCase();
7
+ return blogspotDomainRegex.test(hostname);
7
8
  },
8
9
  resolve: (url) => {
9
- const { origin } = new URL(url);
10
- return [`${origin}/feeds/posts/default`, `${origin}/feeds/posts/default?alt=rss`];
10
+ const { origin, pathname } = new URL(url);
11
+ const uris = [];
12
+ const labelMatch = pathname.match(/^\/search\/label\/([^/]+)/);
13
+ if (labelMatch?.[1]) {
14
+ const label = labelMatch[1];
15
+ uris.push(`${origin}/feeds/posts/default/-/${label}`);
16
+ }
17
+ uris.push(`${origin}/feeds/posts/default`);
18
+ uris.push(`${origin}/feeds/posts/default?alt=rss`);
19
+ return uris;
11
20
  }
12
21
  };
13
22
 
@@ -1,13 +1,21 @@
1
- import { isSubdomainOf } from "../../../common/utils.js";
2
-
3
1
  //#region src/feeds/platform/handlers/blogspot.ts
2
+ const blogspotDomainRegex = /^.+\.blogspot\.(?:com|co\.[a-z]{2}|com\.[a-z]{2}|[a-z]{2,3})$/;
4
3
  const blogspotHandler = {
5
4
  match: (url) => {
6
- return isSubdomainOf(url, "blogspot.com");
5
+ const hostname = new URL(url).hostname.toLowerCase();
6
+ return blogspotDomainRegex.test(hostname);
7
7
  },
8
8
  resolve: (url) => {
9
- const { origin } = new URL(url);
10
- return [`${origin}/feeds/posts/default`, `${origin}/feeds/posts/default?alt=rss`];
9
+ const { origin, pathname } = new URL(url);
10
+ const uris = [];
11
+ const labelMatch = pathname.match(/^\/search\/label\/([^/]+)/);
12
+ if (labelMatch?.[1]) {
13
+ const label = labelMatch[1];
14
+ uris.push(`${origin}/feeds/posts/default/-/${label}`);
15
+ }
16
+ uris.push(`${origin}/feeds/posts/default`);
17
+ uris.push(`${origin}/feeds/posts/default?alt=rss`);
18
+ return uris;
11
19
  }
12
20
  };
13
21
 
@@ -0,0 +1,66 @@
1
+ const require_utils = require('../../../common/utils.cjs');
2
+
3
+ //#region src/feeds/platform/handlers/dailymotion.ts
4
+ const hosts = ["dailymotion.com", "www.dailymotion.com"];
5
+ const userPathRegex = /^\/([a-zA-Z0-9_-]+)$/;
6
+ const playlistPathRegex = /^\/playlist\/([a-zA-Z0-9_-]+)/;
7
+ const excludedPaths = [
8
+ "signin",
9
+ "signout",
10
+ "signup",
11
+ "login",
12
+ "logout",
13
+ "register",
14
+ "search",
15
+ "legal",
16
+ "about",
17
+ "careers",
18
+ "terms",
19
+ "privacy",
20
+ "feedback",
21
+ "help",
22
+ "settings",
23
+ "upload",
24
+ "partner",
25
+ "monetize",
26
+ "studio",
27
+ "video",
28
+ "live",
29
+ "channels",
30
+ "playlist",
31
+ "topics",
32
+ "trending",
33
+ "dm",
34
+ "creator",
35
+ "premium",
36
+ "explore",
37
+ "following",
38
+ "subscriptions",
39
+ "notifications",
40
+ "history",
41
+ "watch",
42
+ "contact",
43
+ "ads",
44
+ "dmca",
45
+ "copyright",
46
+ "community"
47
+ ];
48
+ const dailymotionHandler = {
49
+ match: (url) => {
50
+ return require_utils.isHostOf(url, hosts);
51
+ },
52
+ resolve: (url) => {
53
+ const { pathname } = new URL(url);
54
+ const playlistMatch = pathname.match(playlistPathRegex);
55
+ if (playlistMatch?.[1]) return [`https://www.dailymotion.com/rss/playlist/${playlistMatch[1]}`];
56
+ const userMatch = pathname.match(userPathRegex);
57
+ if (userMatch?.[1]) {
58
+ const username = userMatch[1];
59
+ if (!require_utils.isAnyOf(username, excludedPaths)) return [`https://www.dailymotion.com/rss/${username}`];
60
+ }
61
+ return [];
62
+ }
63
+ };
64
+
65
+ //#endregion
66
+ exports.dailymotionHandler = dailymotionHandler;