feedscout 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/dist/blogrolls/index.cjs +8 -3
- package/dist/blogrolls/index.d.cts +1 -1
- package/dist/blogrolls/index.d.ts +1 -1
- package/dist/blogrolls/index.js +8 -3
- package/dist/blogrolls.cjs +1 -0
- package/dist/common/discover/utils.cjs +14 -0
- package/dist/common/discover/utils.js +14 -1
- package/dist/common/types.d.cts +4 -4
- package/dist/common/types.d.ts +4 -4
- package/dist/common/uris/headers/index.cjs +6 -3
- package/dist/common/uris/headers/index.js +6 -3
- package/dist/common/utils.cjs +1 -0
- package/dist/common/utils.d.cts +9 -0
- package/dist/common/utils.d.ts +9 -0
- package/dist/common/utils.js +1 -1
- package/dist/feeds/defaults.cjs +39 -8
- package/dist/feeds/defaults.js +39 -8
- package/dist/feeds/index.cjs +9 -3
- package/dist/feeds/index.d.cts +1 -1
- package/dist/feeds/index.d.ts +1 -1
- package/dist/feeds/index.js +9 -3
- package/dist/feeds/platform/handlers/behance.cjs +45 -0
- package/dist/feeds/platform/handlers/behance.js +45 -0
- package/dist/feeds/platform/handlers/blogspot.cjs +13 -4
- package/dist/feeds/platform/handlers/blogspot.js +13 -5
- package/dist/feeds/platform/handlers/dailymotion.cjs +66 -0
- package/dist/feeds/platform/handlers/dailymotion.js +66 -0
- package/dist/feeds/platform/handlers/deviantart.cjs +50 -0
- package/dist/feeds/platform/handlers/deviantart.js +50 -0
- package/dist/feeds/platform/handlers/devto.cjs +44 -0
- package/dist/feeds/platform/handlers/devto.js +44 -0
- package/dist/feeds/platform/handlers/github.cjs +9 -3
- package/dist/feeds/platform/handlers/github.js +9 -3
- package/dist/feeds/platform/handlers/githubGist.cjs +42 -0
- package/dist/feeds/platform/handlers/githubGist.js +42 -0
- package/dist/feeds/platform/handlers/gitlab.cjs +36 -2
- package/dist/feeds/platform/handlers/gitlab.js +37 -3
- package/dist/feeds/platform/handlers/kickstarter.cjs +4 -7
- package/dist/feeds/platform/handlers/kickstarter.js +4 -7
- package/dist/feeds/platform/handlers/lobsters.cjs +34 -0
- package/dist/feeds/platform/handlers/lobsters.js +34 -0
- package/dist/feeds/platform/handlers/medium.cjs +47 -0
- package/dist/feeds/platform/handlers/medium.js +47 -0
- package/dist/feeds/platform/handlers/pinterest.cjs +48 -0
- package/dist/feeds/platform/handlers/pinterest.js +48 -0
- package/dist/feeds/platform/handlers/producthunt.cjs +22 -0
- package/dist/feeds/platform/handlers/producthunt.js +22 -0
- package/dist/feeds/platform/handlers/reddit.cjs +3 -0
- package/dist/feeds/platform/handlers/reddit.js +3 -0
- package/dist/feeds/platform/handlers/soundcloud.cjs +1 -1
- package/dist/feeds/platform/handlers/soundcloud.js +2 -2
- package/dist/feeds/platform/handlers/tumblr.cjs +4 -1
- package/dist/feeds/platform/handlers/tumblr.js +4 -1
- package/dist/feeds/platform/handlers/wordpress.cjs +17 -6
- package/dist/feeds/platform/handlers/wordpress.js +17 -6
- package/dist/feeds/platform/handlers/youtube.cjs +16 -4
- package/dist/feeds/platform/handlers/youtube.js +16 -4
- package/dist/feeds.cjs +1 -0
- package/dist/hubs/discover/index.cjs +7 -6
- package/dist/hubs/discover/index.js +5 -4
- package/dist/hubs/discover/types.d.cts +2 -1
- package/dist/hubs/discover/types.d.ts +2 -1
- package/dist/hubs/headers/index.cjs +3 -3
- package/dist/hubs/headers/index.js +3 -3
- package/dist/hubs/html/index.cjs +3 -3
- package/dist/hubs/html/index.js +3 -3
- package/dist/index.cjs +1 -0
- package/dist/index.d.cts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/methods.cjs +1 -0
- package/dist/utils.cjs +9 -0
- package/dist/utils.d.cts +2 -0
- package/dist/utils.d.ts +2 -0
- package/dist/utils.js +3 -0
- package/package.json +18 -19
- package/dist/adapters.cjs +0 -6
- package/dist/adapters.d.cts +0 -2
- package/dist/adapters.d.ts +0 -2
- package/dist/adapters.js +0 -3
- package/dist/common/discover/adapters.cjs +0 -76
- package/dist/common/discover/adapters.d.cts +0 -10
- package/dist/common/discover/adapters.d.ts +0 -10
- package/dist/common/discover/adapters.js +0 -72
package/README.md
CHANGED
|
@@ -56,9 +56,7 @@ npm install feedscout
|
|
|
56
56
|
```typescript
|
|
57
57
|
import { discoverFeeds } from 'feedscout'
|
|
58
58
|
|
|
59
|
-
const feeds = await discoverFeeds('https://example.com'
|
|
60
|
-
methods: ['html', 'headers'],
|
|
61
|
-
})
|
|
59
|
+
const feeds = await discoverFeeds('https://example.com')
|
|
62
60
|
|
|
63
61
|
// [{
|
|
64
62
|
// url: 'https://example.com/feed.xml',
|
|
@@ -70,6 +68,12 @@ const feeds = await discoverFeeds('https://example.com', {
|
|
|
70
68
|
// }]
|
|
71
69
|
```
|
|
72
70
|
|
|
71
|
+
```typescript
|
|
72
|
+
const feeds = await discoverFeeds('https://example.com', {
|
|
73
|
+
methods: ['html', 'headers'],
|
|
74
|
+
})
|
|
75
|
+
```
|
|
76
|
+
|
|
73
77
|
Or with existing HTML content:
|
|
74
78
|
|
|
75
79
|
```html
|
|
@@ -136,9 +140,7 @@ const feeds = await discoverFeeds(
|
|
|
136
140
|
```typescript
|
|
137
141
|
import { discoverBlogrolls } from 'feedscout'
|
|
138
142
|
|
|
139
|
-
const blogrolls = await discoverBlogrolls('https://example.com'
|
|
140
|
-
methods: ['html'],
|
|
141
|
-
})
|
|
143
|
+
const blogrolls = await discoverBlogrolls('https://example.com')
|
|
142
144
|
|
|
143
145
|
// [{
|
|
144
146
|
// url: 'https://example.com/blogroll.opml',
|
package/dist/blogrolls/index.cjs
CHANGED
|
@@ -1,14 +1,19 @@
|
|
|
1
|
-
const require_adapters = require('../common/discover/adapters.cjs');
|
|
2
1
|
const require_utils = require('../common/utils.cjs');
|
|
2
|
+
const require_utils$1 = require('../common/discover/utils.cjs');
|
|
3
3
|
const require_index = require('../common/discover/index.cjs');
|
|
4
4
|
const require_defaults = require('./defaults.cjs');
|
|
5
5
|
const require_extractors = require('./extractors.cjs');
|
|
6
6
|
|
|
7
7
|
//#region src/blogrolls/index.ts
|
|
8
|
-
const discoverBlogrolls = async (input, options) => {
|
|
8
|
+
const discoverBlogrolls = async (input, options = {}) => {
|
|
9
9
|
return require_index.discover(input, {
|
|
10
10
|
...options,
|
|
11
|
-
|
|
11
|
+
methods: options.methods ?? [
|
|
12
|
+
"html",
|
|
13
|
+
"headers",
|
|
14
|
+
"guess"
|
|
15
|
+
],
|
|
16
|
+
fetchFn: options.fetchFn ?? require_utils$1.defaultFetchFn,
|
|
12
17
|
extractFn: options.extractFn ?? require_extractors.defaultExtractor,
|
|
13
18
|
normalizeUrlFn: options.normalizeUrlFn ?? require_utils.normalizeUrl
|
|
14
19
|
}, {
|
|
@@ -2,6 +2,6 @@ import { DiscoverInput, DiscoverOptions, DiscoverResult } from "../common/types.
|
|
|
2
2
|
import { BlogrollResult } from "./types.cjs";
|
|
3
3
|
|
|
4
4
|
//#region src/blogrolls/index.d.ts
|
|
5
|
-
declare const discoverBlogrolls: <TValid extends BlogrollResult = BlogrollResult>(input: DiscoverInput, options
|
|
5
|
+
declare const discoverBlogrolls: <TValid extends BlogrollResult = BlogrollResult>(input: DiscoverInput, options?: DiscoverOptions<TValid>) => Promise<Array<DiscoverResult<TValid>>>;
|
|
6
6
|
//#endregion
|
|
7
7
|
export { discoverBlogrolls };
|
|
@@ -2,6 +2,6 @@ import { DiscoverInput, DiscoverOptions, DiscoverResult } from "../common/types.
|
|
|
2
2
|
import { BlogrollResult } from "./types.js";
|
|
3
3
|
|
|
4
4
|
//#region src/blogrolls/index.d.ts
|
|
5
|
-
declare const discoverBlogrolls: <TValid extends BlogrollResult = BlogrollResult>(input: DiscoverInput, options
|
|
5
|
+
declare const discoverBlogrolls: <TValid extends BlogrollResult = BlogrollResult>(input: DiscoverInput, options?: DiscoverOptions<TValid>) => Promise<Array<DiscoverResult<TValid>>>;
|
|
6
6
|
//#endregion
|
|
7
7
|
export { discoverBlogrolls };
|
package/dist/blogrolls/index.js
CHANGED
|
@@ -1,14 +1,19 @@
|
|
|
1
|
-
import { createNativeFetchAdapter } from "../common/discover/adapters.js";
|
|
2
1
|
import { normalizeUrl } from "../common/utils.js";
|
|
2
|
+
import { defaultFetchFn } from "../common/discover/utils.js";
|
|
3
3
|
import { discover } from "../common/discover/index.js";
|
|
4
4
|
import { defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions } from "./defaults.js";
|
|
5
5
|
import { defaultExtractor } from "./extractors.js";
|
|
6
6
|
|
|
7
7
|
//#region src/blogrolls/index.ts
|
|
8
|
-
const discoverBlogrolls = async (input, options) => {
|
|
8
|
+
const discoverBlogrolls = async (input, options = {}) => {
|
|
9
9
|
return discover(input, {
|
|
10
10
|
...options,
|
|
11
|
-
|
|
11
|
+
methods: options.methods ?? [
|
|
12
|
+
"html",
|
|
13
|
+
"headers",
|
|
14
|
+
"guess"
|
|
15
|
+
],
|
|
16
|
+
fetchFn: options.fetchFn ?? defaultFetchFn,
|
|
12
17
|
extractFn: options.extractFn ?? defaultExtractor,
|
|
13
18
|
normalizeUrlFn: options.normalizeUrlFn ?? normalizeUrl
|
|
14
19
|
}, {
|
package/dist/blogrolls.cjs
CHANGED
|
@@ -1,6 +1,19 @@
|
|
|
1
1
|
const require_locales = require('../locales.cjs');
|
|
2
2
|
|
|
3
3
|
//#region src/common/discover/utils.ts
|
|
4
|
+
const defaultFetchFn = async (url, options) => {
|
|
5
|
+
const response = await fetch(url, {
|
|
6
|
+
method: options?.method ?? "GET",
|
|
7
|
+
headers: options?.headers
|
|
8
|
+
});
|
|
9
|
+
return {
|
|
10
|
+
headers: response.headers,
|
|
11
|
+
body: await response.text(),
|
|
12
|
+
url: response.url,
|
|
13
|
+
status: response.status,
|
|
14
|
+
statusText: response.statusText
|
|
15
|
+
};
|
|
16
|
+
};
|
|
4
17
|
const normalizeInput = async (input, fetchFn) => {
|
|
5
18
|
if (typeof input === "object") return input;
|
|
6
19
|
const response = await fetchFn(input);
|
|
@@ -62,5 +75,6 @@ const normalizeMethodsConfig = (input, methods, defaults) => {
|
|
|
62
75
|
};
|
|
63
76
|
|
|
64
77
|
//#endregion
|
|
78
|
+
exports.defaultFetchFn = defaultFetchFn;
|
|
65
79
|
exports.normalizeInput = normalizeInput;
|
|
66
80
|
exports.normalizeMethodsConfig = normalizeMethodsConfig;
|
|
@@ -1,6 +1,19 @@
|
|
|
1
1
|
import { errors } from "../locales.js";
|
|
2
2
|
|
|
3
3
|
//#region src/common/discover/utils.ts
|
|
4
|
+
const defaultFetchFn = async (url, options) => {
|
|
5
|
+
const response = await fetch(url, {
|
|
6
|
+
method: options?.method ?? "GET",
|
|
7
|
+
headers: options?.headers
|
|
8
|
+
});
|
|
9
|
+
return {
|
|
10
|
+
headers: response.headers,
|
|
11
|
+
body: await response.text(),
|
|
12
|
+
url: response.url,
|
|
13
|
+
status: response.status,
|
|
14
|
+
statusText: response.statusText
|
|
15
|
+
};
|
|
16
|
+
};
|
|
4
17
|
const normalizeInput = async (input, fetchFn) => {
|
|
5
18
|
if (typeof input === "object") return input;
|
|
6
19
|
const response = await fetchFn(input);
|
|
@@ -62,4 +75,4 @@ const normalizeMethodsConfig = (input, methods, defaults) => {
|
|
|
62
75
|
};
|
|
63
76
|
|
|
64
77
|
//#endregion
|
|
65
|
-
export { normalizeInput, normalizeMethodsConfig };
|
|
78
|
+
export { defaultFetchFn, normalizeInput, normalizeMethodsConfig };
|
package/dist/common/types.d.cts
CHANGED
|
@@ -27,7 +27,7 @@ type DiscoverProgress = {
|
|
|
27
27
|
found: number;
|
|
28
28
|
current: string;
|
|
29
29
|
};
|
|
30
|
-
type
|
|
30
|
+
type DiscoverOnProgressFn = (progress: DiscoverProgress) => void;
|
|
31
31
|
type DiscoverResult<TValid = object> = ({
|
|
32
32
|
url: string;
|
|
33
33
|
isValid: true;
|
|
@@ -54,14 +54,14 @@ type DiscoverMethodsConfig = Array<'platform' | 'html' | 'headers' | 'guess'> |
|
|
|
54
54
|
guess?: true | Partial<Omit<GuessMethodOptions, 'baseUrl'>>;
|
|
55
55
|
};
|
|
56
56
|
type DiscoverOptions<TValid> = {
|
|
57
|
-
methods
|
|
57
|
+
methods?: DiscoverMethodsConfig;
|
|
58
58
|
fetchFn?: DiscoverFetchFn;
|
|
59
59
|
extractFn?: DiscoverExtractFn<TValid>;
|
|
60
60
|
normalizeUrlFn?: DiscoverNormalizeUrlFn;
|
|
61
61
|
concurrency?: number;
|
|
62
62
|
stopOnFirstResult?: boolean;
|
|
63
63
|
includeInvalid?: boolean;
|
|
64
|
-
onProgress?:
|
|
64
|
+
onProgress?: DiscoverOnProgressFn;
|
|
65
65
|
};
|
|
66
66
|
//#endregion
|
|
67
|
-
export { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethodsConfig, DiscoverNormalizeUrlFn, DiscoverOptions, DiscoverProgress,
|
|
67
|
+
export { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethodsConfig, DiscoverNormalizeUrlFn, DiscoverOnProgressFn, DiscoverOptions, DiscoverProgress, DiscoverResult, LinkSelector };
|
package/dist/common/types.d.ts
CHANGED
|
@@ -27,7 +27,7 @@ type DiscoverProgress = {
|
|
|
27
27
|
found: number;
|
|
28
28
|
current: string;
|
|
29
29
|
};
|
|
30
|
-
type
|
|
30
|
+
type DiscoverOnProgressFn = (progress: DiscoverProgress) => void;
|
|
31
31
|
type DiscoverResult<TValid = object> = ({
|
|
32
32
|
url: string;
|
|
33
33
|
isValid: true;
|
|
@@ -54,14 +54,14 @@ type DiscoverMethodsConfig = Array<'platform' | 'html' | 'headers' | 'guess'> |
|
|
|
54
54
|
guess?: true | Partial<Omit<GuessMethodOptions, 'baseUrl'>>;
|
|
55
55
|
};
|
|
56
56
|
type DiscoverOptions<TValid> = {
|
|
57
|
-
methods
|
|
57
|
+
methods?: DiscoverMethodsConfig;
|
|
58
58
|
fetchFn?: DiscoverFetchFn;
|
|
59
59
|
extractFn?: DiscoverExtractFn<TValid>;
|
|
60
60
|
normalizeUrlFn?: DiscoverNormalizeUrlFn;
|
|
61
61
|
concurrency?: number;
|
|
62
62
|
stopOnFirstResult?: boolean;
|
|
63
63
|
includeInvalid?: boolean;
|
|
64
|
-
onProgress?:
|
|
64
|
+
onProgress?: DiscoverOnProgressFn;
|
|
65
65
|
};
|
|
66
66
|
//#endregion
|
|
67
|
-
export { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethodsConfig, DiscoverNormalizeUrlFn, DiscoverOptions, DiscoverProgress,
|
|
67
|
+
export { DiscoverExtractFn, DiscoverFetchFn, DiscoverFetchFnOptions, DiscoverFetchFnResponse, DiscoverInput, DiscoverInputObject, DiscoverMethodsConfig, DiscoverNormalizeUrlFn, DiscoverOnProgressFn, DiscoverOptions, DiscoverProgress, DiscoverResult, LinkSelector };
|
|
@@ -1,15 +1,18 @@
|
|
|
1
1
|
const require_utils = require('../../utils.cjs');
|
|
2
2
|
|
|
3
3
|
//#region src/common/uris/headers/index.ts
|
|
4
|
+
const urlRegex = /<([^<>]+)>/;
|
|
5
|
+
const relRegex = /rel\s*=\s*["']?([^"';,]+)["']?/i;
|
|
6
|
+
const typeRegex = /type\s*=\s*["']?([^"';,]+)["']?/i;
|
|
4
7
|
const discoverUrisFromHeaders = (headers, options) => {
|
|
5
8
|
const uris = /* @__PURE__ */ new Set();
|
|
6
9
|
const linkHeader = headers.get("link");
|
|
7
10
|
if (!linkHeader) return [];
|
|
8
11
|
const links = linkHeader.split(/,(?=\s*<)/);
|
|
9
12
|
for (const link of links) {
|
|
10
|
-
const urlMatch = link.match(
|
|
11
|
-
const relMatch = link.match(
|
|
12
|
-
const typeMatch = link.match(
|
|
13
|
+
const urlMatch = link.match(urlRegex);
|
|
14
|
+
const relMatch = link.match(relRegex);
|
|
15
|
+
const typeMatch = link.match(typeRegex);
|
|
13
16
|
if (!urlMatch) continue;
|
|
14
17
|
const url = urlMatch[1];
|
|
15
18
|
const rel = relMatch?.[1]?.toLowerCase();
|
|
@@ -1,15 +1,18 @@
|
|
|
1
1
|
import { matchesAnyOfLinkSelectors } from "../../utils.js";
|
|
2
2
|
|
|
3
3
|
//#region src/common/uris/headers/index.ts
|
|
4
|
+
const urlRegex = /<([^<>]+)>/;
|
|
5
|
+
const relRegex = /rel\s*=\s*["']?([^"';,]+)["']?/i;
|
|
6
|
+
const typeRegex = /type\s*=\s*["']?([^"';,]+)["']?/i;
|
|
4
7
|
const discoverUrisFromHeaders = (headers, options) => {
|
|
5
8
|
const uris = /* @__PURE__ */ new Set();
|
|
6
9
|
const linkHeader = headers.get("link");
|
|
7
10
|
if (!linkHeader) return [];
|
|
8
11
|
const links = linkHeader.split(/,(?=\s*<)/);
|
|
9
12
|
for (const link of links) {
|
|
10
|
-
const urlMatch = link.match(
|
|
11
|
-
const relMatch = link.match(
|
|
12
|
-
const typeMatch = link.match(
|
|
13
|
+
const urlMatch = link.match(urlRegex);
|
|
14
|
+
const relMatch = link.match(relRegex);
|
|
15
|
+
const typeMatch = link.match(typeRegex);
|
|
13
16
|
if (!urlMatch) continue;
|
|
14
17
|
const url = urlMatch[1];
|
|
15
18
|
const rel = relMatch?.[1]?.toLowerCase();
|
package/dist/common/utils.cjs
CHANGED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
//#region src/common/utils.d.ts
|
|
2
|
+
declare const isSubdomainOf: (url: string, domain: string) => boolean;
|
|
3
|
+
declare const isHostOf: (url: string, hosts: Array<string>) => boolean;
|
|
4
|
+
declare const includesAnyOf: (value: string, patterns: Array<string>, parser?: (value: string) => string) => boolean;
|
|
5
|
+
declare const isAnyOf: (value: string, patterns: Array<string>, parser?: (value: string) => string) => boolean;
|
|
6
|
+
declare const anyWordMatchesAnyOf: (value: string, patterns: Array<string>) => boolean;
|
|
7
|
+
declare const endsWithAnyOf: (value: string, patterns: Array<string>) => boolean;
|
|
8
|
+
//#endregion
|
|
9
|
+
export { anyWordMatchesAnyOf, endsWithAnyOf, includesAnyOf, isAnyOf, isHostOf, isSubdomainOf };
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
//#region src/common/utils.d.ts
|
|
2
|
+
declare const isSubdomainOf: (url: string, domain: string) => boolean;
|
|
3
|
+
declare const isHostOf: (url: string, hosts: Array<string>) => boolean;
|
|
4
|
+
declare const includesAnyOf: (value: string, patterns: Array<string>, parser?: (value: string) => string) => boolean;
|
|
5
|
+
declare const isAnyOf: (value: string, patterns: Array<string>, parser?: (value: string) => string) => boolean;
|
|
6
|
+
declare const anyWordMatchesAnyOf: (value: string, patterns: Array<string>) => boolean;
|
|
7
|
+
declare const endsWithAnyOf: (value: string, patterns: Array<string>) => boolean;
|
|
8
|
+
//#endregion
|
|
9
|
+
export { anyWordMatchesAnyOf, endsWithAnyOf, includesAnyOf, isAnyOf, isHostOf, isSubdomainOf };
|
package/dist/common/utils.js
CHANGED
|
@@ -55,4 +55,4 @@ const processConcurrently = async (items, processFn, options) => {
|
|
|
55
55
|
};
|
|
56
56
|
|
|
57
57
|
//#endregion
|
|
58
|
-
export { endsWithAnyOf, includesAnyOf, isAnyOf, isHostOf, isSubdomainOf, matchesAnyOfLinkSelectors, normalizeUrl, processConcurrently };
|
|
58
|
+
export { anyWordMatchesAnyOf, endsWithAnyOf, includesAnyOf, isAnyOf, isHostOf, isSubdomainOf, matchesAnyOfLinkSelectors, normalizeUrl, processConcurrently };
|
package/dist/feeds/defaults.cjs
CHANGED
|
@@ -1,8 +1,17 @@
|
|
|
1
|
+
const require_behance = require('./platform/handlers/behance.cjs');
|
|
1
2
|
const require_blogspot = require('./platform/handlers/blogspot.cjs');
|
|
2
3
|
const require_bluesky = require('./platform/handlers/bluesky.cjs');
|
|
4
|
+
const require_dailymotion = require('./platform/handlers/dailymotion.cjs');
|
|
5
|
+
const require_deviantart = require('./platform/handlers/deviantart.cjs');
|
|
6
|
+
const require_devto = require('./platform/handlers/devto.cjs');
|
|
3
7
|
const require_github = require('./platform/handlers/github.cjs');
|
|
8
|
+
const require_githubGist = require('./platform/handlers/githubGist.cjs');
|
|
4
9
|
const require_gitlab = require('./platform/handlers/gitlab.cjs');
|
|
5
10
|
const require_kickstarter = require('./platform/handlers/kickstarter.cjs');
|
|
11
|
+
const require_lobsters = require('./platform/handlers/lobsters.cjs');
|
|
12
|
+
const require_medium = require('./platform/handlers/medium.cjs');
|
|
13
|
+
const require_pinterest = require('./platform/handlers/pinterest.cjs');
|
|
14
|
+
const require_producthunt = require('./platform/handlers/producthunt.cjs');
|
|
6
15
|
const require_reddit = require('./platform/handlers/reddit.cjs');
|
|
7
16
|
const require_soundcloud = require('./platform/handlers/soundcloud.cjs');
|
|
8
17
|
const require_substack = require('./platform/handlers/substack.cjs');
|
|
@@ -44,26 +53,38 @@ const urisBalanced = [
|
|
|
44
53
|
const urisComprehensive = [
|
|
45
54
|
...urisBalanced,
|
|
46
55
|
"/atom",
|
|
56
|
+
"/rss/",
|
|
57
|
+
"/rss2.xml",
|
|
47
58
|
"/feed.rss",
|
|
48
59
|
"/feed.atom",
|
|
49
60
|
"/feed.rss.xml",
|
|
50
61
|
"/feed.atom.xml",
|
|
62
|
+
"/feed/atom/",
|
|
63
|
+
"/feed/rss/",
|
|
64
|
+
"/feed/rss2/",
|
|
65
|
+
"/feed/rdf",
|
|
66
|
+
"/feed/rdf/",
|
|
51
67
|
"/index.rss.xml",
|
|
52
68
|
"/index.atom.xml",
|
|
53
|
-
"
|
|
54
|
-
"
|
|
55
|
-
"
|
|
56
|
-
"
|
|
57
|
-
"
|
|
58
|
-
"
|
|
59
|
-
"
|
|
69
|
+
"?feed=rss",
|
|
70
|
+
"?feed=rss2",
|
|
71
|
+
"?feed=atom",
|
|
72
|
+
"?feed=rdf",
|
|
73
|
+
"?feed=comments-rss2",
|
|
74
|
+
"?feed=comments-atom",
|
|
75
|
+
"?format=rss",
|
|
76
|
+
"?format=atom",
|
|
77
|
+
"?rss=1",
|
|
78
|
+
"?atom=1",
|
|
60
79
|
"/.rss",
|
|
61
80
|
"/f.json",
|
|
62
81
|
"/f.rss",
|
|
63
82
|
"/json",
|
|
64
83
|
"/.feed",
|
|
65
84
|
"/comments/feed",
|
|
66
|
-
"/feeds/posts/default"
|
|
85
|
+
"/feeds/posts/default",
|
|
86
|
+
"/feeds/posts/default?alt=rss",
|
|
87
|
+
"/feeds/comments/default"
|
|
67
88
|
];
|
|
68
89
|
const ignoredUris = ["wp-json/oembed/", "wp-json/wp/"];
|
|
69
90
|
const anchorLabels = [
|
|
@@ -72,6 +93,7 @@ const anchorLabels = [
|
|
|
72
93
|
"atom",
|
|
73
94
|
"subscribe",
|
|
74
95
|
"syndicate",
|
|
96
|
+
"syndication",
|
|
75
97
|
"json feed"
|
|
76
98
|
];
|
|
77
99
|
const linkSelectors = [{
|
|
@@ -87,11 +109,20 @@ const defaultHtmlOptions = {
|
|
|
87
109
|
const defaultHeadersOptions = { linkSelectors };
|
|
88
110
|
const defaultGuessOptions = { uris: urisBalanced };
|
|
89
111
|
const defaultPlatformOptions = { handlers: [
|
|
112
|
+
require_behance.behanceHandler,
|
|
90
113
|
require_blogspot.blogspotHandler,
|
|
91
114
|
require_bluesky.blueskyHandler,
|
|
115
|
+
require_deviantart.deviantartHandler,
|
|
116
|
+
require_dailymotion.dailymotionHandler,
|
|
117
|
+
require_devto.devtoHandler,
|
|
92
118
|
require_github.githubHandler,
|
|
119
|
+
require_githubGist.githubGistHandler,
|
|
93
120
|
require_gitlab.gitlabHandler,
|
|
94
121
|
require_kickstarter.kickstarterHandler,
|
|
122
|
+
require_medium.mediumHandler,
|
|
123
|
+
require_pinterest.pinterestHandler,
|
|
124
|
+
require_producthunt.producthuntHandler,
|
|
125
|
+
require_lobsters.lobstersHandler,
|
|
95
126
|
require_reddit.redditHandler,
|
|
96
127
|
require_soundcloud.soundcloudHandler,
|
|
97
128
|
require_substack.substackHandler,
|
package/dist/feeds/defaults.js
CHANGED
|
@@ -1,8 +1,17 @@
|
|
|
1
|
+
import { behanceHandler } from "./platform/handlers/behance.js";
|
|
1
2
|
import { blogspotHandler } from "./platform/handlers/blogspot.js";
|
|
2
3
|
import { blueskyHandler } from "./platform/handlers/bluesky.js";
|
|
4
|
+
import { dailymotionHandler } from "./platform/handlers/dailymotion.js";
|
|
5
|
+
import { deviantartHandler } from "./platform/handlers/deviantart.js";
|
|
6
|
+
import { devtoHandler } from "./platform/handlers/devto.js";
|
|
3
7
|
import { githubHandler } from "./platform/handlers/github.js";
|
|
8
|
+
import { githubGistHandler } from "./platform/handlers/githubGist.js";
|
|
4
9
|
import { gitlabHandler } from "./platform/handlers/gitlab.js";
|
|
5
10
|
import { kickstarterHandler } from "./platform/handlers/kickstarter.js";
|
|
11
|
+
import { lobstersHandler } from "./platform/handlers/lobsters.js";
|
|
12
|
+
import { mediumHandler } from "./platform/handlers/medium.js";
|
|
13
|
+
import { pinterestHandler } from "./platform/handlers/pinterest.js";
|
|
14
|
+
import { producthuntHandler } from "./platform/handlers/producthunt.js";
|
|
6
15
|
import { redditHandler } from "./platform/handlers/reddit.js";
|
|
7
16
|
import { soundcloudHandler } from "./platform/handlers/soundcloud.js";
|
|
8
17
|
import { substackHandler } from "./platform/handlers/substack.js";
|
|
@@ -44,26 +53,38 @@ const urisBalanced = [
|
|
|
44
53
|
const urisComprehensive = [
|
|
45
54
|
...urisBalanced,
|
|
46
55
|
"/atom",
|
|
56
|
+
"/rss/",
|
|
57
|
+
"/rss2.xml",
|
|
47
58
|
"/feed.rss",
|
|
48
59
|
"/feed.atom",
|
|
49
60
|
"/feed.rss.xml",
|
|
50
61
|
"/feed.atom.xml",
|
|
62
|
+
"/feed/atom/",
|
|
63
|
+
"/feed/rss/",
|
|
64
|
+
"/feed/rss2/",
|
|
65
|
+
"/feed/rdf",
|
|
66
|
+
"/feed/rdf/",
|
|
51
67
|
"/index.rss.xml",
|
|
52
68
|
"/index.atom.xml",
|
|
53
|
-
"
|
|
54
|
-
"
|
|
55
|
-
"
|
|
56
|
-
"
|
|
57
|
-
"
|
|
58
|
-
"
|
|
59
|
-
"
|
|
69
|
+
"?feed=rss",
|
|
70
|
+
"?feed=rss2",
|
|
71
|
+
"?feed=atom",
|
|
72
|
+
"?feed=rdf",
|
|
73
|
+
"?feed=comments-rss2",
|
|
74
|
+
"?feed=comments-atom",
|
|
75
|
+
"?format=rss",
|
|
76
|
+
"?format=atom",
|
|
77
|
+
"?rss=1",
|
|
78
|
+
"?atom=1",
|
|
60
79
|
"/.rss",
|
|
61
80
|
"/f.json",
|
|
62
81
|
"/f.rss",
|
|
63
82
|
"/json",
|
|
64
83
|
"/.feed",
|
|
65
84
|
"/comments/feed",
|
|
66
|
-
"/feeds/posts/default"
|
|
85
|
+
"/feeds/posts/default",
|
|
86
|
+
"/feeds/posts/default?alt=rss",
|
|
87
|
+
"/feeds/comments/default"
|
|
67
88
|
];
|
|
68
89
|
const ignoredUris = ["wp-json/oembed/", "wp-json/wp/"];
|
|
69
90
|
const anchorLabels = [
|
|
@@ -72,6 +93,7 @@ const anchorLabels = [
|
|
|
72
93
|
"atom",
|
|
73
94
|
"subscribe",
|
|
74
95
|
"syndicate",
|
|
96
|
+
"syndication",
|
|
75
97
|
"json feed"
|
|
76
98
|
];
|
|
77
99
|
const linkSelectors = [{
|
|
@@ -87,11 +109,20 @@ const defaultHtmlOptions = {
|
|
|
87
109
|
const defaultHeadersOptions = { linkSelectors };
|
|
88
110
|
const defaultGuessOptions = { uris: urisBalanced };
|
|
89
111
|
const defaultPlatformOptions = { handlers: [
|
|
112
|
+
behanceHandler,
|
|
90
113
|
blogspotHandler,
|
|
91
114
|
blueskyHandler,
|
|
115
|
+
deviantartHandler,
|
|
116
|
+
dailymotionHandler,
|
|
117
|
+
devtoHandler,
|
|
92
118
|
githubHandler,
|
|
119
|
+
githubGistHandler,
|
|
93
120
|
gitlabHandler,
|
|
94
121
|
kickstarterHandler,
|
|
122
|
+
mediumHandler,
|
|
123
|
+
pinterestHandler,
|
|
124
|
+
producthuntHandler,
|
|
125
|
+
lobstersHandler,
|
|
95
126
|
redditHandler,
|
|
96
127
|
soundcloudHandler,
|
|
97
128
|
substackHandler,
|
package/dist/feeds/index.cjs
CHANGED
|
@@ -1,14 +1,20 @@
|
|
|
1
|
-
const require_adapters = require('../common/discover/adapters.cjs');
|
|
2
1
|
const require_utils = require('../common/utils.cjs');
|
|
2
|
+
const require_utils$1 = require('../common/discover/utils.cjs');
|
|
3
3
|
const require_index = require('../common/discover/index.cjs');
|
|
4
4
|
const require_defaults = require('./defaults.cjs');
|
|
5
5
|
const require_extractors = require('./extractors.cjs');
|
|
6
6
|
|
|
7
7
|
//#region src/feeds/index.ts
|
|
8
|
-
const discoverFeeds = async (input, options) => {
|
|
8
|
+
const discoverFeeds = async (input, options = {}) => {
|
|
9
9
|
return require_index.discover(input, {
|
|
10
10
|
...options,
|
|
11
|
-
|
|
11
|
+
methods: options.methods ?? [
|
|
12
|
+
"platform",
|
|
13
|
+
"html",
|
|
14
|
+
"headers",
|
|
15
|
+
"guess"
|
|
16
|
+
],
|
|
17
|
+
fetchFn: options.fetchFn ?? require_utils$1.defaultFetchFn,
|
|
12
18
|
extractFn: options.extractFn ?? require_extractors.defaultExtractor,
|
|
13
19
|
normalizeUrlFn: options.normalizeUrlFn ?? require_utils.normalizeUrl
|
|
14
20
|
}, {
|
package/dist/feeds/index.d.cts
CHANGED
|
@@ -2,6 +2,6 @@ import { DiscoverInput, DiscoverOptions, DiscoverResult } from "../common/types.
|
|
|
2
2
|
import { FeedResult } from "./types.cjs";
|
|
3
3
|
|
|
4
4
|
//#region src/feeds/index.d.ts
|
|
5
|
-
declare const discoverFeeds: <TValid extends FeedResult = FeedResult>(input: DiscoverInput, options
|
|
5
|
+
declare const discoverFeeds: <TValid extends FeedResult = FeedResult>(input: DiscoverInput, options?: DiscoverOptions<TValid>) => Promise<Array<DiscoverResult<TValid>>>;
|
|
6
6
|
//#endregion
|
|
7
7
|
export { discoverFeeds };
|
package/dist/feeds/index.d.ts
CHANGED
|
@@ -2,6 +2,6 @@ import { DiscoverInput, DiscoverOptions, DiscoverResult } from "../common/types.
|
|
|
2
2
|
import { FeedResult } from "./types.js";
|
|
3
3
|
|
|
4
4
|
//#region src/feeds/index.d.ts
|
|
5
|
-
declare const discoverFeeds: <TValid extends FeedResult = FeedResult>(input: DiscoverInput, options
|
|
5
|
+
declare const discoverFeeds: <TValid extends FeedResult = FeedResult>(input: DiscoverInput, options?: DiscoverOptions<TValid>) => Promise<Array<DiscoverResult<TValid>>>;
|
|
6
6
|
//#endregion
|
|
7
7
|
export { discoverFeeds };
|
package/dist/feeds/index.js
CHANGED
|
@@ -1,14 +1,20 @@
|
|
|
1
|
-
import { createNativeFetchAdapter } from "../common/discover/adapters.js";
|
|
2
1
|
import { normalizeUrl } from "../common/utils.js";
|
|
2
|
+
import { defaultFetchFn } from "../common/discover/utils.js";
|
|
3
3
|
import { discover } from "../common/discover/index.js";
|
|
4
4
|
import { defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions } from "./defaults.js";
|
|
5
5
|
import { defaultExtractor } from "./extractors.js";
|
|
6
6
|
|
|
7
7
|
//#region src/feeds/index.ts
|
|
8
|
-
const discoverFeeds = async (input, options) => {
|
|
8
|
+
const discoverFeeds = async (input, options = {}) => {
|
|
9
9
|
return discover(input, {
|
|
10
10
|
...options,
|
|
11
|
-
|
|
11
|
+
methods: options.methods ?? [
|
|
12
|
+
"platform",
|
|
13
|
+
"html",
|
|
14
|
+
"headers",
|
|
15
|
+
"guess"
|
|
16
|
+
],
|
|
17
|
+
fetchFn: options.fetchFn ?? defaultFetchFn,
|
|
12
18
|
extractFn: options.extractFn ?? defaultExtractor,
|
|
13
19
|
normalizeUrlFn: options.normalizeUrlFn ?? normalizeUrl
|
|
14
20
|
}, {
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
const require_utils = require('../../../common/utils.cjs');
|
|
2
|
+
|
|
3
|
+
//#region src/feeds/platform/handlers/behance.ts
|
|
4
|
+
const hosts = ["behance.net", "www.behance.net"];
|
|
5
|
+
const userPathRegex = /^\/([a-zA-Z0-9_-]+)(?:\/(appreciated))?\/?$/;
|
|
6
|
+
const excludedPaths = [
|
|
7
|
+
"search",
|
|
8
|
+
"galleries",
|
|
9
|
+
"curated",
|
|
10
|
+
"features",
|
|
11
|
+
"live",
|
|
12
|
+
"joblist",
|
|
13
|
+
"hire",
|
|
14
|
+
"blog",
|
|
15
|
+
"about",
|
|
16
|
+
"privacy",
|
|
17
|
+
"tos",
|
|
18
|
+
"help",
|
|
19
|
+
"onboarding",
|
|
20
|
+
"settings",
|
|
21
|
+
"notifications",
|
|
22
|
+
"messages",
|
|
23
|
+
"adobe"
|
|
24
|
+
];
|
|
25
|
+
const behanceHandler = {
|
|
26
|
+
match: (url) => {
|
|
27
|
+
return require_utils.isHostOf(url, hosts);
|
|
28
|
+
},
|
|
29
|
+
resolve: (url) => {
|
|
30
|
+
const { pathname } = new URL(url);
|
|
31
|
+
const userMatch = pathname.match(userPathRegex);
|
|
32
|
+
if (userMatch?.[1]) {
|
|
33
|
+
const username = userMatch[1];
|
|
34
|
+
const subpage = userMatch[2];
|
|
35
|
+
if (!require_utils.isAnyOf(username, excludedPaths)) {
|
|
36
|
+
if (subpage === "appreciated") return [`https://www.behance.net/feeds/user?username=${username}&content=appreciated`];
|
|
37
|
+
return [`https://www.behance.net/feeds/user?username=${username}`];
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
return [];
|
|
41
|
+
}
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
//#endregion
|
|
45
|
+
exports.behanceHandler = behanceHandler;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { isAnyOf, isHostOf } from "../../../common/utils.js";
|
|
2
|
+
|
|
3
|
+
//#region src/feeds/platform/handlers/behance.ts
|
|
4
|
+
const hosts = ["behance.net", "www.behance.net"];
|
|
5
|
+
const userPathRegex = /^\/([a-zA-Z0-9_-]+)(?:\/(appreciated))?\/?$/;
|
|
6
|
+
const excludedPaths = [
|
|
7
|
+
"search",
|
|
8
|
+
"galleries",
|
|
9
|
+
"curated",
|
|
10
|
+
"features",
|
|
11
|
+
"live",
|
|
12
|
+
"joblist",
|
|
13
|
+
"hire",
|
|
14
|
+
"blog",
|
|
15
|
+
"about",
|
|
16
|
+
"privacy",
|
|
17
|
+
"tos",
|
|
18
|
+
"help",
|
|
19
|
+
"onboarding",
|
|
20
|
+
"settings",
|
|
21
|
+
"notifications",
|
|
22
|
+
"messages",
|
|
23
|
+
"adobe"
|
|
24
|
+
];
|
|
25
|
+
const behanceHandler = {
|
|
26
|
+
match: (url) => {
|
|
27
|
+
return isHostOf(url, hosts);
|
|
28
|
+
},
|
|
29
|
+
resolve: (url) => {
|
|
30
|
+
const { pathname } = new URL(url);
|
|
31
|
+
const userMatch = pathname.match(userPathRegex);
|
|
32
|
+
if (userMatch?.[1]) {
|
|
33
|
+
const username = userMatch[1];
|
|
34
|
+
const subpage = userMatch[2];
|
|
35
|
+
if (!isAnyOf(username, excludedPaths)) {
|
|
36
|
+
if (subpage === "appreciated") return [`https://www.behance.net/feeds/user?username=${username}&content=appreciated`];
|
|
37
|
+
return [`https://www.behance.net/feeds/user?username=${username}`];
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
return [];
|
|
41
|
+
}
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
//#endregion
|
|
45
|
+
export { behanceHandler };
|