feedscout 1.0.0 → 1.2.0

package/dist/feeds/defaults.js

@@ -1,3 +1,24 @@
+import { behanceHandler } from "./platform/handlers/behance.js";
+import { blogspotHandler } from "./platform/handlers/blogspot.js";
+import { blueskyHandler } from "./platform/handlers/bluesky.js";
+import { dailymotionHandler } from "./platform/handlers/dailymotion.js";
+import { deviantartHandler } from "./platform/handlers/deviantart.js";
+import { devtoHandler } from "./platform/handlers/devto.js";
+import { githubHandler } from "./platform/handlers/github.js";
+import { githubGistHandler } from "./platform/handlers/githubGist.js";
+import { gitlabHandler } from "./platform/handlers/gitlab.js";
+import { kickstarterHandler } from "./platform/handlers/kickstarter.js";
+import { lobstersHandler } from "./platform/handlers/lobsters.js";
+import { mediumHandler } from "./platform/handlers/medium.js";
+import { pinterestHandler } from "./platform/handlers/pinterest.js";
+import { producthuntHandler } from "./platform/handlers/producthunt.js";
+import { redditHandler } from "./platform/handlers/reddit.js";
+import { soundcloudHandler } from "./platform/handlers/soundcloud.js";
+import { substackHandler } from "./platform/handlers/substack.js";
+import { tumblrHandler } from "./platform/handlers/tumblr.js";
+import { wordpressHandler } from "./platform/handlers/wordpress.js";
+import { youtubeHandler } from "./platform/handlers/youtube.js";
+
 //#region src/feeds/defaults.ts
 const mimeTypes = [
 	"application/rss+xml",
@@ -74,6 +95,28 @@ const defaultHtmlOptions = {
 };
 const defaultHeadersOptions = { linkSelectors };
 const defaultGuessOptions = { uris: urisBalanced };
+const defaultPlatformOptions = { handlers: [
+	behanceHandler,
+	blogspotHandler,
+	blueskyHandler,
+	deviantartHandler,
+	dailymotionHandler,
+	devtoHandler,
+	githubHandler,
+	githubGistHandler,
+	gitlabHandler,
+	kickstarterHandler,
+	mediumHandler,
+	pinterestHandler,
+	producthuntHandler,
+	lobstersHandler,
+	redditHandler,
+	soundcloudHandler,
+	substackHandler,
+	tumblrHandler,
+	wordpressHandler,
+	youtubeHandler
+] };
 
 //#endregion
-export { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
+export { anchorLabels, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };

package/dist/feeds/index.cjs

@@ -1,17 +1,24 @@
-const require_adapters = require('../common/discover/adapters.cjs');
 const require_utils = require('../common/utils.cjs');
+const require_utils$1 = require('../common/discover/utils.cjs');
 const require_index = require('../common/discover/index.cjs');
 const require_defaults = require('./defaults.cjs');
 const require_extractors = require('./extractors.cjs');
 
 //#region src/feeds/index.ts
-const discoverFeeds = async (input, options) => {
+const discoverFeeds = async (input, options = {}) => {
 	return require_index.discover(input, {
 		...options,
-		fetchFn: options.fetchFn ?? require_adapters.createNativeFetchAdapter(),
+		methods: options.methods ?? [
+			"platform",
+			"html",
+			"headers",
+			"guess"
+		],
+		fetchFn: options.fetchFn ?? require_utils$1.defaultFetchFn,
 		extractFn: options.extractFn ?? require_extractors.defaultExtractor,
 		normalizeUrlFn: options.normalizeUrlFn ?? require_utils.normalizeUrl
 	}, {
+		platform: require_defaults.defaultPlatformOptions,
 		html: require_defaults.defaultHtmlOptions,
 		headers: require_defaults.defaultHeadersOptions,
 		guess: require_defaults.defaultGuessOptions

package/dist/feeds/index.d.cts

@@ -2,6 +2,6 @@ import { DiscoverInput, DiscoverOptions, DiscoverResult } from "../common/types.
 import { FeedResult } from "./types.cjs";
 
 //#region src/feeds/index.d.ts
-declare const discoverFeeds: <TValid extends FeedResult = FeedResult>(input: DiscoverInput, options: DiscoverOptions<TValid>) => Promise<Array<DiscoverResult<TValid>>>;
+declare const discoverFeeds: <TValid extends FeedResult = FeedResult>(input: DiscoverInput, options?: DiscoverOptions<TValid>) => Promise<Array<DiscoverResult<TValid>>>;
 //#endregion
 export { discoverFeeds };

package/dist/feeds/index.d.ts

@@ -2,6 +2,6 @@ import { DiscoverInput, DiscoverOptions, DiscoverResult } from "../common/types.
 import { FeedResult } from "./types.js";
 
 //#region src/feeds/index.d.ts
-declare const discoverFeeds: <TValid extends FeedResult = FeedResult>(input: DiscoverInput, options: DiscoverOptions<TValid>) => Promise<Array<DiscoverResult<TValid>>>;
+declare const discoverFeeds: <TValid extends FeedResult = FeedResult>(input: DiscoverInput, options?: DiscoverOptions<TValid>) => Promise<Array<DiscoverResult<TValid>>>;
 //#endregion
 export { discoverFeeds };

package/dist/feeds/index.js

@@ -1,17 +1,24 @@
-import { createNativeFetchAdapter } from "../common/discover/adapters.js";
 import { normalizeUrl } from "../common/utils.js";
+import { defaultFetchFn } from "../common/discover/utils.js";
 import { discover } from "../common/discover/index.js";
-import { defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions } from "./defaults.js";
+import { defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions } from "./defaults.js";
 import { defaultExtractor } from "./extractors.js";
 
 //#region src/feeds/index.ts
-const discoverFeeds = async (input, options) => {
+const discoverFeeds = async (input, options = {}) => {
 	return discover(input, {
 		...options,
-		fetchFn: options.fetchFn ?? createNativeFetchAdapter(),
+		methods: options.methods ?? [
+			"platform",
+			"html",
+			"headers",
+			"guess"
+		],
+		fetchFn: options.fetchFn ?? defaultFetchFn,
 		extractFn: options.extractFn ?? defaultExtractor,
 		normalizeUrlFn: options.normalizeUrlFn ?? normalizeUrl
 	}, {
+		platform: defaultPlatformOptions,
 		html: defaultHtmlOptions,
 		headers: defaultHeadersOptions,
 		guess: defaultGuessOptions

package/dist/feeds/platform/handlers/behance.cjs

@@ -0,0 +1,45 @@
+const require_utils = require('../../../common/utils.cjs');
+
+//#region src/feeds/platform/handlers/behance.ts
+const hosts = ["behance.net", "www.behance.net"];
+const userPathRegex = /^\/([a-zA-Z0-9_-]+)(?:\/(appreciated))?\/?$/;
+const excludedPaths = [
+	"search",
+	"galleries",
+	"curated",
+	"features",
+	"live",
+	"joblist",
+	"hire",
+	"blog",
+	"about",
+	"privacy",
+	"tos",
+	"help",
+	"onboarding",
+	"settings",
+	"notifications",
+	"messages",
+	"adobe"
+];
+const behanceHandler = {
+	match: (url) => {
+		return require_utils.isHostOf(url, hosts);
+	},
+	resolve: (url) => {
+		const { pathname } = new URL(url);
+		const userMatch = pathname.match(userPathRegex);
+		if (userMatch?.[1]) {
+			const username = userMatch[1];
+			const subpage = userMatch[2];
+			if (!require_utils.isAnyOf(username, excludedPaths)) {
+				if (subpage === "appreciated") return [`https://www.behance.net/feeds/user?username=${username}&content=appreciated`];
+				return [`https://www.behance.net/feeds/user?username=${username}`];
+			}
+		}
+		return [];
+	}
+};
+
+//#endregion
+exports.behanceHandler = behanceHandler;

package/dist/feeds/platform/handlers/behance.js

@@ -0,0 +1,45 @@
+import { isAnyOf, isHostOf } from "../../../common/utils.js";
+
+//#region src/feeds/platform/handlers/behance.ts
+const hosts = ["behance.net", "www.behance.net"];
+const userPathRegex = /^\/([a-zA-Z0-9_-]+)(?:\/(appreciated))?\/?$/;
+const excludedPaths = [
+	"search",
+	"galleries",
+	"curated",
+	"features",
+	"live",
+	"joblist",
+	"hire",
+	"blog",
+	"about",
+	"privacy",
+	"tos",
+	"help",
+	"onboarding",
+	"settings",
+	"notifications",
+	"messages",
+	"adobe"
+];
+const behanceHandler = {
+	match: (url) => {
+		return isHostOf(url, hosts);
+	},
+	resolve: (url) => {
+		const { pathname } = new URL(url);
+		const userMatch = pathname.match(userPathRegex);
+		if (userMatch?.[1]) {
+			const username = userMatch[1];
+			const subpage = userMatch[2];
+			if (!isAnyOf(username, excludedPaths)) {
+				if (subpage === "appreciated") return [`https://www.behance.net/feeds/user?username=${username}&content=appreciated`];
+				return [`https://www.behance.net/feeds/user?username=${username}`];
+			}
+		}
+		return [];
+	}
+};
+
+//#endregion
+export { behanceHandler };

package/dist/feeds/platform/handlers/blogspot.cjs

@@ -0,0 +1,24 @@
+
+//#region src/feeds/platform/handlers/blogspot.ts
+const blogspotDomainRegex = /^.+\.blogspot\.(?:com|co\.[a-z]{2}|com\.[a-z]{2}|[a-z]{2,3})$/;
+const blogspotHandler = {
+	match: (url) => {
+		const hostname = new URL(url).hostname.toLowerCase();
+		return blogspotDomainRegex.test(hostname);
+	},
+	resolve: (url) => {
+		const { origin, pathname } = new URL(url);
+		const uris = [];
+		const labelMatch = pathname.match(/^\/search\/label\/([^/]+)/);
+		if (labelMatch?.[1]) {
+			const label = labelMatch[1];
+			uris.push(`${origin}/feeds/posts/default/-/${label}`);
+		}
+		uris.push(`${origin}/feeds/posts/default`);
+		uris.push(`${origin}/feeds/posts/default?alt=rss`);
+		return uris;
+	}
+};
+
+//#endregion
+exports.blogspotHandler = blogspotHandler;

package/dist/feeds/platform/handlers/blogspot.js

@@ -0,0 +1,23 @@
+//#region src/feeds/platform/handlers/blogspot.ts
+const blogspotDomainRegex = /^.+\.blogspot\.(?:com|co\.[a-z]{2}|com\.[a-z]{2}|[a-z]{2,3})$/;
+const blogspotHandler = {
+	match: (url) => {
+		const hostname = new URL(url).hostname.toLowerCase();
+		return blogspotDomainRegex.test(hostname);
+	},
+	resolve: (url) => {
+		const { origin, pathname } = new URL(url);
+		const uris = [];
+		const labelMatch = pathname.match(/^\/search\/label\/([^/]+)/);
+		if (labelMatch?.[1]) {
+			const label = labelMatch[1];
+			uris.push(`${origin}/feeds/posts/default/-/${label}`);
+		}
+		uris.push(`${origin}/feeds/posts/default`);
+		uris.push(`${origin}/feeds/posts/default?alt=rss`);
+		return uris;
+	}
+};
+
+//#endregion
+export { blogspotHandler };

package/dist/feeds/platform/handlers/bluesky.cjs

@@ -0,0 +1,18 @@
+const require_utils = require('../../../common/utils.cjs');
+
+//#region src/feeds/platform/handlers/bluesky.ts
+const hosts = ["bsky.app"];
+const blueskyHandler = {
+	match: (url) => {
+		return require_utils.isHostOf(url, hosts);
+	},
+	resolve: (url) => {
+		const { pathname } = new URL(url);
+		const handle = pathname.match(/^\/profile\/([^/]+)/)?.[1];
+		if (!handle) return [];
+		return [`https://bsky.app/profile/${handle}/rss`];
+	}
+};
+
+//#endregion
+exports.blueskyHandler = blueskyHandler;

package/dist/feeds/platform/handlers/bluesky.js

@@ -0,0 +1,18 @@
+import { isHostOf } from "../../../common/utils.js";
+
+//#region src/feeds/platform/handlers/bluesky.ts
+const hosts = ["bsky.app"];
+const blueskyHandler = {
+	match: (url) => {
+		return isHostOf(url, hosts);
+	},
+	resolve: (url) => {
+		const { pathname } = new URL(url);
+		const handle = pathname.match(/^\/profile\/([^/]+)/)?.[1];
+		if (!handle) return [];
+		return [`https://bsky.app/profile/${handle}/rss`];
+	}
+};
+
+//#endregion
+export { blueskyHandler };

package/dist/feeds/platform/handlers/dailymotion.cjs

@@ -0,0 +1,66 @@
+const require_utils = require('../../../common/utils.cjs');
+
+//#region src/feeds/platform/handlers/dailymotion.ts
+const hosts = ["dailymotion.com", "www.dailymotion.com"];
+const userPathRegex = /^\/([a-zA-Z0-9_-]+)$/;
+const playlistPathRegex = /^\/playlist\/([a-zA-Z0-9_-]+)/;
+const excludedPaths = [
+	"signin",
+	"signout",
+	"signup",
+	"login",
+	"logout",
+	"register",
+	"search",
+	"legal",
+	"about",
+	"careers",
+	"terms",
+	"privacy",
+	"feedback",
+	"help",
+	"settings",
+	"upload",
+	"partner",
+	"monetize",
+	"studio",
+	"video",
+	"live",
+	"channels",
+	"playlist",
+	"topics",
+	"trending",
+	"dm",
+	"creator",
+	"premium",
+	"explore",
+	"following",
+	"subscriptions",
+	"notifications",
+	"history",
+	"watch",
+	"contact",
+	"ads",
+	"dmca",
+	"copyright",
+	"community"
+];
+const dailymotionHandler = {
+	match: (url) => {
+		return require_utils.isHostOf(url, hosts);
+	},
+	resolve: (url) => {
+		const { pathname } = new URL(url);
+		const playlistMatch = pathname.match(playlistPathRegex);
+		if (playlistMatch?.[1]) return [`https://www.dailymotion.com/rss/playlist/${playlistMatch[1]}`];
+		const userMatch = pathname.match(userPathRegex);
+		if (userMatch?.[1]) {
+			const username = userMatch[1];
+			if (!require_utils.isAnyOf(username, excludedPaths)) return [`https://www.dailymotion.com/rss/${username}`];
+		}
+		return [];
+	}
+};
+
+//#endregion
+exports.dailymotionHandler = dailymotionHandler;

package/dist/feeds/platform/handlers/dailymotion.js

@@ -0,0 +1,66 @@
+import { isAnyOf, isHostOf } from "../../../common/utils.js";
+
+//#region src/feeds/platform/handlers/dailymotion.ts
+const hosts = ["dailymotion.com", "www.dailymotion.com"];
+const userPathRegex = /^\/([a-zA-Z0-9_-]+)$/;
+const playlistPathRegex = /^\/playlist\/([a-zA-Z0-9_-]+)/;
+const excludedPaths = [
+	"signin",
+	"signout",
+	"signup",
+	"login",
+	"logout",
+	"register",
+	"search",
+	"legal",
+	"about",
+	"careers",
+	"terms",
+	"privacy",
+	"feedback",
+	"help",
+	"settings",
+	"upload",
+	"partner",
+	"monetize",
+	"studio",
+	"video",
+	"live",
+	"channels",
+	"playlist",
+	"topics",
+	"trending",
+	"dm",
+	"creator",
+	"premium",
+	"explore",
+	"following",
+	"subscriptions",
+	"notifications",
+	"history",
+	"watch",
+	"contact",
+	"ads",
+	"dmca",
+	"copyright",
+	"community"
+];
+const dailymotionHandler = {
+	match: (url) => {
+		return isHostOf(url, hosts);
+	},
+	resolve: (url) => {
+		const { pathname } = new URL(url);
+		const playlistMatch = pathname.match(playlistPathRegex);
+		if (playlistMatch?.[1]) return [`https://www.dailymotion.com/rss/playlist/${playlistMatch[1]}`];
+		const userMatch = pathname.match(userPathRegex);
+		if (userMatch?.[1]) {
+			const username = userMatch[1];
+			if (!isAnyOf(username, excludedPaths)) return [`https://www.dailymotion.com/rss/${username}`];
+		}
+		return [];
+	}
+};
+
+//#endregion
+export { dailymotionHandler };

package/dist/feeds/platform/handlers/deviantart.cjs

@@ -0,0 +1,50 @@
+const require_utils = require('../../../common/utils.cjs');
+
+//#region src/feeds/platform/handlers/deviantart.ts
+const hosts = ["deviantart.com", "www.deviantart.com"];
+const feedBaseUrl = "https://backend.deviantart.com/rss.xml";
+const excludedPaths = [
+	"about",
+	"join",
+	"search",
+	"topic",
+	"watch",
+	"notifications",
+	"settings",
+	"submit",
+	"shop",
+	"core-membership",
+	"team",
+	"developers"
+];
+const deviantartHandler = {
+	match: (url) => {
+		return require_utils.isHostOf(url, hosts);
+	},
+	resolve: (url) => {
+		const { pathname } = new URL(url);
+		const tagMatch = pathname.match(/^\/tag\/([^/]+)/);
+		if (tagMatch?.[1]) {
+			const tag = tagMatch[1];
+			return [`${feedBaseUrl}?type=deviation&q=${encodeURIComponent(`tag:${tag}`)}`];
+		}
+		const favMatch = pathname.match(/^\/([a-zA-Z0-9_-]+)\/favourites\/?$/);
+		if (favMatch?.[1]) {
+			const username = favMatch[1];
+			if (!require_utils.isAnyOf(username, excludedPaths)) return [`${feedBaseUrl}?type=deviation&q=${encodeURIComponent(`favby:${username}`)}`];
+		}
+		const folderMatch = pathname.match(/^\/([a-zA-Z0-9_-]+)\/gallery\/(\d+)(?:\/|$)/);
+		if (folderMatch?.[1] && folderMatch?.[2]) {
+			const username = folderMatch[1];
+			const folderId = folderMatch[2];
+			if (!require_utils.isAnyOf(username, excludedPaths)) return [`${feedBaseUrl}?type=deviation&q=${encodeURIComponent(`gallery:${username}/${folderId}`)}`];
+		}
+		const username = pathname.match(/^\/([a-zA-Z0-9_-]+)(?:\/gallery(?:\/all)?)?(?:\/|$)/)?.[1];
+		if (!username || require_utils.isAnyOf(username, excludedPaths)) return [];
+		const query = `by:${username} sort:time meta:all`;
+		return [`${feedBaseUrl}?type=deviation&q=${encodeURIComponent(query)}`];
+	}
+};
+
+//#endregion
+exports.deviantartHandler = deviantartHandler;

package/dist/feeds/platform/handlers/deviantart.js

@@ -0,0 +1,50 @@
+import { isAnyOf, isHostOf } from "../../../common/utils.js";
+
+//#region src/feeds/platform/handlers/deviantart.ts
+const hosts = ["deviantart.com", "www.deviantart.com"];
+const feedBaseUrl = "https://backend.deviantart.com/rss.xml";
+const excludedPaths = [
+	"about",
+	"join",
+	"search",
+	"topic",
+	"watch",
+	"notifications",
+	"settings",
+	"submit",
+	"shop",
+	"core-membership",
+	"team",
+	"developers"
+];
+const deviantartHandler = {
+	match: (url) => {
+		return isHostOf(url, hosts);
+	},
+	resolve: (url) => {
+		const { pathname } = new URL(url);
+		const tagMatch = pathname.match(/^\/tag\/([^/]+)/);
+		if (tagMatch?.[1]) {
+			const tag = tagMatch[1];
+			return [`${feedBaseUrl}?type=deviation&q=${encodeURIComponent(`tag:${tag}`)}`];
+		}
+		const favMatch = pathname.match(/^\/([a-zA-Z0-9_-]+)\/favourites\/?$/);
+		if (favMatch?.[1]) {
+			const username = favMatch[1];
+			if (!isAnyOf(username, excludedPaths)) return [`${feedBaseUrl}?type=deviation&q=${encodeURIComponent(`favby:${username}`)}`];
+		}
+		const folderMatch = pathname.match(/^\/([a-zA-Z0-9_-]+)\/gallery\/(\d+)(?:\/|$)/);
+		if (folderMatch?.[1] && folderMatch?.[2]) {
+			const username = folderMatch[1];
+			const folderId = folderMatch[2];
+			if (!isAnyOf(username, excludedPaths)) return [`${feedBaseUrl}?type=deviation&q=${encodeURIComponent(`gallery:${username}/${folderId}`)}`];
+		}
+		const username = pathname.match(/^\/([a-zA-Z0-9_-]+)(?:\/gallery(?:\/all)?)?(?:\/|$)/)?.[1];
+		if (!username || isAnyOf(username, excludedPaths)) return [];
+		const query = `by:${username} sort:time meta:all`;
+		return [`${feedBaseUrl}?type=deviation&q=${encodeURIComponent(query)}`];
+	}
+};
+
+//#endregion
+export { deviantartHandler };

package/dist/feeds/platform/handlers/devto.cjs

@@ -0,0 +1,44 @@
+const require_utils = require('../../../common/utils.cjs');
+
+//#region src/feeds/platform/handlers/devto.ts
+const hosts = ["dev.to", "www.dev.to"];
+const userPathRegex = /^\/([a-zA-Z0-9_]+)\/?$/;
+const tagPathRegex = /^\/t\/([^/]+)/;
+const excludedPaths = [
+	"tag",
+	"tags",
+	"search",
+	"top",
+	"latest",
+	"about",
+	"contact",
+	"privacy",
+	"terms",
+	"code-of-conduct",
+	"faq",
+	"enter",
+	"settings",
+	"signout-confirm",
+	"notifications",
+	"reading-list",
+	"dashboard"
+];
+const devtoHandler = {
+	match: (url) => {
+		return require_utils.isHostOf(url, hosts);
+	},
+	resolve: (url) => {
+		const { pathname } = new URL(url);
+		const userMatch = pathname.match(userPathRegex);
+		if (userMatch?.[1]) {
+			const username = userMatch[1];
+			if (!require_utils.isAnyOf(username, excludedPaths)) return [`https://dev.to/feed/${username}`];
+		}
+		const tagMatch = pathname.match(tagPathRegex);
+		if (tagMatch?.[1]) return [`https://dev.to/feed/tag/${tagMatch[1]}`];
+		return [];
+	}
+};
+
+//#endregion
+exports.devtoHandler = devtoHandler;

package/dist/feeds/platform/handlers/devto.js

@@ -0,0 +1,44 @@
+import { isAnyOf, isHostOf } from "../../../common/utils.js";
+
+//#region src/feeds/platform/handlers/devto.ts
+const hosts = ["dev.to", "www.dev.to"];
+const userPathRegex = /^\/([a-zA-Z0-9_]+)\/?$/;
+const tagPathRegex = /^\/t\/([^/]+)/;
+const excludedPaths = [
+	"tag",
+	"tags",
+	"search",
+	"top",
+	"latest",
+	"about",
+	"contact",
+	"privacy",
+	"terms",
+	"code-of-conduct",
+	"faq",
+	"enter",
+	"settings",
+	"signout-confirm",
+	"notifications",
+	"reading-list",
+	"dashboard"
+];
+const devtoHandler = {
+	match: (url) => {
+		return isHostOf(url, hosts);
+	},
+	resolve: (url) => {
+		const { pathname } = new URL(url);
+		const userMatch = pathname.match(userPathRegex);
+		if (userMatch?.[1]) {
+			const username = userMatch[1];
+			if (!isAnyOf(username, excludedPaths)) return [`https://dev.to/feed/${username}`];
+		}
+		const tagMatch = pathname.match(tagPathRegex);
+		if (tagMatch?.[1]) return [`https://dev.to/feed/tag/${tagMatch[1]}`];
+		return [];
+	}
+};
+
+//#endregion
+export { devtoHandler };