feedscout 2.0.0-rc.1 → 2.0.0-rc.2

package/dist/feeds.d.ts

@@ -1,4 +1,4 @@
-import { anchorAttributes, anchorLabels, anchorPathSegments, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal } from "./src/feeds/defaults.js";
+import { anchorAttributes, anchorLabels, anchorPathSegments, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, sectionNames, urisBalanced, urisComprehensive, urisMinimal } from "./src/feeds/defaults.js";
 import { FeedResult } from "./src/feeds/types.js";
 import { defaultExtractFn } from "./src/feeds/extractors.js";
-export { FeedResult, anchorAttributes, anchorLabels, anchorPathSegments, defaultExtractFn, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
+export { FeedResult, anchorAttributes, anchorLabels, anchorPathSegments, defaultExtractFn, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, sectionNames, urisBalanced, urisComprehensive, urisMinimal };

package/dist/feeds.js

@@ -1,3 +1,3 @@
-import { anchorAttributes, anchorLabels, anchorPathSegments, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal } from "./src/feeds/defaults.js";
+import { anchorAttributes, anchorLabels, anchorPathSegments, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, sectionNames, urisBalanced, urisComprehensive, urisMinimal } from "./src/feeds/defaults.js";
 import { defaultExtractFn } from "./src/feeds/extractors.js";
-export { anchorAttributes, anchorLabels, anchorPathSegments, defaultExtractFn, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
+export { anchorAttributes, anchorLabels, anchorPathSegments, defaultExtractFn, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, sectionNames, urisBalanced, urisComprehensive, urisMinimal };

package/dist/methods.d.ts

@@ -1,7 +1,7 @@
 import { discoverUrisFromFeed } from "./src/common/uris/feed/index.js";
 import { discoverUrisFromGuess } from "./src/common/uris/guess/index.js";
-import { generateUrlCombinations, getSubdomainVariants, getWwwCounterpart } from "./src/common/uris/guess/utils.js";
+import { extractSectionBaseUrls, generateUrlCombinations, getSubdomainVariants, getWwwCounterpart } from "./src/common/uris/guess/utils.js";
 import { discoverUrisFromHeaders } from "./src/common/uris/headers/index.js";
 import { discoverUrisFromHtml } from "./src/common/uris/html/index.js";
 import { discoverUrisFromPlatform } from "./src/common/uris/platform/index.js";
-export { discoverUrisFromFeed, discoverUrisFromGuess, discoverUrisFromHeaders, discoverUrisFromHtml, discoverUrisFromPlatform, generateUrlCombinations, getSubdomainVariants, getWwwCounterpart };
+export { discoverUrisFromFeed, discoverUrisFromGuess, discoverUrisFromHeaders, discoverUrisFromHtml, discoverUrisFromPlatform, extractSectionBaseUrls, generateUrlCombinations, getSubdomainVariants, getWwwCounterpart };

package/dist/methods.js

@@ -1,7 +1,7 @@
 import { discoverUrisFromFeed } from "./src/common/uris/feed/index.js";
-import { generateUrlCombinations, getSubdomainVariants, getWwwCounterpart } from "./src/common/uris/guess/utils.js";
+import { extractSectionBaseUrls, generateUrlCombinations, getSubdomainVariants, getWwwCounterpart } from "./src/common/uris/guess/utils.js";
 import { discoverUrisFromGuess } from "./src/common/uris/guess/index.js";
 import { discoverUrisFromHeaders } from "./src/common/uris/headers/index.js";
 import { discoverUrisFromHtml } from "./src/common/uris/html/index.js";
 import { discoverUrisFromPlatform } from "./src/common/uris/platform/index.js";
-export { discoverUrisFromFeed, discoverUrisFromGuess, discoverUrisFromHeaders, discoverUrisFromHtml, discoverUrisFromPlatform, generateUrlCombinations, getSubdomainVariants, getWwwCounterpart };
+export { discoverUrisFromFeed, discoverUrisFromGuess, discoverUrisFromHeaders, discoverUrisFromHtml, discoverUrisFromPlatform, extractSectionBaseUrls, generateUrlCombinations, getSubdomainVariants, getWwwCounterpart };

package/dist/node_modules/trousse/dist/urls.js

@@ -6,6 +6,9 @@ const parseUrl = (value, base) => {
 		return new URL(value, base);
 	} catch {}
 };
+const getPathSegments = (value) => {
+	return parseUrl(value)?.pathname.split("/").filter(Boolean) ?? [];
+};
 const isHostOf = (url, hosts) => {
 	const hostname = parseUrl(url)?.hostname;
 	if (!hostname) return false;
@@ -17,4 +20,4 @@ const isSubdomainOf = (url, domains) => {
 	return endsWithAnyOf(hostname, (Array.isArray(domains) ? domains : [domains]).map((domain) => `.${domain}`));
 };
 //#endregion
-export { isHostOf, isSubdomainOf, parseUrl };
+export { getPathSegments, isHostOf, isSubdomainOf, parseUrl };

package/dist/src/common/discover/utils.js

@@ -94,6 +94,7 @@ const normalizeMethodsConfig = (sourceInput, siteInput, methods, defaults) => {
 		const guessOptions = methodsObj.guess === true ? {} : methodsObj.guess;
 		methodsConfig.guess = { options: {
 			...defaults.guess,
+			content: resolvedInput.content,
 			...guessOptions,
 			baseUrl: resolvedInput.url
 		} };

package/dist/src/common/uris/guess/index.js

@@ -1,8 +1,16 @@
-import { generateUrlCombinations } from "./utils.js";
+import { extractSectionBaseUrls, generatePathUrlCombinations, generateUrlCombinations, getAncestorPathBases } from "./utils.js";
 //#region src/common/uris/guess/index.ts
 const discoverUrisFromGuess = (options) => {
-	const { baseUrl, uris, additionalBaseUrls = [] } = options;
-	return generateUrlCombinations([baseUrl, ...additionalBaseUrls], uris);
+	const { baseUrl, uris, additionalBaseUrls = [], maxAncestorDepth = 0 } = options;
+	const { content, sectionNames } = options;
+	const combinations = generateUrlCombinations([baseUrl, ...additionalBaseUrls], uris);
+	const pathBases = [];
+	if (maxAncestorDepth > 0) pathBases.push(...getAncestorPathBases(baseUrl, maxAncestorDepth));
+	if (content && sectionNames && sectionNames.length > 0) {
+		for (const sectionBase of extractSectionBaseUrls(content, baseUrl, sectionNames)) if (!pathBases.includes(sectionBase)) pathBases.push(sectionBase);
+	}
+	if (pathBases.length > 0) combinations.push(...generatePathUrlCombinations(pathBases, uris));
+	return combinations;
 };
 //#endregion
 export { discoverUrisFromGuess };

package/dist/src/common/uris/guess/types.d.ts

@@ -5,6 +5,9 @@ type GuessMethodOptions = {
   baseUrl: string;
   uris: Array<UriEntry>;
   additionalBaseUrls?: Array<string>;
+  maxAncestorDepth?: number;
+  content?: string;
+  sectionNames?: Array<string>;
 };
 //#endregion
 export { GuessMethodOptions };

package/dist/src/common/uris/guess/utils.d.ts

@@ -2,7 +2,8 @@ import { UriEntry } from "../../types.js";
 
 //#region src/common/uris/guess/utils.d.ts
 declare const generateUrlCombinations: (baseUrls: Array<string>, uris: Array<UriEntry>) => Array<UriEntry>;
+declare const extractSectionBaseUrls: (content: string, baseUrl: string, sectionNames: Array<string>) => Array<string>;
 declare const getWwwCounterpart: (baseUrl: string) => string;
 declare const getSubdomainVariants: (baseUrl: string, prefixes: Array<string>) => Array<string>;
 //#endregion
-export { generateUrlCombinations, getSubdomainVariants, getWwwCounterpart };
+export { extractSectionBaseUrls, generateUrlCombinations, getSubdomainVariants, getWwwCounterpart };

package/dist/src/common/uris/guess/utils.js

@@ -1,3 +1,5 @@
+import { getPathSegments, parseUrl } from "../../../../node_modules/trousse/dist/urls.js";
+import { Parser } from "htmlparser2";
 //#region src/common/uris/guess/utils.ts
 const ipAddressRegex = /^\d+\.\d+\.\d+\.\d+$/;
 const resolveUri = (uri, base, origin, pathname) => {
@@ -16,6 +18,47 @@ const generateUrlCombinations = (baseUrls, uris) => {
 		});
 	});
 };
+const resolvePathUri = (uri, base) => {
+	if (uri.startsWith("/")) return new URL(uri.slice(1), base).href;
+	return new URL(uri, base).href;
+};
+const generatePathUrlCombinations = (pathBases, uris) => {
+	return pathBases.flatMap((base) => {
+		const normalizedBase = base.endsWith("/") ? base : `${base}/`;
+		return uris.map((uri) => {
+			if (typeof uri === "string") return resolvePathUri(uri, normalizedBase);
+			return uri.map((alternative) => resolvePathUri(alternative, normalizedBase));
+		});
+	});
+};
+const getAncestorPathBases = (baseUrl, maxDepth) => {
+	const url = parseUrl(baseUrl);
+	if (!url) return [];
+	const segments = url.pathname.split("/").slice(1, -1).filter((segment) => segment !== "");
+	const bases = [];
+	let base = `${url.origin}/`;
+	for (const segment of segments.slice(0, maxDepth)) {
+		base += `${segment}/`;
+		bases.push(base);
+	}
+	return bases;
+};
+const extractSectionBaseUrls = (content, baseUrl, sectionNames) => {
+	const base = parseUrl(baseUrl);
+	if (!base) return [];
+	const bases = /* @__PURE__ */ new Set();
+	const parser = new Parser({ onopentag: (name, attributes) => {
+		if (name !== "a" || !attributes.href) return;
+		const url = parseUrl(attributes.href, base);
+		if (!url || url.origin !== base.origin) return;
+		const segments = getPathSegments(url);
+		const segment = segments.length === 1 ? segments[0] : void 0;
+		if (segment && sectionNames.includes(segment.toLowerCase())) bases.add(`${url.origin}/${segment}/`);
+	} }, { decodeEntities: true });
+	parser.write(content);
+	parser.end();
+	return [...bases];
+};
 const getWwwCounterpart = (baseUrl) => {
 	const url = new URL(baseUrl);
 	const port = url.port ? `:${url.port}` : "";
@@ -37,4 +80,4 @@ const getSubdomainVariants = (baseUrl, prefixes) => {
 	});
 };
 //#endregion
-export { generateUrlCombinations, getSubdomainVariants, getWwwCounterpart };
+export { extractSectionBaseUrls, generatePathUrlCombinations, generateUrlCombinations, getAncestorPathBases, getSubdomainVariants, getWwwCounterpart };

package/dist/src/feeds/defaults.d.ts

@@ -16,7 +16,8 @@ declare const anchorPathSegments: RegExp[];
 declare const anchorAttributes: string[];
 declare const defaultHtmlOptions: Omit<HtmlMethodOptions, 'baseUrl'>;
 declare const defaultHeadersOptions: Omit<HeadersMethodOptions, 'baseUrl'>;
+declare const sectionNames: string[];
 declare const defaultGuessOptions: Omit<GuessMethodOptions, 'baseUrl'>;
 declare const defaultPlatformOptions: Omit<PlatformMethodOptions, 'baseUrl'>;
 //#endregion
-export { anchorAttributes, anchorLabels, anchorPathSegments, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
+export { anchorAttributes, anchorLabels, anchorPathSegments, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, sectionNames, urisBalanced, urisComprehensive, urisMinimal };

package/dist/src/feeds/defaults.js

@@ -188,7 +188,22 @@ const defaultHtmlOptions = {
 	anchorAttributes
 };
 const defaultHeadersOptions = { linkSelectors };
-const defaultGuessOptions = { uris: urisBalanced };
+const sectionNames = [
+	"blog",
+	"news",
+	"posts",
+	"articles",
+	"writing",
+	"notes",
+	"journal",
+	"podcast",
+	"changelog"
+];
+const defaultGuessOptions = {
+	uris: urisBalanced,
+	maxAncestorDepth: 2,
+	sectionNames
+};
 const defaultPlatformOptions = { handlers: [
 	acastHandler,
 	amebloHandler,
@@ -282,4 +297,4 @@ const defaultPlatformOptions = { handlers: [
 	zennHandler
 ] };
 //#endregion
-export { anchorAttributes, anchorLabels, anchorPathSegments, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, urisBalanced, urisComprehensive, urisMinimal };
+export { anchorAttributes, anchorLabels, anchorPathSegments, defaultGuessOptions, defaultHeadersOptions, defaultHtmlOptions, defaultPlatformOptions, ignoredUris, linkSelectors, mimeTypes, sectionNames, urisBalanced, urisComprehensive, urisMinimal };

package/package.json

@@ -76,5 +76,5 @@
     "tsdown": "^0.22.2",
     "vitepress": "^2.0.0-alpha.17"
   },
-  "version": "2.0.0-rc.1"
+  "version": "2.0.0-rc.2"
 }