@vannizhang/living-atlas-content-validator 1.5.17 → 1.5.18-beta.2

package/dist/lib/checkTitleAndSnippetSearchability/matchLocationInfo.d.ts

@@ -0,0 +1,34 @@
+import { IItem } from '../../types';
+import { MatchResult } from './helpers';
+/**
+ * Sets extra matching and rejected patterns for locations and updates the corresponding regex and sets.
+ *
+ * @param extraMatchingPatternsForLocations - An array of strings representing additional patterns to match locations.
+ * @param extraRejectedPatternsForLocations - An array of strings representing additional patterns to reject locations.
+ * @param {boolean} [shouldAvoidUsingWordBoundary=false] - A boolean indicating whether to avoid using word boundaries (useful for languages like Chinese and Japanese).
+ *
+ * @remarks
+ * - For languages like Japanese that don't use spaces between words (or where word boundaries are not easily determined by spaces or punctuation),
+ *   set `shouldAvoidUsingWordBoundary` to true to remove `\b` and use lookarounds to ensure correct boundaries.
+ */
+export declare const setExtraMatchingPatternsForLocations: (extraMatchingPatternsForLocations?: string[], extraRejectedPatternsForLocations?: string[], shouldAvoidUsingWordBoundary?: boolean) => void;
+/**
+ * Matches location information from an item's title and snippet using predefined or custom matching patterns.
+ *
+ * @param {IItem} item - The item containing title and snippet to search for location information.
+ * @param {string[]} [customMatchingPattenrs] - Optional custom matching patterns to be included in the search.
+ * @returns {string[]} - An array of matched results for location info after deduplication and removing overlaps.
+ */
+export declare const matchLocationInfo: (item: IItem, customMatchingPattenrs?: string[]) => MatchResult[];
+/**
+ * Determines if a given string is a recognized location information.
+ * @param topic - The string to check.
+ * @returns {boolean} - True if the string is a recognized location, false otherwise.
+ */
+export declare const isRecognizedLocation: (str: string) => boolean;
+/**
+ * Determines if a given string is a rejected source information.
+ * @param str - The string to check.
+ * @returns {boolean} - True if the string is a rejected source information, false otherwise.
+ */
+export declare const isRejectedLocationInfo: (str: string) => boolean;

package/dist/lib/checkTitleAndSnippetSearchability/matchLocationInfo.js

@@ -0,0 +1,170 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isRejectedLocationInfo = exports.isRecognizedLocation = exports.matchLocationInfo = exports.setExtraMatchingPatternsForLocations = void 0;
+const helpers_1 = require("./helpers");
+const TitleSummaryMatchingPatterns_Locations_json_1 = __importDefault(require("../../data/TitleSummaryMatchingPatterns_Locations.json"));
+const TitleSummaryRejectedPatterns_json_1 = __importDefault(require("../../data/TitleSummaryRejectedPatterns.json"));
+const createWordBoundaryRegex_1 = require("../util/createWordBoundaryRegex");
+// const matchingPatternsForLocations = TitleSummaryMatchingPatterns.locations;
+// Combine patterns into one
+// const matchingPatterns = ALL_LOCATIONS.join('|');
+const regexForLocations = (0, createWordBoundaryRegex_1.createWordBoundaryRegex)(TitleSummaryMatchingPatterns_Locations_json_1.default);
+// const combinedRegex = new RegExp(`\\b(${matchingPatterns})\\b`, 'gi');
+// /**
+//  * Set of recognized locations.
+//  */
+// let locationSet: Set<string> = null;
+/**
+ * Set of rejected locations.
+ */
+let rejectedLocationSet = null; //new Set<string>(TitleSummaryMatchingPatterns.topics)
+/**
+ * Regular expression to match additional location information.
+ */
+let regexForExtraLocations = null;
+// /**
+//  * Creates a set of recognized locations from predefined and extra matching patterns.
+//  * @param extraMatchingPatternsForLocations
+//  */
+// const createLocationSet = (
+//     extraMatchingPatternsForLocations: string[] = []
+// ) => {
+//     locationSet = toLowercaseSet([
+//         ...matchingPatternsForLocations,
+//         ...extraMatchingPatternsForLocations,
+//     ]);
+// };
+/**
+ * Creates a set of rejected locations from predefined and extra rejected patterns.
+ * @param extraRejectedPatternsForLocations
+ */
+const createRejectedLocationSet = (extraRejectedPatternsForLocations = []) => {
+    rejectedLocationSet = (0, helpers_1.toLowercaseSet)([
+        ...TitleSummaryRejectedPatterns_json_1.default.common,
+        ...TitleSummaryRejectedPatterns_json_1.default.locations,
+        ...extraRejectedPatternsForLocations,
+    ]);
+};
+/**
+ * Sets extra matching and rejected patterns for locations and updates the corresponding regex and sets.
+ *
+ * @param extraMatchingPatternsForLocations - An array of strings representing additional patterns to match locations.
+ * @param extraRejectedPatternsForLocations - An array of strings representing additional patterns to reject locations.
+ * @param {boolean} [shouldAvoidUsingWordBoundary=false] - A boolean indicating whether to avoid using word boundaries (useful for languages like Chinese and Japanese).
+ *
+ * @remarks
+ * - For languages like Japanese that don't use spaces between words (or where word boundaries are not easily determined by spaces or punctuation),
+ *   set `shouldAvoidUsingWordBoundary` to true to remove `\b` and use lookarounds to ensure correct boundaries.
+ */
+const setExtraMatchingPatternsForLocations = (extraMatchingPatternsForLocations = [], extraRejectedPatternsForLocations = [], shouldAvoidUsingWordBoundary = false) => {
+    if (extraMatchingPatternsForLocations.length) {
+        regexForExtraLocations = (0, createWordBoundaryRegex_1.createWordBoundaryRegex)(extraMatchingPatternsForLocations, {
+            shouldAvoidUsingWordBoundary,
+        });
+    }
+    // createLocationSet(extraMatchingPatternsForLocations);
+    createRejectedLocationSet(extraRejectedPatternsForLocations);
+};
+exports.setExtraMatchingPatternsForLocations = setExtraMatchingPatternsForLocations;
+/**
+ * Matches location information from an item's title and snippet using predefined or custom matching patterns.
+ *
+ * @param {IItem} item - The item containing title and snippet to search for location information.
+ * @param {string[]} [customMatchingPattenrs] - Optional custom matching patterns to be included in the search.
+ * @returns {string[]} - An array of matched results for location info after deduplication and removing overlaps.
+ */
+const matchLocationInfo = (item, customMatchingPattenrs) => {
+    // const title = item.title || '';
+    // const snippet = item.snippet || '';
+    // const matchedLocationsFromTitle: string[] =
+    //     title.match(combinedRegex) || [];
+    // const matchedLocationsFromSnippet: string[] =
+    //     snippet.match(combinedRegex) || [];
+    // if (
+    //     !matchedLocationsFromTitle.length &&
+    //     !matchedLocationsFromSnippet.length
+    // ) {
+    //     return matchWithCustomPatterns(
+    //         item,
+    //         customMatchingPattenrs,
+    //         rejectedLocationSet
+    //     );
+    // }
+    // return cleanUpMatchedResults([
+    //     ...matchedLocationsFromTitle,
+    //     ...matchedLocationsFromSnippet,
+    // ]);
+    // let matched: string[] = matchPatternsInTitleAndSnippet(
+    //     item,
+    //     regexForLocations
+    // );
+    // if (!matched.length && regexForExtraLocations) {
+    //     matched = matchPatternsInTitleAndSnippet(item, regexForExtraLocations);
+    // }
+    // if (!matched.length && customMatchingPattenrs?.length) {
+    //     if (!rejectedLocationSet) {
+    //         createRejectedLocationSet();
+    //     }
+    //     matched = matchWithCustomPatterns(
+    //         item,
+    //         customMatchingPattenrs,
+    //         rejectedLocationSet
+    //     );
+    // }
+    // return matched;
+    if (!rejectedLocationSet) {
+        createRejectedLocationSet();
+    }
+    return (0, helpers_1.matchUsingMultiplePatterns)({
+        item,
+        predefinedPatterns: regexForLocations,
+        extraPatterns: regexForExtraLocations,
+        customTerms: customMatchingPattenrs,
+        rejectedSet: rejectedLocationSet,
+    });
+};
+exports.matchLocationInfo = matchLocationInfo;
+/**
+ * Determines if a given string is a recognized location information.
+ * @param topic - The string to check.
+ * @returns {boolean} - True if the string is a recognized location, false otherwise.
+ */
+const isRecognizedLocation = (str) => {
+    // if (!locationSet) {
+    //     const sources = ALL_LOCATIONS.map((d) => d.toLowerCase());
+    //     locationSet = new Set<string>(sources);
+    // }
+    // locationSet = locationSet || toLowercaseSet(matchingPatternsForLocations);
+    // if (!locationSet) {
+    //     createLocationSet();
+    // }
+    // return isFoundInSet(str, locationSet);
+    const locationInfo = (0, exports.matchLocationInfo)({
+        title: str,
+        snippet: '',
+    });
+    return locationInfo.length > 0;
+};
+exports.isRecognizedLocation = isRecognizedLocation;
+/**
+ * Determines if a given string is a rejected source information.
+ * @param str - The string to check.
+ * @returns {boolean} - True if the string is a rejected source information, false otherwise.
+ */
+const isRejectedLocationInfo = (str) => {
+    // if (!rejectedLocationSet) {
+    //     rejectedLocationSet = toLowercaseSet([
+    //         ...TitleSummaryRejectedPatterns.common,
+    //         ...TitleSummaryRejectedPatterns.locations,
+    //     ]);
+    // }
+    if (!rejectedLocationSet) {
+        createRejectedLocationSet();
+    }
+    return (0, helpers_1.isFoundInSet)(str, rejectedLocationSet);
+};
+exports.isRejectedLocationInfo = isRejectedLocationInfo;
+//# sourceMappingURL=matchLocationInfo.js.map

package/dist/lib/checkTitleAndSnippetSearchability/matchLocationInfo.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"matchLocationInfo.js","sourceRoot":"","sources":["../../../src/lib/checkTitleAndSnippetSearchability/matchLocationInfo.ts"],"names":[],"mappings":";;;;;;AACA,uCAUmB;AACnB,yIAAkG;AAClG,qHAAwF;AACxF,6EAA0E;AAE1E,+EAA+E;AAE/E,4BAA4B;AAC5B,oDAAoD;AAEpD,MAAM,iBAAiB,GAAG,IAAA,iDAAuB,EAAC,qDAA4B,CAAC,CAAC;AAChF,yEAAyE;AAEzE,MAAM;AACN,kCAAkC;AAClC,MAAM;AACN,uCAAuC;AAEvC;;GAEG;AACH,IAAI,mBAAmB,GAAgB,IAAI,CAAC,CAAC,sDAAsD;AAEnG;;GAEG;AACH,IAAI,sBAAsB,GAAW,IAAI,CAAC;AAE1C,MAAM;AACN,wFAAwF;AACxF,8CAA8C;AAC9C,MAAM;AACN,8BAA8B;AAC9B,uDAAuD;AACvD,SAAS;AACT,qCAAqC;AACrC,2CAA2C;AAC3C,gDAAgD;AAChD,UAAU;AACV,KAAK;AAEL;;;GAGG;AACH,MAAM,yBAAyB,GAAG,CAC9B,oCAA8C,EAAE,EAClD,EAAE;IACA,mBAAmB,GAAG,IAAA,wBAAc,EAAC;QACjC,GAAG,2CAA4B,CAAC,MAAM;QACtC,GAAG,2CAA4B,CAAC,SAAS;QACzC,GAAG,iCAAiC;KACvC,CAAC,CAAC;AACP,CAAC,CAAC;AAEF;;;;;;;;;;GAUG;AACI,MAAM,oCAAoC,GAAG,CAChD,oCAA8C,EAAE,EAChD,oCAA8C,EAAE,EAChD,4BAA4B,GAAG,KAAK,EACtC,EAAE;IACA,IAAI,iCAAiC,CAAC,MAAM,EAAE;QAC1C,sBAAsB,GAAG,IAAA,iDAAuB,EAC5C,iCAAiC,EACjC;YACI,4BAA4B;SAC/B,CACJ,CAAC;KACL;IAED,wDAAwD;IACxD,yBAAyB,CAAC,iCAAiC,CAAC,CAAC;AACjE,CAAC,CAAC;AAhBW,QAAA,oCAAoC,wCAgB/C;AAEF;;;;;;GAMG;AACI,MAAM,iBAAiB,GAAG,CAC7B,IAAW,EACX,sBAAiC,EACpB,EAAE;IACf,kCAAkC;IAClC,sCAAsC;IAEtC,8CAA8C;IAC9C,wCAAwC;IACxC,gDAAgD;IAChD,0CAA0C;IAE1C,OAAO;IACP,2CAA2C;IAC3C,0CAA0C;IAC1C,MAAM;IACN,sCAAsC;IACtC,gBAAgB;IAChB,kCAAkC;IAClC,8BAA8B;IAC9B,SAAS;IACT,IAAI;IAEJ,iCAAiC;IACjC,oCAAoC;IACpC,sCAAsC;IACtC,MAAM;IAEN,0DAA0D;IAC1D,YAAY;IACZ,wBAAwB;IACxB,KAAK;IAEL,mDAAmD;IACnD,8EAA8E;IAC9E,IAAI;IAEJ,2DAA2D;IAC3D,kCAAkC;IAClC,uCAAuC;IACvC,QAAQ;IAER,yCAAyC;IACzC,gBAAgB;IAChB,kCAAkC;IAClC,8BAA8B;IAC9B,SAAS;IACT,IAAI;IAEJ,kBAAkB;IAElB,IAAI,CAAC,mBAAmB,EAAE;QACtB,yBAAyB,EAAE,CAAC;KAC/B;IAED,OAAO,IAAA,oCAA0B,EAAC;QAC9B,IAAI;QACJ,kBAAkB,EAAE,iBAAiB;QACrC,aAAa,EAAE,sBAAsB;QACrC,WAAW,EAAE,sBAAsB;QACnC,WAAW,EAAE,mBAAmB;KACnC,CAAC,CAAC;AACP,CAAC,CAAC;AA9DW,QAAA,iBAAiB,qBA8D5B;AAEF;;;;GAIG;AACI,MAAM,oBAAoB,GAAG,CAAC,GAAW,EAAW,EAAE;IACzD,sBAAsB;IACtB,iEAAiE;IACjE,8CAA8C;IAC9C,IAAI;IAEJ,6EAA6E;IAE7E,sBAAsB;IACtB,2BAA2B;IAC3B,IAAI;IAEJ,yCAAyC;IAEzC,MAAM,YAAY,GAAG,IAAA,yBAAiB,EAAC;QACnC,KAAK,EAAE,GAAG;QACV,OAAO,EAAE,EAAE;KACL,CAAC,CAAC;IAEZ,OAAO,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC;AACnC,CAAC,CAAC;AApBW,QAAA,oBAAoB,wBAoB/B;AAEF;;;;GAIG;AACI,MAAM,sBAAsB,GAAG,CAAC,GAAW,EAAW,EAAE;IAC3D,8BAA8B;IAC9B,6CAA6C;IAC7C,kDAAkD;IAClD,qDAAqD;IACrD,UAAU;IACV,IAAI;IAEJ,IAAI,CAAC,mBAAmB,EAAE;QACtB,yBAAyB,EAAE,CAAC;KAC/B;IAED,OAAO,IAAA,sBAAY,EAAC,GAAG,EAAE,mBAAmB,CAAC,CAAC;AAClD,CAAC,CAAC;AAbW,QAAA,sBAAsB,0BAajC"}

package/dist/lib/checkTitleAndSnippetSearchability/matchSourceInfo.d.ts

@@ -0,0 +1,34 @@
+import { IItem } from '../../types';
+import { MatchResult } from './helpers';
+/**
+ * Sets extra matching patterns for sources and creates corresponding regex and sets.
+ *
+ * @param extraMatchingPatternsForSources - An array of strings representing extra matching patterns for sources.
+ * @param extraRejectedPatternsForSources - An array of strings representing extra rejected patterns for sources.
+ * @param {boolean} [shouldAvoidUsingWordBoundary=false] - A boolean indicating whether to avoid using word boundaries (useful for languages like Chinese and Japanese).
+ *
+ * @remarks
+ * - For languages like Japanese that don't use spaces between words (or where word boundaries are not easily determined by spaces or punctuation),
+ *   set `shouldAvoidUsingWordBoundary` to true to remove `\b` and use lookarounds to ensure correct boundaries.
+ */
+export declare const setExtraMatchingPatternsForSources: (extraMatchingPatternsForSources: string[], extraRejectedPatternsForSources: string[], shouldAvoidUsingWordBoundary?: boolean) => void;
+/**
+ * Matches source information from an item's title and snippet using predefined or custom matching patterns.
+ *
+ * @param {IItem} item - The item containing title and snippet to search for matching patterns.
+ * @param {string[]} [customMatchingPattenrs] - Optional custom matching patterns to be used in addition to the predefined ones.
+ * @returns {string[]} - An array of matched results for source info after deduplication and removal of overlaps.
+ */
+export declare const matchSourceInfo: (item: IItem, customMatchingPattenrs?: string[]) => MatchResult[];
+/**
+ * Determines if a given string is a recognized source information.
+ * @param topic - The string to check.
+ * @returns {boolean} - True if the string is a recognized source, false otherwise.
+ */
+export declare const isRecognizedSource: (str: string) => boolean;
+/**
+ * Determines if a given string is a rejected source information.
+ * @param str - The string to check.
+ * @returns {boolean} - True if the string is a rejected source information, false otherwise.
+ */
+export declare const isRejectedSourceInfo: (str: string) => boolean;

package/dist/lib/checkTitleAndSnippetSearchability/matchSourceInfo.js

@@ -0,0 +1,172 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isRejectedSourceInfo = exports.isRecognizedSource = exports.matchSourceInfo = exports.setExtraMatchingPatternsForSources = void 0;
+const helpers_1 = require("./helpers");
+const TitleSummaryMatchingPatterns_Sources_json_1 = __importDefault(require("../../data/TitleSummaryMatchingPatterns_Sources.json"));
+const TitleSummaryRejectedPatterns_json_1 = __importDefault(require("../../data/TitleSummaryRejectedPatterns.json"));
+const createWordBoundaryRegex_1 = require("../util/createWordBoundaryRegex");
+// /**
+//  * Pre-defined matching patterns for sources.
+//  */
+// const matchingPatternsForSources = TitleSummaryMatchingPatterns.sources;
+/**
+ * Regular expression to match source information in a case-insensitive manner.
+ * The patterns in TitleSummaryMatchingPatterns is already escaped.
+ */
+const regexForSourceInfo = (0, createWordBoundaryRegex_1.createWordBoundaryRegex)(TitleSummaryMatchingPatterns_Sources_json_1.default);
+// const regexForSourceInfo = new RegExp(`\\b(${matchingPatternsForSources})\\b`, 'gi');
+// /**
+//  * set of matching patterns for source info
+//  */
+// let sourcesSet: Set<string> = null;
+/**
+ * Set of rejected sources.
+ */
+let rejectedSourcesSet = null;
+/**
+ * Regular expression to match additional source information.
+ */
+let regexForExtraSources = null;
+// /**
+//  * Create a set of recognized sources from predefined and extra matching patterns.
+//  * @param extraMatchingPatternsForSources
+//  */
+// const createSourcesSet = (extraMatchingPatternsForSources: string[] = []) => {
+//     sourcesSet = toLowercaseSet([
+//         ...TitleSummaryMatchingPatterns.sources,
+//         ...extraMatchingPatternsForSources,
+//     ]);
+// };
+/**
+ * Creates a set of rejected sources from predefined and extra rejected patterns.
+ *
+ * @param {string[]} extraRejectedPatternsForSources - Additional rejected patterns to be set.
+ */
+const createRejectedSourcesSet = (extraRejectedPatternsForSources = []) => {
+    rejectedSourcesSet = (0, helpers_1.toLowercaseSet)([
+        ...TitleSummaryRejectedPatterns_json_1.default.common,
+        ...TitleSummaryRejectedPatterns_json_1.default.sources,
+        ...extraRejectedPatternsForSources,
+    ]);
+};
+/**
+ * Sets extra matching patterns for sources and creates corresponding regex and sets.
+ *
+ * @param extraMatchingPatternsForSources - An array of strings representing extra matching patterns for sources.
+ * @param extraRejectedPatternsForSources - An array of strings representing extra rejected patterns for sources.
+ * @param {boolean} [shouldAvoidUsingWordBoundary=false] - A boolean indicating whether to avoid using word boundaries (useful for languages like Chinese and Japanese).
+ *
+ * @remarks
+ * - For languages like Japanese that don't use spaces between words (or where word boundaries are not easily determined by spaces or punctuation),
+ *   set `shouldAvoidUsingWordBoundary` to true to remove `\b` and use lookarounds to ensure correct boundaries.
+ */
+const setExtraMatchingPatternsForSources = (extraMatchingPatternsForSources, extraRejectedPatternsForSources, shouldAvoidUsingWordBoundary = false) => {
+    if (extraMatchingPatternsForSources &&
+        extraMatchingPatternsForSources.length > 0) {
+        regexForExtraSources = (0, createWordBoundaryRegex_1.createWordBoundaryRegex)(extraMatchingPatternsForSources, {
+            shouldAvoidUsingWordBoundary,
+        });
+    }
+    // createSourcesSet(extraMatchingPatternsForSources);
+    createRejectedSourcesSet(extraRejectedPatternsForSources);
+};
+exports.setExtraMatchingPatternsForSources = setExtraMatchingPatternsForSources;
+/**
+ * Matches source information from an item's title and snippet using predefined or custom matching patterns.
+ *
+ * @param {IItem} item - The item containing title and snippet to search for matching patterns.
+ * @param {string[]} [customMatchingPattenrs] - Optional custom matching patterns to be used in addition to the predefined ones.
+ * @returns {string[]} - An array of matched results for source info after deduplication and removal of overlaps.
+ */
+const matchSourceInfo = (item, customMatchingPattenrs) => {
+    // const title = item.title || '';
+    // const snippet = item.snippet || '';
+    // const matchedStrFromTitle: string[] = title.match(regex4SourceInfo) || [];
+    // const matchedStrFromSnippet: string[] =
+    //     snippet.match(regex4SourceInfo) || [];
+    // if (!matchedStrFromTitle.length && !matchedStrFromSnippet.length) {
+    //     return matchWithCustomPatterns(
+    //         item,
+    //         customMatchingPattenrs,
+    //         rejectedSourcesSet
+    //     );
+    // }
+    // return cleanUpMatchedResults([
+    //     ...matchedStrFromTitle,
+    //     ...matchedStrFromSnippet,
+    // ]);
+    // let matched: string[] = matchPatternsInTitleAndSnippet(
+    //     item,
+    //     regexForSourceInfo
+    // );
+    // if (!matched.length && regexForExtraSources) {
+    //     matched = matchPatternsInTitleAndSnippet(item, regexForExtraSources);
+    // }
+    // if (!matched.length && customMatchingPattenrs?.length) {
+    //     if (!rejectedSourcesSet) {
+    //         createRejectedSourcesSet();
+    //     }
+    //     matched = matchWithCustomPatterns(
+    //         item,
+    //         customMatchingPattenrs,
+    //         rejectedSourcesSet
+    //     );
+    // }
+    // return matched;
+    if (!rejectedSourcesSet) {
+        createRejectedSourcesSet();
+    }
+    return (0, helpers_1.matchUsingMultiplePatterns)({
+        item,
+        predefinedPatterns: regexForSourceInfo,
+        extraPatterns: regexForExtraSources,
+        customTerms: customMatchingPattenrs,
+        rejectedSet: rejectedSourcesSet,
+    });
+};
+exports.matchSourceInfo = matchSourceInfo;
+/**
+ * Determines if a given string is a recognized source information.
+ * @param topic - The string to check.
+ * @returns {boolean} - True if the string is a recognized source, false otherwise.
+ */
+const isRecognizedSource = (str) => {
+    // if (!sourcesSet) {
+    //     const sources = TitleSummaryMatchingPatterns.sources.map((d) =>
+    //         d.toLowerCase()
+    //     );
+    //     sourcesSet = new Set<string>(sources);
+    // }
+    // sourcesSet =
+    //     sourcesSet || toLowercaseSet(TitleSummaryMatchingPatterns?.sources);
+    // if (!sourcesSet) {
+    //     createSourcesSet();
+    // }
+    // return isFoundInSet(str, sourcesSet);
+    const sourceInfo = (0, exports.matchSourceInfo)({
+        title: str,
+        snippet: '',
+    });
+    return sourceInfo.length > 0;
+};
+exports.isRecognizedSource = isRecognizedSource;
+/**
+ * Determines if a given string is a rejected source information.
+ * @param str - The string to check.
+ * @returns {boolean} - True if the string is a rejected source information, false otherwise.
+ */
+const isRejectedSourceInfo = (str) => {
+    if (!rejectedSourcesSet) {
+        // rejectedSourcesSet = toLowercaseSet([
+        //     ...TitleSummaryRejectedPatterns.common,
+        //     ...TitleSummaryRejectedPatterns.sources,
+        // ]);
+        createRejectedSourcesSet();
+    }
+    return (0, helpers_1.isFoundInSet)(str, rejectedSourcesSet);
+};
+exports.isRejectedSourceInfo = isRejectedSourceInfo;
+//# sourceMappingURL=matchSourceInfo.js.map

package/dist/lib/checkTitleAndSnippetSearchability/matchSourceInfo.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"matchSourceInfo.js","sourceRoot":"","sources":["../../../src/lib/checkTitleAndSnippetSearchability/matchSourceInfo.ts"],"names":[],"mappings":";;;;;;AACA,uCAUmB;AACnB,qIAA8F;AAC9F,qHAAwF;AACxF,6EAA0E;AAE1E,MAAM;AACN,gDAAgD;AAChD,MAAM;AACN,2EAA2E;AAE3E;;;GAGG;AACH,MAAM,kBAAkB,GAAG,IAAA,iDAAuB,EAAC,mDAA0B,CAAC,CAAC;AAC/E,wFAAwF;AAExF,MAAM;AACN,8CAA8C;AAC9C,MAAM;AACN,sCAAsC;AACtC;;GAEG;AACH,IAAI,kBAAkB,GAAgB,IAAI,CAAC;AAE3C;;GAEG;AACH,IAAI,oBAAoB,GAAW,IAAI,CAAC;AAExC,MAAM;AACN,qFAAqF;AACrF,4CAA4C;AAC5C,MAAM;AACN,iFAAiF;AACjF,oCAAoC;AACpC,mDAAmD;AACnD,8CAA8C;AAC9C,UAAU;AACV,KAAK;AAEL;;;;GAIG;AACH,MAAM,wBAAwB,GAAG,CAC7B,kCAA4C,EAAE,EAChD,EAAE;IACA,kBAAkB,GAAG,IAAA,wBAAc,EAAC;QAChC,GAAG,2CAA4B,CAAC,MAAM;QACtC,GAAG,2CAA4B,CAAC,OAAO;QACvC,GAAG,+BAA+B;KACrC,CAAC,CAAC;AACP,CAAC,CAAC;AAEF;;;;;;;;;;GAUG;AACI,MAAM,kCAAkC,GAAG,CAC9C,+BAAyC,EACzC,+BAAyC,EACzC,4BAA4B,GAAG,KAAK,EACtC,EAAE;IACA,IACI,+BAA+B;QAC/B,+BAA+B,CAAC,MAAM,GAAG,CAAC,EAC5C;QACE,oBAAoB,GAAG,IAAA,iDAAuB,EAC1C,+BAA+B,EAC/B;YACI,4BAA4B;SAC/B,CACJ,CAAC;KACL;IAED,qDAAqD;IACrD,wBAAwB,CAAC,+BAA+B,CAAC,CAAC;AAC9D,CAAC,CAAC;AAnBW,QAAA,kCAAkC,sCAmB7C;AAEF;;;;;;GAMG;AACI,MAAM,eAAe,GAAG,CAC3B,IAAW,EACX,sBAAiC,EACpB,EAAE;IACf,kCAAkC;IAClC,sCAAsC;IAEtC,6EAA6E;IAC7E,0CAA0C;IAC1C,6CAA6C;IAE7C,sEAAsE;IACtE,sCAAsC;IACtC,gBAAgB;IAChB,kCAAkC;IAClC,6BAA6B;IAC7B,SAAS;IACT,IAAI;IAEJ,iCAAiC;IACjC,8BAA8B;IAC9B,gCAAgC;IAChC,MAAM;IAEN,0DAA0D;IAC1D,YAAY;IACZ,yBAAyB;IACzB,KAAK;IAEL,iDAAiD;IACjD,4EAA4E;IAC5E,IAAI;IAEJ,2DAA2D;IAC3D,iCAAiC;IACjC,sCAAsC;IACtC,QAAQ;IAER,yCAAyC;IACzC,gBAAgB;IAChB,kCAAkC;IAClC,6BAA6B;IAC7B,SAAS;IACT,IAAI;IAEJ,kBAAkB;IAElB,IAAI,CAAC,kBAAkB,EAAE;QACrB,wBAAwB,EAAE,CAAC;KAC9B;IAED,OAAO,IAAA,oCAA0B,EAAC;QAC9B,IAAI;QACJ,kBAAkB,EAAE,kBAAkB;QACtC,aAAa,EAAE,oBAAoB;QACnC,WAAW,EAAE,sBAAsB;QACnC,WAAW,EAAE,kBAAkB;KAClC,CAAC,CAAC;AACP,CAAC,CAAC;AA1DW,QAAA,eAAe,mBA0D1B;AAEF;;;;GAIG;AACI,MAAM,kBAAkB,GAAG,CAAC,GAAW,EAAW,EAAE;IACvD,qBAAqB;IACrB,sEAAsE;IACtE,0BAA0B;IAC1B,SAAS;IACT,6CAA6C;IAC7C,IAAI;IAEJ,eAAe;IACf,2EAA2E;IAE3E,qBAAqB;IACrB,0BAA0B;IAC1B,IAAI;IAEJ,wCAAwC;IAExC,MAAM,UAAU,GAAG,IAAA,uBAAe,EAAC;QAC/B,KAAK,EAAE,GAAG;QACV,OAAO,EAAE,EAAE;KACL,CAAC,CAAC;IAEZ,OAAO,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC;AACjC,CAAC,CAAC;AAvBW,QAAA,kBAAkB,sBAuB7B;AAEF;;;;GAIG;AACI,MAAM,oBAAoB,GAAG,CAAC,GAAW,EAAW,EAAE;IACzD,IAAI,CAAC,kBAAkB,EAAE;QACrB,wCAAwC;QACxC,8CAA8C;QAC9C,+CAA+C;QAC/C,MAAM;QACN,wBAAwB,EAAE,CAAC;KAC9B;IAED,OAAO,IAAA,sBAAY,EAAC,GAAG,EAAE,kBAAkB,CAAC,CAAC;AACjD,CAAC,CAAC;AAVW,QAAA,oBAAoB,wBAU/B"}

package/dist/lib/checkTitleAndSnippetSearchability/matchTopicInfo.d.ts

@@ -0,0 +1,36 @@
+import { IItem } from '../../types';
+import { MatchResult } from './helpers';
+/**
+ * Sets additional matching patterns for topics.
+ * The additional matching patterns and rejected patterns are not included in the TitleSummaryMatchingPatterns
+ * but fetched from the media folder from the Living Atlas server.
+ *
+ * @param {string[]} extraMatchingPatterns - Additional matching patterns to be set.
+ * @param {string[]} extraRejectedPatterns - Additional rejected patterns to be set.
+ * @param {boolean} [shouldAvoidUsingWordBoundary=false] - A boolean indicating whether to avoid using word boundaries (useful for languages like Chinese and Japanese).
+ *
+ * @remarks
+ * - For languages like Japanese that don't use spaces between words (or where word boundaries are not easily determined by spaces or punctuation),
+ *   set `shouldAvoidUsingWordBoundary` to true to remove `\b` and use lookarounds to ensure correct boundaries.
+ */
+export declare const setExtraMatchingPatternsForTopics: (extraMatchingPatterns: string[], extraRejectedPatterns: string[], shouldAvoidUsingWordBoundary?: boolean) => void;
+/**
+ * Matches topic information from an item's title and snippet using predefined or custom matching patterns.
+ *
+ * @param {IItem} item - The item containing the title and snippet to be matched.
+ * @param {string[]} [customMatchingPattenrs] - Optional custom matching patterns to be used in addition to the predefined ones.
+ * @returns {MatchResult[]} - An array of matched results for topic, deduplicated and with overlaps removed.
+ */
+export declare const matchTopicInfo: (item: IItem, customMatchingPattenrs?: string[]) => MatchResult[];
+/**
+ * Determines if a given string is a recognized topic.
+ * @param topic - The string to check.
+ * @returns {boolean} - True if the string is a recognized topic, false otherwise.
+ */
+export declare const isRecognizedTopic: (str: string) => boolean;
+/**
+ * Determines if a given string is a rejected topic.
+ * @param str - The string to check.
+ * @returns {boolean} - True if the string is a rejected topic, false otherwise.
+ */
+export declare const isRejectedTopicInfo: (str: string) => boolean;

package/dist/lib/checkTitleAndSnippetSearchability/matchTopicInfo.js

@@ -0,0 +1,175 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isRejectedTopicInfo = exports.isRecognizedTopic = exports.matchTopicInfo = exports.setExtraMatchingPatternsForTopics = void 0;
+const helpers_1 = require("./helpers");
+const TitleSummaryMatchingPatterns_Topics_json_1 = __importDefault(require("../../data/TitleSummaryMatchingPatterns_Topics.json"));
+const TitleSummaryRejectedPatterns_json_1 = __importDefault(require("../../data/TitleSummaryRejectedPatterns.json"));
+const createWordBoundaryRegex_1 = require("../util/createWordBoundaryRegex");
+/**
+ * Regular expression to match topic information in a case-insensitive manner.
+ * The pattern in TitleSummaryMatchingPatterns is already escaped.
+ */
+const regexForTopics = (0, createWordBoundaryRegex_1.createWordBoundaryRegex)(TitleSummaryMatchingPatterns_Topics_json_1.default, {
+    shouldMatchPlural: true,
+});
+// const regexForTopics = new RegExp(`\\b(${matchingPatternsForTopics})\\b`, 'gi');
+// /**
+//  * Set of recognized topics.
+//  */
+// let topicsSet: Set<string> = null; //new Set<string>(TitleSummaryMatchingPatterns.topics);
+/**
+ * Set of rejected topics.
+ */
+let rejectedTopicsSet = null; //new Set<string>(TitleSummaryMatchingPatterns.topics)
+/**
+ * Regular expression to match additional topic information.
+ */
+let regexForExtraTopics = null;
+// /**
+//  * Array of additional matching patterns for topics.
+//  */
+// let extraMatchingPatternsForTopics: string[] = [];
+// /**
+//  * Array of additional rejected patterns for topics.
+//  */
+// let extraRejectedPatternsForTopics: string[] = [];
+// /**
+//  * Creates a set of recognized topics from predefined and extra matching patterns.
+//  *
+//  * @param {string[]} extraMatchingPatternsForTopics - Additional matching patterns to be set.
+//  */
+// const createTopicsSet = (extraMatchingPatternsForTopics: string[] = []) => {
+//     topicsSet = toLowercaseSet([
+//         ...TitleSummaryMatchingPatterns.topics,
+//         ...extraMatchingPatternsForTopics,
+//     ]);
+// };
+/**
+ * Creates a set of rejected topics from predefined and extra rejected patterns.
+ *
+ * @param {string[]} extraRejectedPatternsForTopics - Additional rejected patterns to be set.
+ */
+const createRejectedTopicsSet = (extraRejectedPatternsForTopics = []) => {
+    rejectedTopicsSet = (0, helpers_1.toLowercaseSet)([
+        ...TitleSummaryRejectedPatterns_json_1.default.common,
+        ...TitleSummaryRejectedPatterns_json_1.default.topics,
+        ...extraRejectedPatternsForTopics,
+    ]);
+};
+/**
+ * Sets additional matching patterns for topics.
+ * The additional matching patterns and rejected patterns are not included in the TitleSummaryMatchingPatterns
+ * but fetched from the media folder from the Living Atlas server.
+ *
+ * @param {string[]} extraMatchingPatterns - Additional matching patterns to be set.
+ * @param {string[]} extraRejectedPatterns - Additional rejected patterns to be set.
+ * @param {boolean} [shouldAvoidUsingWordBoundary=false] - A boolean indicating whether to avoid using word boundaries (useful for languages like Chinese and Japanese).
+ *
+ * @remarks
+ * - For languages like Japanese that don't use spaces between words (or where word boundaries are not easily determined by spaces or punctuation),
+ *   set `shouldAvoidUsingWordBoundary` to true to remove `\b` and use lookarounds to ensure correct boundaries.
+ */
+const setExtraMatchingPatternsForTopics = (extraMatchingPatterns, extraRejectedPatterns, shouldAvoidUsingWordBoundary = false) => {
+    // extraMatchingPatternsForTopics = matchingPatterns || [];
+    // extraRejectedPatternsForTopics = rejectedPatterns || [];
+    if (extraMatchingPatterns && extraMatchingPatterns.length) {
+        regexForExtraTopics = (0, createWordBoundaryRegex_1.createWordBoundaryRegex)(extraMatchingPatterns, {
+            shouldAvoidUsingWordBoundary,
+            shouldMatchPlural: true,
+        });
+    }
+    // createTopicsSet(extraMatchingPatterns);
+    createRejectedTopicsSet(extraRejectedPatterns);
+};
+exports.setExtraMatchingPatternsForTopics = setExtraMatchingPatternsForTopics;
+/**
+ * Matches topic information from an item's title and snippet using predefined or custom matching patterns.
+ *
+ * @param {IItem} item - The item containing the title and snippet to be matched.
+ * @param {string[]} [customMatchingPattenrs] - Optional custom matching patterns to be used in addition to the predefined ones.
+ * @returns {MatchResult[]} - An array of matched results for topic, deduplicated and with overlaps removed.
+ */
+const matchTopicInfo = (item, customMatchingPattenrs) => {
+    // const title = item.title || '';
+    // const snippet = item.snippet || '';
+    // const matchedStrFromTitle: string[] = title.match(regexForTopics) || [];
+    // const matchedStrFromSnippet: string[] = snippet.match(regexForTopics) || [];
+    // if (!matchedStrFromTitle.length && !matchedStrFromSnippet.length) {
+    //     return matchWithCustomPatterns(item, customMatchingPattenrs);
+    // }
+    // return cleanUpMatchedResults([
+    //     ...matchedStrFromTitle,
+    //     ...matchedStrFromSnippet,
+    // ]);
+    // // match topics using patterns from the TitleSummaryMatchingPatterns
+    // let matched: string[] = matchPatternsInTitleAndSnippet(
+    //     item,
+    //     regexForTopics
+    // );
+    // // if no matches found, try matching with extra patterns if provided
+    // // the extra patterns are not included in the TitleSummaryMatchingPatterns
+    // // but fetched from the media folder from the Living Atlas server
+    // if (!matched.length && regexForExtraTopics) {
+    //     matched = matchPatternsInTitleAndSnippet(item, regexForExtraTopics);
+    // }
+    // // if still no matches found, try matching with custom patterns if provided
+    // if (!matched.length && customMatchingPattenrs?.length) {
+    //     if (!rejectedTopicsSet) {
+    //         createRejectedTopicsSet();
+    //     }
+    //     matched = matchWithCustomPatterns(
+    //         item,
+    //         customMatchingPattenrs,
+    //         rejectedTopicsSet
+    //     );
+    // }
+    // return matched;
+    if (!rejectedTopicsSet) {
+        createRejectedTopicsSet();
+    }
+    return (0, helpers_1.matchUsingMultiplePatterns)({
+        item,
+        predefinedPatterns: regexForTopics,
+        extraPatterns: regexForExtraTopics,
+        customTerms: customMatchingPattenrs,
+        rejectedSet: rejectedTopicsSet,
+    });
+};
+exports.matchTopicInfo = matchTopicInfo;
+/**
+ * Determines if a given string is a recognized topic.
+ * @param topic - The string to check.
+ * @returns {boolean} - True if the string is a recognized topic, false otherwise.
+ */
+const isRecognizedTopic = (str) => {
+    // if (!topicsSet) {
+    //     // topicsSet = toLowercaseSet([
+    //     //     ...TitleSummaryMatchingPatterns.topics,
+    //     //     ...extraMatchingPatternsForTopics,
+    //     // ]);
+    //     createTopicsSet();
+    // }
+    // return isFoundInSet(str, topicsSet);
+    const topicInfo = (0, exports.matchTopicInfo)({
+        title: str,
+        snippet: '',
+    });
+    return topicInfo.length > 0;
+};
+exports.isRecognizedTopic = isRecognizedTopic;
+/**
+ * Determines if a given string is a rejected topic.
+ * @param str - The string to check.
+ * @returns {boolean} - True if the string is a rejected topic, false otherwise.
+ */
+const isRejectedTopicInfo = (str) => {
+    if (!rejectedTopicsSet) {
+        createRejectedTopicsSet();
+    }
+    return (0, helpers_1.isFoundInSet)(str, rejectedTopicsSet);
+};
+exports.isRejectedTopicInfo = isRejectedTopicInfo;
+//# sourceMappingURL=matchTopicInfo.js.map

package/dist/lib/checkTitleAndSnippetSearchability/matchTopicInfo.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"matchTopicInfo.js","sourceRoot":"","sources":["../../../src/lib/checkTitleAndSnippetSearchability/matchTopicInfo.ts"],"names":[],"mappings":";;;;;;AACA,uCAUmB;AACnB,mIAA4F;AAC5F,qHAAwF;AACxF,6EAA0E;AAE1E;;;GAGG;AACH,MAAM,cAAc,GAAG,IAAA,iDAAuB,EAAC,kDAAyB,EAAE;IACtE,iBAAiB,EAAE,IAAI;CAC1B,CAAC,CAAC;AACH,mFAAmF;AAEnF,MAAM;AACN,+BAA+B;AAC/B,MAAM;AACN,6FAA6F;AAE7F;;GAEG;AACH,IAAI,iBAAiB,GAAgB,IAAI,CAAC,CAAC,sDAAsD;AAEjG;;GAEG;AACH,IAAI,mBAAmB,GAAW,IAAI,CAAC;AAEvC,MAAM;AACN,uDAAuD;AACvD,MAAM;AACN,qDAAqD;AACrD,MAAM;AACN,uDAAuD;AACvD,MAAM;AACN,qDAAqD;AAErD,MAAM;AACN,qFAAqF;AACrF,KAAK;AACL,gGAAgG;AAChG,MAAM;AACN,+EAA+E;AAC/E,mCAAmC;AACnC,kDAAkD;AAClD,6CAA6C;AAC7C,UAAU;AACV,KAAK;AAEL;;;;GAIG;AACH,MAAM,uBAAuB,GAAG,CAC5B,iCAA2C,EAAE,EAC/C,EAAE;IACA,iBAAiB,GAAG,IAAA,wBAAc,EAAC;QAC/B,GAAG,2CAA4B,CAAC,MAAM;QACtC,GAAG,2CAA4B,CAAC,MAAM;QACtC,GAAG,8BAA8B;KACpC,CAAC,CAAC;AACP,CAAC,CAAC;AAEF;;;;;;;;;;;;GAYG;AACI,MAAM,iCAAiC,GAAG,CAC7C,qBAA+B,EAC/B,qBAA+B,EAC/B,4BAA4B,GAAG,KAAK,EAChC,EAAE;IACN,2DAA2D;IAC3D,2DAA2D;IAE3D,IAAI,qBAAqB,IAAI,qBAAqB,CAAC,MAAM,EAAE;QACvD,mBAAmB,GAAG,IAAA,iDAAuB,EAAC,qBAAqB,EAAE;YACjE,4BAA4B;YAC5B,iBAAiB,EAAE,IAAI;SAC1B,CAAC,CAAC;KACN;IAED,0CAA0C;IAC1C,uBAAuB,CAAC,qBAAqB,CAAC,CAAC;AACnD,CAAC,CAAC;AAjBW,QAAA,iCAAiC,qCAiB5C;AAEF;;;;;;GAMG;AACI,MAAM,cAAc,GAAG,CAC1B,IAAW,EACX,sBAAiC,EACpB,EAAE;IACf,kCAAkC;IAClC,sCAAsC;IAEtC,2EAA2E;IAC3E,+EAA+E;IAE/E,sEAAsE;IACtE,oEAAoE;IACpE,IAAI;IAEJ,iCAAiC;IACjC,8BAA8B;IAC9B,gCAAgC;IAChC,MAAM;IAEN,uEAAuE;IACvE,0DAA0D;IAC1D,YAAY;IACZ,qBAAqB;IACrB,KAAK;IAEL,uEAAuE;IACvE,6EAA6E;IAC7E,oEAAoE;IACpE,gDAAgD;IAChD,2EAA2E;IAC3E,IAAI;IAEJ,8EAA8E;IAC9E,2DAA2D;IAC3D,gCAAgC;IAChC,qCAAqC;IACrC,QAAQ;IAER,yCAAyC;IACzC,gBAAgB;IAChB,kCAAkC;IAClC,4BAA4B;IAC5B,SAAS;IACT,IAAI;IAEJ,kBAAkB;IAElB,IAAI,CAAC,iBAAiB,EAAE;QACpB,uBAAuB,EAAE,CAAC;KAC7B;IAED,OAAO,IAAA,oCAA0B,EAAC;QAC9B,IAAI;QACJ,kBAAkB,EAAE,cAAc;QAClC,aAAa,EAAE,mBAAmB;QAClC,WAAW,EAAE,sBAAsB;QACnC,WAAW,EAAE,iBAAiB;KACjC,CAAC,CAAC;AACP,CAAC,CAAC;AA1DW,QAAA,cAAc,kBA0DzB;AAEF;;;;GAIG;AACI,MAAM,iBAAiB,GAAG,CAAC,GAAW,EAAW,EAAE;IACtD,oBAAoB;IACpB,sCAAsC;IACtC,qDAAqD;IACrD,gDAAgD;IAChD,aAAa;IACb,yBAAyB;IACzB,IAAI;IAEJ,uCAAuC;IAEvC,MAAM,SAAS,GAAG,IAAA,sBAAc,EAAC;QAC7B,KAAK,EAAE,GAAG;QACV,OAAO,EAAE,EAAE;KACL,CAAC,CAAC;IAEZ,OAAO,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC;AAChC,CAAC,CAAC;AAjBW,QAAA,iBAAiB,qBAiB5B;AAEF;;;;GAIG;AACI,MAAM,mBAAmB,GAAG,CAAC,GAAW,EAAW,EAAE;IACxD,IAAI,CAAC,iBAAiB,EAAE;QACpB,uBAAuB,EAAE,CAAC;KAC7B;IAED,OAAO,IAAA,sBAAY,EAAC,GAAG,EAAE,iBAAiB,CAAC,CAAC;AAChD,CAAC,CAAC;AANW,QAAA,mBAAmB,uBAM9B"}

package/dist/lib/checkTitleAndSnippetSearchability/scoringConfig.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scoringConfig.js","sourceRoot":"","sources":["../../../src/lib/checkTitleAndSnippetSearchability/scoringConfig.ts"],"names":[],"mappings":";AAAA,+CAA+C;;;AAElC,QAAA,aAAa,GAAG;IACzB,MAAM,EAAE,EAAE;IACV,aAAa,EAAE;QACX,OAAO,EAAE;YACL,sBAAsB,EAAE,IAAI;YAC5B,kBAAkB,EAAE,IAAI;YACxB,oBAAoB,EAAE,IAAI;YAC1B,mBAAmB,EAAE,IAAI;SAC5B;KACJ;CACJ,CAAC"}