@memberjunction/content-autotagging 3.4.0 → 4.1.0

package/dist/RSSFeed/generic/AutotagRSSFeed.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AutotagRSSFeed.js","sourceRoot":"","sources":["../../../src/RSSFeed/generic/AutotagRSSFeed.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAY,QAAQ,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAuB,MAAM,cAAc,CAAC;AAEtE,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,MAAM,OAAO,CAAA;AACzB,OAAO,MAAM,MAAM,QAAQ,CAAA;AAC3B,OAAO,MAAM,MAAM,YAAY,CAAA;AAC/B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;AAGvB,IAAM,cAAc,GAApB,MAAM,cAAe,SAAQ,WAAW;IAK3C;QACI,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,iBAAiB,CAAC,QAAQ,CAAC;IAC7C,CAAC;IAES,cAAc;QACpB,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,OAAO,CAAC,WAAqB;QACtC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,mBAAmB,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,UAAU,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACxG,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC1G,MAAM,qBAAqB,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,cAAc,CAAC,CAAC;QAClF,MAAM,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,qBAAqB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAC5F,CAAC;IAED;;;;;OAKG;IACI,KAAK,CAAC,wBAAwB,CAAC,cAAqC;QACvE,MAAM,qBAAqB,GAAwB,EAAE,CAAA;QACrD,KAAK,MAAM,aAAa,IAAI,cAAc,EAAE,CAAC;YAEzC,2FAA2F;YAC3F,MAAM,sBAAsB,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,aAAa,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YACzG,IAAI,sBAAsB,EAAE,CAAC;gBACzB,wDAAwD;gBACxD,sBAAsB,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;oBAC1C,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;wBACb,IAAY,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;oBAC/B,CAAC;gBACL,CAAC,CAAC,CAAA;YACN,CAAC;YAED,MAAM,mBAAmB,GAAwB;gBAC7C,eAAe,EAAE,aAAa,CAAC,EAAE;gBACjC,IAAI,EAAE,aAAa,CAAC,IAAI;gBACxB,aAAa,EAAE,aAAa,CAAC,aAAa;gBAC1C,iBAAiB,EAAE,aAAa,CAAC,iBAAiB;gBAClD,mBAAmB,EAAE,aAAa,CAAC,mBAAmB;gBACtD,GAAG,EAAE,aAAa,CAAC,GAAG;aACzB,CAAA;YAED,MAAM,WAAW,GAAc,MAAM,IAAI,CAAC,YAAY,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;YAEhF,MAAM,YAAY,GAAwB,MAAM,IAAI,CAAC,6BAA6B,CAAC,WAAW,EAAE,mBAAmB,CAAC,CAAA;YAEpH,IAAI,YAAY,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1C,qBAAqB,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;YAChD,CAAC;iBACI,CAAC;gBACF,oCAAoC;gBACpC,OAAO,CAAC,GAAG,CAAC,yDAAyD,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACtG,CAAC;QACL,CAAC;QACD,OAAO,qBAAqB,CAAA;IAChC,CAAC;IAEM,KAAK,CAAC,6BAA6B,CAAC,WAAsB,EAAE,mBAAwC;QACvG,MAAM,qBAAqB,GAAwB,EAAE,CAAC;QACtD,KAAK,MAAM,cAAc,IAAI,WAAW,EAAE,CAAC;YACvC,MAAM,EAAE,GAAG,IAAI,OAAO,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC;gBAC7B,UAAU,EAAE,eAAe;gBAC3B,WAAW,EAAE,sBAAsB,mBAAmB,CAAC,eAAe,iBAAiB,cAAc,CAAC,IAAI,uBAAuB,cAAc,CAAC,WAAW,IAAI,EAAE,qFAAqF;gBACtP,UAAU,EAAE,eAAe;aAC9B,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;YAEpB,IAAI,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC5C,MAAM,iBAAiB,GAAuB,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBACjE,uFAAuF;gBACvF,MAAM,cAAc,GAAW,iBAAiB,CAAC,QAAQ,CAAA;gBACzD,MAAM,OAAO,GAAW,MAAM,IAAI,CAAC,sBAAsB,CAAC,cAAc,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;gBAE3F,IAAI,cAAc,KAAK,OAAO,EAAE,CAAC;oBAC7B,sCAAsC;oBACtC,MAAM,EAAE,GAAG,IAAI,QAAQ,EAAE,CAAC;oBAC1B,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,eAAe,CAAoB,eAAe,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;oBACnG,WAAW,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;oBACvC,WAAW,CAAC,QAAQ,GAAG,OAAO,CAAA;oBAC9B,WAAW,CAAC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAA;oBAEjD,MAAM,WAAW,CAAC,IAAI,EAAE,CAAC;oBACzB,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,yCAAyC;gBACtF,CAAC;YACL,CAAC;iBACI,CAAC;gBACF,2CAA2C;gBAC3C,MAAM,EAAE,GAAG,IAAI,QAAQ,EAAE,CAAC;gBAC1B,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,eAAe,CAAoB,eAAe,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;gBACnG,WAAW,CAAC,eAAe,GAAG,mBAAmB,CAAC,eAAe,CAAA;gBACjE,WAAW,CAAC,IAAI,GAAG,mBAAmB,CAAC,IAAI,CAAA;gBAC3C,WAAW,CAAC,WAAW,GAAG,cAAc,CAAC,WAAW,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,yBAAyB,CAAC,mBAAmB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;gBAC1I,WAAW,CAAC,aAAa,GAAG,mBAAmB,CAAC,aAAa,CAAA;gBAC7D,WAAW,CAAC,iBAAiB,GAAG,mBAAmB,CAAC,iBAAiB,CAAA;gBACrE,WAAW,CAAC,mBAAmB,GAAG,mBAAmB,CAAC,mBAAmB,CAAA;gBACzE,WAAW,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,cAAc,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;gBAC1F,WAAW,CAAC,GAAG,GAAG,cAAc,CAAC,IAAI,IAAI,mBAAmB,CAAC,GAAG,CAAA;gBAChE,WAAW,CAAC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAA;gBAEjD,MAAM,WAAW,CAAC,IAAI,EAAE,CAAC;gBACzB,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,sCAAsC;YAEnF,CAAC;QACL,CAAC;QACD,OAAO,qBAAqB,CAAA;IAChC,CAAC;IAEM,KAAK,CAAC,YAAY,CAAC,GAAW;QACjC,IAAI,CAAC;YACD,IAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,QAAQ,GAAc,EAAE,CAAA;gBAC9B,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;gBAC5B,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;gBACxC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;gBAEzB,sEAAsE;gBACtE,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,IAAS,EAAE,EAAE;oBAC9B,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;oBAC9B,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;oBACjC,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;oBAC/B,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;oBAC7C,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;oBACrC,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;oBAC/B,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;oBACvC,MAAM,OAAO,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;oBACjE,OAAO,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;oBACvD,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;oBACnC,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;oBACvC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;oBACnC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC3B,CAAC,CAAC,CAAC;gBAEH,OAAO,QAAQ,CAAA;YACnB,CAAC;iBACI,CAAC;gBACF,MAAM,IAAI,KAAK,CAAC,gBAAgB,GAAG,EAAE,CAAC,CAAC;YAC3C,CAAC;QACL,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,0BAA0B,EAAE,KAAK,CAAC,CAAC;YACjD,OAAO,EAAE,CAAC;QACZ,CAAC;IACP,CAAC;IAES,KAAK,CAAC,UAAU,CAAC,GAAW;QAClC,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvC,OAAO,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC;QACnC,CAAC;QACD,OAAO,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,KAAK,CAAC,gBAAgB,GAAG,EAAE,CAAC,CAAC;YACrC,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,sBAAsB,CAAC,cAAuB,EAAE,WAAqB;QAC9E,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAC7F,OAAO,IAAI,CAAA;IACf,CAAC;CACJ,CAAA;AA9KY,cAAc;IAD1B,aAAa,CAAC,WAAW,EAAE,gBAAgB,CAAC;;GAChC,cAAc,CA8K1B"}

package/dist/RSSFeed/generic/RSS.types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"RSS.types.d.ts","sourceRoot":"","sources":["../../../src/RSSFeed/generic/RSS.types.ts"],"names":[],"mappings":"AAAA,qBAAa,OAAO;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB"}

package/dist/RSSFeed/generic/RSS.types.js

@@ -0,0 +1,3 @@
+export class RSSItem {
+}
+//# sourceMappingURL=RSS.types.js.map

package/dist/RSSFeed/generic/RSS.types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"RSS.types.js","sourceRoot":"","sources":["../../../src/RSSFeed/generic/RSS.types.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,OAAO;CAWnB"}

package/dist/RSSFeed/index.d.ts

@@ -0,0 +1,3 @@
+export * from './generic/RSS.types.js';
+export * from './generic/AutotagRSSFeed.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/RSSFeed/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/RSSFeed/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAA;AACnC,cAAc,0BAA0B,CAAA"}

package/dist/RSSFeed/index.js

@@ -0,0 +1,3 @@
+export * from './generic/RSS.types.js';
+export * from './generic/AutotagRSSFeed.js';
+//# sourceMappingURL=index.js.map

package/dist/RSSFeed/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/RSSFeed/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAA;AACnC,cAAc,0BAA0B,CAAA"}

package/dist/Websites/generic/AutotagWebsite.d.ts

@@ -0,0 +1,85 @@
+import { AutotagBase } from '../../Core/index.js';
+import { ContentSourceParams } from '../../Engine/index.js';
+import { UserInfo } from '@memberjunction/core';
+import { ContentSourceEntity, ContentItemEntity } from '@memberjunction/core-entities';
+import * as cheerio from 'cheerio';
+export declare class AutotagWebsite extends AutotagBase {
+    private contextUser;
+    private engine;
+    protected contentSourceTypeID: string;
+    protected CrawlOtherSitesInTopLevelDomain: boolean;
+    protected CrawlSitesInLowerLevelDomain: boolean;
+    protected MaxDepth: number;
+    protected RootURL: string;
+    protected URLPattern: string;
+    protected visitedURLs: Set<string>;
+    constructor();
+    protected getContextUser(): UserInfo;
+    /**
+     * Implemented abstract method from the AutotagBase class. that runs the entire autotagging process. This method is the entry point for the autotagging process.
+     * It initializes the connection, retrieves the content sources corresponding to the content source type, sets the content items that we want to process,
+     * extracts and processes the text, and sets the results in the database.
+     */
+    Autotag(contextUser: UserInfo): Promise<void>;
+    /**
+     * Given a content source, retrieve all content items associated with the content sources.
+     * The content items are then processed to determine if they have been modified since the last time they were processed or if they are new content items.
+     * @param contentSource
+     * @returns
+     */
+    SetContentItemsToProcess(contentSources: ContentSourceEntity[]): Promise<ContentItemEntity[]>;
+    /**
+     * Given a list of content item links, check if the content item already exists in the database.
+     * If the content item exists, check if the content item has been modified since the last time it was processed.
+     * If the content item does not exist, create a new content item and add it to the list of content items to process.
+     * @param contentItemLinks
+     * @param contentSourceParams
+     * @param contextUser
+     * @returns
+     */
+    protected SetNewAndModifiedContentItems(contentItemLinks: string[], contentSourceParams: ContentSourceParams, contextUser: UserInfo): Promise<ContentItemEntity[]>;
+    fetchPageContent(url: string): Promise<string>;
+    getTextWithLineBreaks(element: any, $: cheerio.CheerioAPI): string;
+    /**
+     * Given a URL, this function extracts text from a webpage.
+     * @param url
+     * @returns The text extracted from the webpage
+     */
+    parseWebPage(url: string): Promise<string>;
+    /**
+     * Given a root URL that corresponds to a content source, retrieve all the links in accordance to the crawl settings.
+     * If the crawl settings are set to crawl other sites in the top level domain, then all links in the top level domain will be retrieved.
+     * If the crawl settings are set to crawl sites in lower level domains, then function is recursively called to retrieve all links in the lower level domains.
+     * @param url
+     * @returns
+     */
+    protected getAllLinksFromContentSource(url: string, rootURL: string, regex: RegExp): Promise<string[]>;
+    /**
+     * For a given URL, retrieves all other links at that top level domain.
+     * @param url
+     * @param rootURL
+     * @param visitedURLs
+     * @returns
+     */
+    protected getTopLevelLinks(url: string, rootURL: string): Promise<void>;
+    /**
+     * Simple check to see if the URL is at the highest level domain.
+     * @param url
+     * @returns
+     */
+    protected isHighestDomain(url: string): boolean;
+    protected getBasePath(url: string): string;
+    protected getPathName(url: string): string;
+    protected urlIsValid(url: string): Promise<boolean>;
+    /**
+     * For a given URL, retrieves all links at lower level domains up to the specified crawl depth.
+     * @param url
+     * @param rootURL
+     * @param crawlDepth
+     * @param visitedURLs
+     * @returns
+     */
+    protected getLowerLevelLinks(url: string, rootURL: string, crawlDepth: number, scrapedURLs: Set<string>, regex: RegExp): Promise<Set<string>>;
+    protected delay(ms: number): Promise<unknown>;
+}
+//# sourceMappingURL=AutotagWebsite.d.ts.map

package/dist/Websites/generic/AutotagWebsite.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"AutotagWebsite.d.ts","sourceRoot":"","sources":["../../../src/Websites/generic/AutotagWebsite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAqB,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAEtE,OAAO,EAAE,QAAQ,EAAqB,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AACvF,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAMnC,qBACa,cAAe,SAAQ,WAAW;IAC3C,OAAO,CAAC,WAAW,CAAW;IAC9B,OAAO,CAAC,MAAM,CAAoB;IAClC,SAAS,CAAC,mBAAmB,EAAE,MAAM,CAAA;IACrC,SAAS,CAAC,+BAA+B,EAAE,OAAO,CAAC;IACnD,SAAS,CAAC,4BAA4B,EAAE,OAAO,CAAC;IAChD,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC;IAC7B,SAAS,CAAC,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;;IAQnC,SAAS,CAAC,cAAc,IAAI,QAAQ;IAIpC;;;;OAIG;IACU,OAAO,CAAC,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAS1D;;;;;OAKG;IACU,wBAAwB,CAAC,cAAc,EAAE,mBAAmB,EAAE,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;IAoD1G;;;;;;;;OAQG;cACa,6BAA6B,CAAC,gBAAgB,EAAE,MAAM,EAAE,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;IAsE3J,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAKpD,qBAAqB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,CAAC,UAAU,GAAG,MAAM;IAgBzE;;;;OAIG;IACU,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAavD;;;;;;OAMG;cACa,4BAA4B,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAa5G;;;;;;OAMG;cACa,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAmC7E;;;;OAIG;IACH,SAAS,CAAC,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAW/C,SAAS,CAAC,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAW1C,SAAS,CAAC,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;cAa1B,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAWzD;;;;;;;OAOG;cACa,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;cA+CnI,KAAK,CAAC,EAAE,EAAE,MAAM;CAGnC"}

package/dist/{src/Websites → Websites}/generic/AutotagWebsite.js

@@ -1,68 +1,35 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
 var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
     var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
     if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
     else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
     return c > 3 && r && Object.defineProperty(target, key, r), r;
 };
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
 var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.AutotagWebsite = void 0;
-const Core_1 = require("../../Core");
-const Engine_1 = require("../../Engine");
-const global_1 = require("@memberjunction/global");
-const core_1 = require("@memberjunction/core");
-const cheerio = __importStar(require("cheerio"));
-const axios_1 = __importDefault(require("axios"));
-const url_1 = require("url");
-const dotenv_1 = __importDefault(require("dotenv"));
-dotenv_1.default.config();
-let AutotagWebsite = class AutotagWebsite extends Core_1.AutotagBase {
-    contextUser;
-    engine;
-    contentSourceTypeID;
-    CrawlOtherSitesInTopLevelDomain;
-    CrawlSitesInLowerLevelDomain;
-    MaxDepth;
-    RootURL;
-    URLPattern;
-    visitedURLs;
+import { AutotagBase } from '../../Core/index.js';
+import { AutotagBaseEngine } from '../../Engine/index.js';
+import { RegisterClass } from '@memberjunction/global';
+import { Metadata, RunView } from '@memberjunction/core';
+import * as cheerio from 'cheerio';
+import axios from 'axios';
+import { URL } from 'url';
+import dotenv from 'dotenv';
+dotenv.config({ quiet: true });
+let AutotagWebsite = class AutotagWebsite extends AutotagBase {
     constructor() {
         super();
-        this.engine = Engine_1.AutotagBaseEngine.Instance;
+        this.engine = AutotagBaseEngine.Instance;
         this.visitedURLs = new Set();
     }
     getContextUser() {
         return this.contextUser;
     }
+    /**
+     * Implemented abstract method from the AutotagBase class. that runs the entire autotagging process. This method is the entry point for the autotagging process.
+     * It initializes the connection, retrieves the content sources corresponding to the content source type, sets the content items that we want to process,
+     * extracts and processes the text, and sets the results in the database.
+     */
     async Autotag(contextUser) {
         this.contextUser = contextUser;
         this.contentSourceTypeID = await this.engine.setSubclassContentSourceType('Website', this.contextUser);
@@ -70,11 +37,19 @@ let AutotagWebsite = class AutotagWebsite extends Core_1.AutotagBase {
         const contentItemsToProcess = await this.SetContentItemsToProcess(contentSources);
         await this.engine.ExtractTextAndProcessWithLLM(contentItemsToProcess, this.contextUser);
     }
+    /**
+     * Given a content source, retrieve all content items associated with the content sources.
+     * The content items are then processed to determine if they have been modified since the last time they were processed or if they are new content items.
+     * @param contentSource
+     * @returns
+     */
     async SetContentItemsToProcess(contentSources) {
         const contentItemsToProcess = [];
+        // If content source parameters were provided, set them. Otherwise, use the default values.
         for (const contentSource of contentSources) {
             const contentSourceParamsMap = await this.engine.getContentSourceParams(contentSource, this.contextUser);
             if (contentSourceParamsMap) {
+                // Override defaults with content source specific params
                 contentSourceParamsMap.forEach((value, key) => {
                     if (key in this) {
                         this[key] = value;
@@ -90,8 +65,11 @@ let AutotagWebsite = class AutotagWebsite extends Core_1.AutotagBase {
                 URL: contentSource.URL
             };
             try {
+                // All content items associated with the content source
                 const startURL = contentSourceParams.URL;
+                // root url should be set to this.RootURL if it exists, otherwise it should be set to the base path of the startURL. 
                 const rootURL = this.RootURL ? this.RootURL : this.getBasePath(startURL);
+                // regex should be set to this.URLPattern if it exists, otherwise it should be set to match any URL.
                 const regex = this.URLPattern && new RegExp(this.URLPattern) || new RegExp('.*');
                 const allContentItemLinks = await this.getAllLinksFromContentSource(startURL, rootURL, regex);
                 const contentItems = await this.SetNewAndModifiedContentItems(allContentItemLinks, contentSourceParams, this.contextUser);
@@ -99,6 +77,7 @@ let AutotagWebsite = class AutotagWebsite extends Core_1.AutotagBase {
                     contentItemsToProcess.push(...contentItems);
                 }
                 else {
+                    // No content items found to process
                     console.log(`No content items found to process for content source: ${contentSource.Get('Name')}`);
                 }
             }
@@ -108,12 +87,21 @@ let AutotagWebsite = class AutotagWebsite extends Core_1.AutotagBase {
         }
         return contentItemsToProcess;
     }
+    /**
+     * Given a list of content item links, check if the content item already exists in the database.
+     * If the content item exists, check if the content item has been modified since the last time it was processed.
+     * If the content item does not exist, create a new content item and add it to the list of content items to process.
+     * @param contentItemLinks
+     * @param contentSourceParams
+     * @param contextUser
+     * @returns
+     */
     async SetNewAndModifiedContentItems(contentItemLinks, contentSourceParams, contextUser) {
         const addedContentItems = [];
         for (const contentItemLink of contentItemLinks) {
             try {
                 const newHash = await this.engine.getChecksumFromURL(contentItemLink);
-                const rv = new core_1.RunView();
+                const rv = new RunView();
                 const results = await rv.RunViews([
                     {
                         EntityName: 'Content Items',
@@ -129,26 +117,30 @@ let AutotagWebsite = class AutotagWebsite extends Core_1.AutotagBase {
                 const contentItemResultsWithChecksum = results[0];
                 const contentItemResultsWithURL = results[1];
                 if (contentItemResultsWithChecksum.Success && contentItemResultsWithChecksum.Results.length) {
+                    // We found the checksum so this content item has not changed since we last accessed it, do nothing
                     continue;
                 }
                 else if (contentItemResultsWithURL.Success && contentItemResultsWithURL.Results.length) {
+                    // This content item already exists, update the hash and last updated date
                     const contentItemResult = contentItemResultsWithURL.Results[0];
                     const lastStoredHash = contentItemResult.Checksum;
                     if (lastStoredHash !== newHash) {
-                        const md = new core_1.Metadata();
+                        // This content item has changed since we last access it, update the hash and last updated date
+                        const md = new Metadata();
                         const contentItem = await md.GetEntityObject('Content Items', this.contextUser);
                         contentItem.Load(contentItemResult.ID);
                         contentItem.Checksum = newHash;
                         contentItem.Text = await this.parseWebPage(contentItemLink);
                         await contentItem.Save();
-                        addedContentItems.push(contentItem);
+                        addedContentItems.push(contentItem); // Content item was modified, add to list
                     }
                 }
                 else {
-                    const md = new core_1.Metadata();
+                    // This content item does not exist, add it
+                    const md = new Metadata();
                     const contentItem = await md.GetEntityObject('Content Items', this.contextUser);
                     contentItem.ContentSourceID = contentSourceParams.contentSourceID;
-                    contentItem.Name = this.getPathName(contentItemLink);
+                    contentItem.Name = this.getPathName(contentItemLink); // Will get overwritten by title later if it exists
                     contentItem.Description = await this.engine.getContentItemDescription(contentSourceParams, this.contextUser);
                     contentItem.ContentTypeID = contentSourceParams.ContentTypeID;
                     contentItem.ContentFileTypeID = contentSourceParams.ContentFileTypeID;
@@ -157,7 +149,7 @@ let AutotagWebsite = class AutotagWebsite extends Core_1.AutotagBase {
                     contentItem.URL = contentItemLink;
                     contentItem.Text = await this.parseWebPage(contentItemLink);
                     await contentItem.Save();
-                    addedContentItems.push(contentItem);
+                    addedContentItems.push(contentItem); // Content item was added, add to list
                 }
             }
             catch (e) {
@@ -167,7 +159,7 @@ let AutotagWebsite = class AutotagWebsite extends Core_1.AutotagBase {
         return addedContentItems;
     }
     async fetchPageContent(url) {
-        const { data } = await axios_1.default.get(url);
+        const { data } = await axios.get(url);
         return data;
     }
     getTextWithLineBreaks(element, $) {
@@ -184,6 +176,11 @@ let AutotagWebsite = class AutotagWebsite extends Core_1.AutotagBase {
         }
         return text;
     }
+    /**
+     * Given a URL, this function extracts text from a webpage.
+     * @param url
+     * @returns The text extracted from the webpage
+     */
     async parseWebPage(url) {
         try {
             const pageContent = await this.fetchPageContent(url);
@@ -196,6 +193,13 @@ let AutotagWebsite = class AutotagWebsite extends Core_1.AutotagBase {
             return '';
         }
     }
+    /**
+     * Given a root URL that corresponds to a content source, retrieve all the links in accordance to the crawl settings.
+     * If the crawl settings are set to crawl other sites in the top level domain, then all links in the top level domain will be retrieved.
+     * If the crawl settings are set to crawl sites in lower level domains, then function is recursively called to retrieve all links in the lower level domains.
+     * @param url
+     * @returns
+     */
     async getAllLinksFromContentSource(url, rootURL, regex) {
         try {
             await this.getLowerLevelLinks(url, rootURL, this.MaxDepth, new Set(), regex);
@@ -207,37 +211,51 @@ let AutotagWebsite = class AutotagWebsite extends Core_1.AutotagBase {
             return [];
         }
     }
+    /**
+     * For a given URL, retrieves all other links at that top level domain.
+     * @param url
+     * @param rootURL
+     * @param visitedURLs
+     * @returns
+     */
     async getTopLevelLinks(url, rootURL) {
         if (!this.CrawlOtherSitesInTopLevelDomain) {
             this.visitedURLs.add(url);
             return;
         }
+        // If we have already visited this URL, return an empty array
         if (this.visitedURLs.has(url) || !await this.urlIsValid(url) || this.isHighestDomain(url)) {
             return;
         }
         this.visitedURLs.add(url);
         try {
-            const { data } = await axios_1.default.get(url);
+            const { data } = await axios.get(url);
             const $ = cheerio.load(data);
+            // Get all links on the page for the current URL
             $('a').each((_, element) => {
                 const link = $(element).attr('href');
                 if (link) {
-                    const newURL = new url_1.URL(link, url).href;
+                    const newURL = new URL(link, url).href;
                     if (newURL.startsWith(rootURL) && !this.visitedURLs.has(newURL)) {
                         this.visitedURLs.add(newURL);
                     }
                 }
             });
-            await this.delay(1000);
+            await this.delay(1000); // Delay to prevent rate limiting
         }
         catch (e) {
             console.error(`Failed to get links from ${url}`);
             return;
         }
     }
+    /**
+     * Simple check to see if the URL is at the highest level domain.
+     * @param url
+     * @returns
+     */
     isHighestDomain(url) {
         try {
-            const parsedURL = new url_1.URL(url);
+            const parsedURL = new URL(url);
             return parsedURL.pathname === '/' || parsedURL.pathname === '';
         }
         catch (e) {
@@ -246,17 +264,18 @@ let AutotagWebsite = class AutotagWebsite extends Core_1.AutotagBase {
         }
     }
     getBasePath(url) {
-        const parsedURL = new url_1.URL(url);
+        const parsedURL = new URL(url);
         const pathSegments = parsedURL.pathname.split('/').filter(segment => segment);
         if (pathSegments.length > 0) {
-            pathSegments.pop();
+            pathSegments.pop(); //Remove last segment so that we are in the same level domain
         }
         const basePath = parsedURL.origin + '/' + pathSegments.join('/');
         return basePath;
     }
+    // Creates a URL from input string and returns the path name in the form abc.com/xyz
     getPathName(url) {
         try {
-            const parsedURL = new url_1.URL(url);
+            const parsedURL = new URL(url);
             const pathSegments = parsedURL.pathname.split('/').filter(segment => segment);
             const path = parsedURL.origin + '/' + pathSegments.join('/');
             return path;
@@ -268,7 +287,7 @@ let AutotagWebsite = class AutotagWebsite extends Core_1.AutotagBase {
     }
     async urlIsValid(url) {
         try {
-            const response = await axios_1.default.head(url);
+            const response = await axios.head(url);
             return response.status === 200;
         }
         catch (e) {
@@ -276,32 +295,44 @@ let AutotagWebsite = class AutotagWebsite extends Core_1.AutotagBase {
             return false;
         }
     }
+    /**
+     * For a given URL, retrieves all links at lower level domains up to the specified crawl depth.
+     * @param url
+     * @param rootURL
+     * @param crawlDepth
+     * @param visitedURLs
+     * @returns
+     */
     async getLowerLevelLinks(url, rootURL, crawlDepth, scrapedURLs, regex) {
         try {
             console.log(`Scraping ${url}`);
+            // If we have already visited this URL, return an empty array
             if (scrapedURLs.has(url) || await this.urlIsValid(url) === false || crawlDepth < 0 || !this.CrawlSitesInLowerLevelDomain) {
                 return new Set();
             }
-            let combinedLinks = new Set();
-            const extractedLinks = new Set();
-            const { data } = await axios_1.default.get(url);
+            let combinedLinks = new Set(); // Combined links from the current URL and all lower level URLs
+            const extractedLinks = new Set(); // Links extracted from the input URL
+            const { data } = await axios.get(url);
             const $ = cheerio.load(data);
+            // Get all links on the page for the current URL
             $('a').each((_, element) => {
                 const link = $(element).attr('href');
                 if (link) {
-                    const newURL = new url_1.URL(link, url).href;
+                    const newURL = new URL(link, url).href;
                     if (newURL.startsWith(rootURL) && newURL !== url && !this.visitedURLs.has(newURL) && regex.test(newURL)) {
                         extractedLinks.add(newURL);
                         this.visitedURLs.add(newURL);
                     }
                 }
             });
-            await this.delay(1000);
+            await this.delay(1000); // Delay to prevent rate limiting
             scrapedURLs.add(url);
+            // If we are at the depth limit, return the current set of URLs and don't recurse
             if (crawlDepth === 0) {
                 return extractedLinks;
             }
             for (const subLink of extractedLinks) {
+                //console.log(`Adding ${subLink}`);
                 const lowerLevelLinks = await this.getLowerLevelLinks(subLink, rootURL, crawlDepth - 1, scrapedURLs, regex);
                 combinedLinks = new Set([...extractedLinks, ...lowerLevelLinks]);
             }
@@ -316,9 +347,9 @@ let AutotagWebsite = class AutotagWebsite extends Core_1.AutotagBase {
         return new Promise(resolve => setTimeout(resolve, ms));
     }
 };
-exports.AutotagWebsite = AutotagWebsite;
-exports.AutotagWebsite = AutotagWebsite = __decorate([
-    (0, global_1.RegisterClass)(Core_1.AutotagBase, 'AutotagWebsite'),
+AutotagWebsite = __decorate([
+    RegisterClass(AutotagBase, 'AutotagWebsite'),
     __metadata("design:paramtypes", [])
 ], AutotagWebsite);
+export { AutotagWebsite };
 //# sourceMappingURL=AutotagWebsite.js.map

package/dist/Websites/generic/AutotagWebsite.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AutotagWebsite.js","sourceRoot":"","sources":["../../../src/Websites/generic/AutotagWebsite.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAuB,MAAM,cAAc,CAAC;AACtE,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAY,QAAQ,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAEnE,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAC1B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;AAGvB,IAAM,cAAc,GAApB,MAAM,cAAe,SAAQ,WAAW;IAW3C;QACI,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,iBAAiB,CAAC,QAAQ,CAAC;QACzC,IAAI,CAAC,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACzC,CAAC;IAES,cAAc;QACpB,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,OAAO,CAAC,WAAqB;QACtC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,mBAAmB,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACvG,MAAM,cAAc,GAA0B,MAAM,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;QACjI,MAAM,qBAAqB,GAAwB,MAAM,IAAI,CAAC,wBAAwB,CAAC,cAAc,CAAC,CAAC;QACvG,MAAM,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,qBAAqB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAC5F,CAAC;IAGD;;;;;OAKG;IACI,KAAK,CAAC,wBAAwB,CAAC,cAAqC;QACvE,MAAM,qBAAqB,GAAwB,EAAE,CAAA;QAErD,2FAA2F;QAC3F,KAAK,MAAM,aAAa,IAAI,cAAc,EAAE,CAAC;YACzC,MAAM,sBAAsB,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,aAAa,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YACzG,IAAI,sBAAsB,EAAE,CAAC;gBACzB,wDAAwD;gBACxD,sBAAsB,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;oBAC1C,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;wBACb,IAAY,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;oBAC/B,CAAC;gBACL,CAAC,CAAC,CAAA;YACN,CAAC;YAED,MAAM,mBAAmB,GAAwB;gBAC7C,eAAe,EAAE,aAAa,CAAC,EAAE;gBACjC,IAAI,EAAE,aAAa,CAAC,IAAI;gBACxB,aAAa,EAAE,aAAa,CAAC,aAAa;gBAC1C,iBAAiB,EAAE,aAAa,CAAC,iBAAiB;gBAClD,mBAAmB,EAAE,aAAa,CAAC,mBAAmB;gBACtD,GAAG,EAAE,aAAa,CAAC,GAAG;aACzB,CAAA;YAED,IAAI,CAAC;gBAED,uDAAuD;gBACvD,MAAM,QAAQ,GAAW,mBAAmB,CAAC,GAAG,CAAC;gBAEjD,qHAAqH;gBACrH,MAAM,OAAO,GAAW,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBAEjF,oGAAoG;gBACpG,MAAM,KAAK,GAAW,IAAI,CAAC,UAAU,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC;gBAEzF,MAAM,mBAAmB,GAAa,MAAM,IAAI,CAAC,4BAA4B,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;gBACxG,MAAM,YAAY,GAAwB,MAAM,IAAI,CAAC,6BAA6B,CAAC,mBAAmB,EAAE,mBAAmB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;gBAC/I,IAAI,YAAY,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC1C,qBAAqB,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;gBAChD,CAAC;qBACI,CAAC;oBACF,oCAAoC;oBACpC,OAAO,CAAC,GAAG,CAAC,yDAAyD,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBACtG,CAAC;YACL,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,OAAO,CAAC,KAAK,CAAC,qCAAqC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACpF,CAAC;QACL,CAAC;QAED,OAAO,qBAAqB,CAAC;IACjC,CAAC;IAED;;;;;;;;OAQG;IACO,KAAK,CAAC,6BAA6B,CAAC,gBAA0B,EAAE,mBAAwC,EAAE,WAAqB;QAErI,MAAM,iBAAiB,GAAwB,EAAE,CAAC;QAClD,KAAK,MAAM,eAAe,IAAI,gBAAgB,EAAE,CAAC;YAC7C,IAAI,CAAC;gBACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;gBAEtE,MAAM,EAAE,GAAG,IAAI,OAAO,EAAE,CAAC;gBACzB,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAoB;oBACjD;wBACI,UAAU,EAAE,eAAe;wBAC3B,WAAW,EAAE,eAAe,OAAO,GAAG;wBACtC,UAAU,EAAE,eAAe;qBAC9B;oBACD;wBACI,UAAU,EAAE,eAAe;wBAC3B,WAAW,EAAE,sBAAsB,mBAAmB,CAAC,eAAe,gBAAgB,eAAe,GAAG;wBACxG,UAAU,EAAE,eAAe;qBAC9B;iBACJ,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;gBAEpB,MAAM,8BAA8B,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;gBACjD,MAAM,yBAAyB,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;gBAE5C,IAAI,8BAA8B,CAAC,OAAO,IAAI,8BAA8B,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;oBAC1F,mGAAmG;oBACnG,SAAS;gBACb,CAAC;qBAEI,IAAI,yBAAyB,CAAC,OAAO,IAAI,yBAAyB,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;oBACrF,0EAA0E;oBAC1E,MAAM,iBAAiB,GAAsB,yBAAyB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;oBAClF,MAAM,cAAc,GAAW,iBAAiB,CAAC,QAAQ,CAAA;oBAEzD,IAAI,cAAc,KAAK,OAAO,EAAE,CAAC;wBAC7B,+FAA+F;wBAC/F,MAAM,EAAE,GAAG,IAAI,QAAQ,EAAE,CAAC;wBAC1B,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,eAAe,CAAoB,eAAe,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;wBACnG,WAAW,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;wBACvC,WAAW,CAAC,QAAQ,GAAG,OAAO,CAAA;wBAC9B,WAAW,CAAC,IAAI,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,eAAe,CAAC,CAAA;wBAE3D,MAAM,WAAW,CAAC,IAAI,EAAE,CAAC;wBACzB,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,yCAAyC;oBAClF,CAAC;gBACL,CAAC;qBACI,CAAC;oBACF,2CAA2C;oBAC3C,MAAM,EAAE,GAAG,IAAI,QAAQ,EAAE,CAAC;oBAC1B,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,eAAe,CAAoB,eAAe,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;oBACnG,WAAW,CAAC,eAAe,GAAG,mBAAmB,CAAC,eAAe,CAAA;oBACjE,WAAW,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,CAAA,CAAC,mDAAmD;oBACxG,WAAW,CAAC,WAAW,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,yBAAyB,CAAC,mBAAmB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;oBAC5G,WAAW,CAAC,aAAa,GAAG,mBAAmB,CAAC,aAAa,CAAA;oBAC7D,WAAW,CAAC,iBAAiB,GAAG,mBAAmB,CAAC,iBAAiB,CAAA;oBACrE,WAAW,CAAC,mBAAmB,GAAG,mBAAmB,CAAC,mBAAmB,CAAA;oBACzE,WAAW,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAA;oBAC5E,WAAW,CAAC,GAAG,GAAG,eAAe,CAAA;oBACjC,WAAW,CAAC,IAAI,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,eAAe,CAAC,CAAA;oBAE3D,MAAM,WAAW,CAAC,IAAI,EAAE,CAAC;oBACzB,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,sCAAsC;gBAC/E,CAAC;YACD,CAAC;YAAA,OAAO,CAAC,EAAE,CAAC;gBACR,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YAClB,CAAC;QACT,CAAC;QACD,OAAO,iBAAiB,CAAC;IAC7B,CAAC;IAEM,KAAK,CAAC,gBAAgB,CAAC,GAAW;QACrC,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACtC,OAAO,IAAI,CAAC;IAChB,CAAC;IAEM,qBAAqB,CAAC,OAAY,EAAE,CAAqB;QAC5D,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;QAEvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;YACvB,IAAI,EAAE,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACrB,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC;YACtC,CAAC;iBAAM,IAAI,EAAE,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;gBAC3B,IAAI,IAAI,IAAI,GAAG,IAAI,CAAC,qBAAqB,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC;YAC5D,CAAC;QACL,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,YAAY,CAAC,GAAW;QACjC,IAAI,CAAC;YACD,MAAM,WAAW,GAAW,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACpC,MAAM,IAAI,GAAW,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACjE,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,oBAAoB,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;YACjD,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IAED;;;;;;OAMG;IACO,KAAK,CAAC,4BAA4B,CAAC,GAAW,EAAE,OAAe,EAAE,KAAa;QAEpF,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,GAAG,EAAU,EAAE,KAAK,CAAC,CAAC;YACrF,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC;YAExD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACxC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,OAAO,CAAC,KAAK,CAAC,4BAA4B,GAAG,EAAE,CAAC,CAAC;YACjD,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IAED;;;;;;OAMG;IACO,KAAK,CAAC,gBAAgB,CAAC,GAAW,EAAE,OAAe;QACzD,IAAI,CAAC,IAAI,CAAC,+BAA+B,EAAE,CAAC;YACxC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC1B,OAAM;QACV,CAAC;QAED,6DAA6D;QAC7D,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC;YACxF,OAAM;QACV,CAAC;QAED,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAE1B,IAAI,CAAC;YACD,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAE7B,gDAAgD;YAChD,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;gBACvB,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrC,IAAI,IAAI,EAAE,CAAC;oBACP,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC;oBACvC,IAAI,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;wBAC9D,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBACjC,CAAC;gBACL,CAAC;YACL,CAAC,CAAC,CAAC;YACH,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,iCAAiC;QAC7D,CAAC;QACD,OAAO,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,KAAK,CAAC,4BAA4B,GAAG,EAAE,CAAC,CAAC;YACjD,OAAM;QACV,CAAC;IACL,CAAC;IAED;;;;OAIG;IACO,eAAe,CAAC,GAAW;QACjC,IAAI,CAAC;YACD,MAAM,SAAS,GAAQ,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YACpC,OAAO,SAAS,CAAC,QAAQ,KAAK,GAAG,IAAI,SAAS,CAAC,QAAQ,KAAK,EAAE,CAAC;QACnE,CAAC;QACD,OAAO,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,KAAK,CAAC,uCAAuC,GAAG,EAAE,CAAC,CAAC;YAC5D,MAAM,CAAC,CAAC;QACZ,CAAC;IACL,CAAC;IAES,WAAW,CAAC,GAAW;QAC7B,MAAM,SAAS,GAAQ,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACpC,MAAM,YAAY,GAAa,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;QACxF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,YAAY,CAAC,GAAG,EAAE,CAAC,CAAC,6DAA6D;QACrF,CAAC;QACD,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,GAAG,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjE,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,oFAAoF;IAC1E,WAAW,CAAC,GAAW;QAC7B,IAAI,CAAC;YACD,MAAM,SAAS,GAAQ,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YACpC,MAAM,YAAY,GAAa,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;YACxF,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,GAAG,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC7D,OAAO,IAAI,CAAA;QACf,CAAC;QACD,OAAO,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,KAAK,CAAC,uCAAuC,GAAG,EAAE,CAAC,CAAC;YAC5D,MAAM,CAAC,CAAC;QACZ,CAAC;IACL,CAAC;IAES,KAAK,CAAC,UAAU,CAAC,GAAW;QAClC,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvC,OAAO,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC;QACnC,CAAC;QACD,OAAO,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,KAAK,CAAC,gBAAgB,GAAG,EAAE,CAAC,CAAC;YACrC,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;IAED;;;;;;;OAOG;IACO,KAAK,CAAC,kBAAkB,CAAC,GAAW,EAAE,OAAe,EAAE,UAAkB,EAAE,WAAwB,EAAE,KAAa;QAExH,IAAI,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,YAAY,GAAG,EAAE,CAAC,CAAC;YAC/B,6DAA6D;YAC7D,IAAI,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,KAAK,IAAI,UAAU,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,4BAA4B,EAAE,CAAC;gBACvH,OAAO,IAAI,GAAG,EAAU,CAAC;YAC7B,CAAC;YAED,IAAI,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC,CAAC,+DAA+D;YACtG,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC,CAAC,qCAAqC;YAE/E,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAE7B,gDAAgD;YAChD,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;gBACvB,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrC,IAAI,IAAI,EAAE,CAAC;oBACP,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC;oBACvC,IAAI,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;wBACtG,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;wBAC3B,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBACjC,CAAC;gBACL,CAAC;YACL,CAAC,CAAC,CAAC;YACH,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,iCAAiC;YACzD,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAErB,iFAAiF;YACjF,IAAI,UAAU,KAAK,CAAC,EAAE,CAAC;gBACnB,OAAO,cAAc,CAAC;YAC1B,CAAC;YAED,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;gBACnC,mCAAmC;gBACnC,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,GAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;gBAC1G,aAAa,GAAG,IAAI,GAAG,CAAS,CAAC,GAAG,cAAc,EAAE,GAAG,eAAe,CAAC,CAAC,CAAC;YAC7E,CAAC;YACD,OAAO,aAAa,CAAC;QACzB,CAAC;QACD,OAAO,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,KAAK,CAAC,4BAA4B,GAAG,EAAE,CAAC,CAAC;YACjD,OAAO,IAAI,GAAG,EAAU,CAAC;QAC7B,CAAC;IACL,CAAC;IAES,KAAK,CAAC,KAAK,CAAC,EAAU;QAC5B,OAAO,IAAI,OAAO,CAAE,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAE,CAAC;IAC7D,CAAC;CACJ,CAAA;AA9XY,cAAc;IAD1B,aAAa,CAAC,WAAW,EAAE,gBAAgB,CAAC;;GAChC,cAAc,CA8X1B"}

package/dist/Websites/index.d.ts

@@ -0,0 +1,2 @@
+export * from './generic/AutotagWebsite.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/Websites/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/Websites/index.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}

package/dist/Websites/index.js

@@ -0,0 +1,2 @@
+export * from './generic/AutotagWebsite.js';
+//# sourceMappingURL=index.js.map

package/dist/Websites/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/Websites/index.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}

package/dist/index.d.ts

@@ -0,0 +1,7 @@
+export * from './Core/index.js';
+export * from './Engine/index.js';
+export * from './LocalFileSystem/index.js';
+export * from './RSSFeed/index.js';
+export * from './Websites/index.js';
+export * from './CloudStorage/index.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,QAAQ,CAAC;AACvB,cAAc,UAAU,CAAC;AACzB,cAAc,mBAAmB,CAAC;AAClC,cAAc,WAAW,CAAC;AAC1B,cAAc,YAAY,CAAC;AAC3B,cAAc,gBAAgB,CAAA"}

package/dist/index.js

@@ -0,0 +1,7 @@
+export * from './Core/index.js';
+export * from './Engine/index.js';
+export * from './LocalFileSystem/index.js';
+export * from './RSSFeed/index.js';
+export * from './Websites/index.js';
+export * from './CloudStorage/index.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,QAAQ,CAAC;AACvB,cAAc,UAAU,CAAC;AACzB,cAAc,mBAAmB,CAAC;AAClC,cAAc,WAAW,CAAC;AAC1B,cAAc,YAAY,CAAC;AAC3B,cAAc,gBAAgB,CAAA"}