@memberjunction/content-autotagging 3.3.0 → 4.0.0

package/dist/RSSFeed/generic/AutotagRSSFeed.js

@@ -0,0 +1,177 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+import { Metadata, RunView } from '@memberjunction/core';
+import { RegisterClass } from '@memberjunction/global';
+import { AutotagBase } from "../../Core/index.js";
+import { AutotagBaseEngine } from "../../Engine/index.js";
+import { RSSItem } from './RSS.types.js';
+import axios from 'axios';
+import crypto from 'crypto';
+import Parser from 'rss-parser';
+import dotenv from 'dotenv';
+dotenv.config({ quiet: true });
+let AutotagRSSFeed = class AutotagRSSFeed extends AutotagBase {
+    constructor() {
+        super();
+        this.engine = AutotagBaseEngine.Instance;
+    }
+    getContextUser() {
+        return this.contextUser;
+    }
+    /**
+     * Implemented abstract method from the AutotagBase class. that runs the entire autotagging process. This method is the entry point for the autotagging process.
+     * It initializes the connection, retrieves the content sources corresponding to the content source type, sets the content items that we want to process,
+     * extracts and processes the text, and sets the results in the database.
+     */
+    async Autotag(contextUser) {
+        this.contextUser = contextUser;
+        this.contentSourceTypeID = await this.engine.setSubclassContentSourceType('RSS Feed', this.contextUser);
+        const contentSources = await this.engine.getAllContentSources(this.contextUser, this.contentSourceTypeID);
+        const contentItemsToProcess = await this.SetContentItemsToProcess(contentSources);
+        await this.engine.ExtractTextAndProcessWithLLM(contentItemsToProcess, this.contextUser);
+    }
+    /**
+     * Implemented abstract method from the AutotagBase class. Given a list of content sources, this method should return a list
+     * of content source items that have been modified or added after the most recent process run for that content source.
+     * @param contentSources - An array of content sources to check for modified or added content source items
+     * @returns - An array of content source items that have been modified or added after the most recent process run for that content source
+     */
+    async SetContentItemsToProcess(contentSources) {
+        const contentItemsToProcess = [];
+        for (const contentSource of contentSources) {
+            // If content source parameters were provided, set them. Otherwise, use the default values.
+            const contentSourceParamsMap = await this.engine.getContentSourceParams(contentSource, this.contextUser);
+            if (contentSourceParamsMap) {
+                // Override defaults with content source specific params
+                contentSourceParamsMap.forEach((value, key) => {
+                    if (key in this) {
+                        this[key] = value;
+                    }
+                });
+            }
+            const contentSourceParams = {
+                contentSourceID: contentSource.ID,
+                name: contentSource.Name,
+                ContentTypeID: contentSource.ContentTypeID,
+                ContentFileTypeID: contentSource.ContentFileTypeID,
+                ContentSourceTypeID: contentSource.ContentSourceTypeID,
+                URL: contentSource.URL
+            };
+            const allRSSItems = await this.parseRSSFeed(contentSourceParams.URL);
+            const contentItems = await this.SetNewAndModifiedContentItems(allRSSItems, contentSourceParams);
+            if (contentItems && contentItems.length > 0) {
+                contentItemsToProcess.push(...contentItems);
+            }
+            else {
+                // No content items found to process
+                console.log(`No content items found to process for content source: ${contentSource.Get('Name')}`);
+            }
+        }
+        return contentItemsToProcess;
+    }
+    async SetNewAndModifiedContentItems(allRSSItems, contentSourceParams) {
+        const contentItemsToProcess = [];
+        for (const RSSContentItem of allRSSItems) {
+            const rv = new RunView();
+            const results = await rv.RunView({
+                EntityName: 'Content Items',
+                ExtraFilter: `ContentSourceID = '${contentSourceParams.contentSourceID}' AND (URL = '${RSSContentItem.link}' OR Description = '${RSSContentItem.description}')`, // According to the RSS spec, all items must contain either a title or a description.
+                ResultType: 'entity_object',
+            }, this.contextUser);
+            if (results.Success && results.Results.length) {
+                const contentItemResult = results.Results[0];
+                // This content item already exists, check the last hash to see if it has been modified
+                const lastStoredHash = contentItemResult.Checksum;
+                const newHash = await this.getChecksumFromRSSItem(RSSContentItem, this.contextUser);
+                if (lastStoredHash !== newHash) {
+                    // This content item has been modified
+                    const md = new Metadata();
+                    const contentItem = await md.GetEntityObject('Content Items', this.contextUser);
+                    contentItem.Load(contentItemResult.ID);
+                    contentItem.Checksum = newHash;
+                    contentItem.Text = JSON.stringify(RSSContentItem);
+                    await contentItem.Save();
+                    contentItemsToProcess.push(contentItem); // Content item was modified, add to list
+                }
+            }
+            else {
+                // This content item does not exist, add it
+                const md = new Metadata();
+                const contentItem = await md.GetEntityObject('Content Items', this.contextUser);
+                contentItem.ContentSourceID = contentSourceParams.contentSourceID;
+                contentItem.Name = contentSourceParams.name;
+                contentItem.Description = RSSContentItem.description || await this.engine.getContentItemDescription(contentSourceParams, this.contextUser);
+                contentItem.ContentTypeID = contentSourceParams.ContentTypeID;
+                contentItem.ContentFileTypeID = contentSourceParams.ContentFileTypeID;
+                contentItem.ContentSourceTypeID = contentSourceParams.ContentSourceTypeID;
+                contentItem.Checksum = await this.getChecksumFromRSSItem(RSSContentItem, this.contextUser);
+                contentItem.URL = RSSContentItem.link || contentSourceParams.URL;
+                contentItem.Text = JSON.stringify(RSSContentItem);
+                await contentItem.Save();
+                contentItemsToProcess.push(contentItem); // Content item was added, add to list
+            }
+        }
+        return contentItemsToProcess;
+    }
+    async parseRSSFeed(url) {
+        try {
+            if (await this.urlIsValid(url)) {
+                const RSSItems = [];
+                const parser = new Parser();
+                const feed = await parser.parseURL(url);
+                const items = feed.items;
+                // Map each item to an RSSItem object and add it to the RSSItems array
+                items.forEach(async (item) => {
+                    const rssItem = new RSSItem();
+                    rssItem.title = item.title ?? '';
+                    rssItem.link = item.link ?? '';
+                    rssItem.description = item.description ?? '';
+                    rssItem.pubDate = item.pubDate ?? '';
+                    rssItem.guid = item.guid ?? '';
+                    rssItem.category = item.category ?? '';
+                    const content = item['content:encoded'] ?? item['content'] ?? '';
+                    rssItem.content = await this.engine.parseHTML(content);
+                    rssItem.author = item.author ?? '';
+                    rssItem.comments = item.comments ?? '';
+                    rssItem.source = item.source ?? '';
+                    RSSItems.push(rssItem);
+                });
+                return RSSItems;
+            }
+            else {
+                throw new Error(`Invalid URL: ${url}`);
+            }
+        }
+        catch (error) {
+            console.error('Error fetching RSS feed:', error);
+            return [];
+        }
+    }
+    async urlIsValid(url) {
+        try {
+            const response = await axios.head(url);
+            return response.status === 200;
+        }
+        catch (e) {
+            console.error(`Invalid URL: ${url}`);
+            return false;
+        }
+    }
+    async getChecksumFromRSSItem(RSSContentItem, contextUser) {
+        const hash = crypto.createHash('sha256').update(JSON.stringify(RSSContentItem)).digest('hex');
+        return hash;
+    }
+};
+AutotagRSSFeed = __decorate([
+    RegisterClass(AutotagBase, 'AutotagRSSFeed'),
+    __metadata("design:paramtypes", [])
+], AutotagRSSFeed);
+export { AutotagRSSFeed };
+//# sourceMappingURL=AutotagRSSFeed.js.map

package/dist/RSSFeed/generic/AutotagRSSFeed.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AutotagRSSFeed.js","sourceRoot":"","sources":["../../../src/RSSFeed/generic/AutotagRSSFeed.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAY,QAAQ,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAuB,MAAM,cAAc,CAAC;AAEtE,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,MAAM,OAAO,CAAA;AACzB,OAAO,MAAM,MAAM,QAAQ,CAAA;AAC3B,OAAO,MAAM,MAAM,YAAY,CAAA;AAC/B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;AAGvB,IAAM,cAAc,GAApB,MAAM,cAAe,SAAQ,WAAW;IAK3C;QACI,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,iBAAiB,CAAC,QAAQ,CAAC;IAC7C,CAAC;IAES,cAAc;QACpB,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,OAAO,CAAC,WAAqB;QACtC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,mBAAmB,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,UAAU,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACxG,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC1G,MAAM,qBAAqB,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,cAAc,CAAC,CAAC;QAClF,MAAM,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,qBAAqB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAC5F,CAAC;IAED;;;;;OAKG;IACI,KAAK,CAAC,wBAAwB,CAAC,cAAqC;QACvE,MAAM,qBAAqB,GAAwB,EAAE,CAAA;QACrD,KAAK,MAAM,aAAa,IAAI,cAAc,EAAE,CAAC;YAEzC,2FAA2F;YAC3F,MAAM,sBAAsB,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,aAAa,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YACzG,IAAI,sBAAsB,EAAE,CAAC;gBACzB,wDAAwD;gBACxD,sBAAsB,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;oBAC1C,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;wBACb,IAAY,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;oBAC/B,CAAC;gBACL,CAAC,CAAC,CAAA;YACN,CAAC;YAED,MAAM,mBAAmB,GAAwB;gBAC7C,eAAe,EAAE,aAAa,CAAC,EAAE;gBACjC,IAAI,EAAE,aAAa,CAAC,IAAI;gBACxB,aAAa,EAAE,aAAa,CAAC,aAAa;gBAC1C,iBAAiB,EAAE,aAAa,CAAC,iBAAiB;gBAClD,mBAAmB,EAAE,aAAa,CAAC,mBAAmB;gBACtD,GAAG,EAAE,aAAa,CAAC,GAAG;aACzB,CAAA;YAED,MAAM,WAAW,GAAc,MAAM,IAAI,CAAC,YAAY,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;YAEhF,MAAM,YAAY,GAAwB,MAAM,IAAI,CAAC,6BAA6B,CAAC,WAAW,EAAE,mBAAmB,CAAC,CAAA;YAEpH,IAAI,YAAY,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1C,qBAAqB,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;YAChD,CAAC;iBACI,CAAC;gBACF,oCAAoC;gBACpC,OAAO,CAAC,GAAG,CAAC,yDAAyD,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACtG,CAAC;QACL,CAAC;QACD,OAAO,qBAAqB,CAAA;IAChC,CAAC;IAEM,KAAK,CAAC,6BAA6B,CAAC,WAAsB,EAAE,mBAAwC;QACvG,MAAM,qBAAqB,GAAwB,EAAE,CAAC;QACtD,KAAK,MAAM,cAAc,IAAI,WAAW,EAAE,CAAC;YACvC,MAAM,EAAE,GAAG,IAAI,OAAO,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC;gBAC7B,UAAU,EAAE,eAAe;gBAC3B,WAAW,EAAE,sBAAsB,mBAAmB,CAAC,eAAe,iBAAiB,cAAc,CAAC,IAAI,uBAAuB,cAAc,CAAC,WAAW,IAAI,EAAE,qFAAqF;gBACtP,UAAU,EAAE,eAAe;aAC9B,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;YAEpB,IAAI,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC5C,MAAM,iBAAiB,GAAuB,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBACjE,uFAAuF;gBACvF,MAAM,cAAc,GAAW,iBAAiB,CAAC,QAAQ,CAAA;gBACzD,MAAM,OAAO,GAAW,MAAM,IAAI,CAAC,sBAAsB,CAAC,cAAc,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;gBAE3F,IAAI,cAAc,KAAK,OAAO,EAAE,CAAC;oBAC7B,sCAAsC;oBACtC,MAAM,EAAE,GAAG,IAAI,QAAQ,EAAE,CAAC;oBAC1B,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,eAAe,CAAoB,eAAe,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;oBACnG,WAAW,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;oBACvC,WAAW,CAAC,QAAQ,GAAG,OAAO,CAAA;oBAC9B,WAAW,CAAC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAA;oBAEjD,MAAM,WAAW,CAAC,IAAI,EAAE,CAAC;oBACzB,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,yCAAyC;gBACtF,CAAC;YACL,CAAC;iBACI,CAAC;gBACF,2CAA2C;gBAC3C,MAAM,EAAE,GAAG,IAAI,QAAQ,EAAE,CAAC;gBAC1B,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,eAAe,CAAoB,eAAe,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;gBACnG,WAAW,CAAC,eAAe,GAAG,mBAAmB,CAAC,eAAe,CAAA;gBACjE,WAAW,CAAC,IAAI,GAAG,mBAAmB,CAAC,IAAI,CAAA;gBAC3C,WAAW,CAAC,WAAW,GAAG,cAAc,CAAC,WAAW,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,yBAAyB,CAAC,mBAAmB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;gBAC1I,WAAW,CAAC,aAAa,GAAG,mBAAmB,CAAC,aAAa,CAAA;gBAC7D,WAAW,CAAC,iBAAiB,GAAG,mBAAmB,CAAC,iBAAiB,CAAA;gBACrE,WAAW,CAAC,mBAAmB,GAAG,mBAAmB,CAAC,mBAAmB,CAAA;gBACzE,WAAW,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,cAAc,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;gBAC1F,WAAW,CAAC,GAAG,GAAG,cAAc,CAAC,IAAI,IAAI,mBAAmB,CAAC,GAAG,CAAA;gBAChE,WAAW,CAAC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAA;gBAEjD,MAAM,WAAW,CAAC,IAAI,EAAE,CAAC;gBACzB,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,sCAAsC;YAEnF,CAAC;QACL,CAAC;QACD,OAAO,qBAAqB,CAAA;IAChC,CAAC;IAEM,KAAK,CAAC,YAAY,CAAC,GAAW;QACjC,IAAI,CAAC;YACD,IAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,QAAQ,GAAc,EAAE,CAAA;gBAC9B,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;gBAC5B,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;gBACxC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;gBAEzB,sEAAsE;gBACtE,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,IAAS,EAAE,EAAE;oBAC9B,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;oBAC9B,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;oBACjC,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;oBAC/B,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;oBAC7C,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;oBACrC,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;oBAC/B,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;oBACvC,MAAM,OAAO,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;oBACjE,OAAO,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;oBACvD,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;oBACnC,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;oBACvC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;oBACnC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC3B,CAAC,CAAC,CAAC;gBAEH,OAAO,QAAQ,CAAA;YACnB,CAAC;iBACI,CAAC;gBACF,MAAM,IAAI,KAAK,CAAC,gBAAgB,GAAG,EAAE,CAAC,CAAC;YAC3C,CAAC;QACL,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,0BAA0B,EAAE,KAAK,CAAC,CAAC;YACjD,OAAO,EAAE,CAAC;QACZ,CAAC;IACP,CAAC;IAES,KAAK,CAAC,UAAU,CAAC,GAAW;QAClC,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvC,OAAO,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC;QACnC,CAAC;QACD,OAAO,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,KAAK,CAAC,gBAAgB,GAAG,EAAE,CAAC,CAAC;YACrC,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,sBAAsB,CAAC,cAAuB,EAAE,WAAqB;QAC9E,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAC7F,OAAO,IAAI,CAAA;IACf,CAAC;CACJ,CAAA;AA9KY,cAAc;IAD1B,aAAa,CAAC,WAAW,EAAE,gBAAgB,CAAC;;GAChC,cAAc,CA8K1B"}

package/dist/RSSFeed/generic/RSS.types.d.ts

@@ -0,0 +1,13 @@
+export declare class RSSItem {
+    title?: string;
+    link?: string;
+    description?: string;
+    pubDate?: string;
+    guid?: string;
+    category?: string;
+    content?: string;
+    author?: string;
+    comments?: string;
+    source?: string;
+}
+//# sourceMappingURL=RSS.types.d.ts.map

package/dist/RSSFeed/generic/RSS.types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"RSS.types.d.ts","sourceRoot":"","sources":["../../../src/RSSFeed/generic/RSS.types.ts"],"names":[],"mappings":"AAAA,qBAAa,OAAO;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB"}

package/dist/RSSFeed/generic/RSS.types.js

@@ -0,0 +1,3 @@
+export class RSSItem {
+}
+//# sourceMappingURL=RSS.types.js.map

package/dist/RSSFeed/generic/RSS.types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"RSS.types.js","sourceRoot":"","sources":["../../../src/RSSFeed/generic/RSS.types.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,OAAO;CAWnB"}

package/dist/RSSFeed/index.d.ts

@@ -0,0 +1,3 @@
+export * from './generic/RSS.types.js';
+export * from './generic/AutotagRSSFeed.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/RSSFeed/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/RSSFeed/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAA;AACnC,cAAc,0BAA0B,CAAA"}

package/dist/RSSFeed/index.js

@@ -0,0 +1,3 @@
+export * from './generic/RSS.types.js';
+export * from './generic/AutotagRSSFeed.js';
+//# sourceMappingURL=index.js.map

package/dist/RSSFeed/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/RSSFeed/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAA;AACnC,cAAc,0BAA0B,CAAA"}

package/dist/Websites/generic/AutotagWebsite.d.ts

@@ -0,0 +1,85 @@
+import { AutotagBase } from '../../Core/index.js';
+import { ContentSourceParams } from '../../Engine/index.js';
+import { UserInfo } from '@memberjunction/core';
+import { ContentSourceEntity, ContentItemEntity } from '@memberjunction/core-entities';
+import * as cheerio from 'cheerio';
+export declare class AutotagWebsite extends AutotagBase {
+    private contextUser;
+    private engine;
+    protected contentSourceTypeID: string;
+    protected CrawlOtherSitesInTopLevelDomain: boolean;
+    protected CrawlSitesInLowerLevelDomain: boolean;
+    protected MaxDepth: number;
+    protected RootURL: string;
+    protected URLPattern: string;
+    protected visitedURLs: Set<string>;
+    constructor();
+    protected getContextUser(): UserInfo;
+    /**
+     * Implemented abstract method from the AutotagBase class. that runs the entire autotagging process. This method is the entry point for the autotagging process.
+     * It initializes the connection, retrieves the content sources corresponding to the content source type, sets the content items that we want to process,
+     * extracts and processes the text, and sets the results in the database.
+     */
+    Autotag(contextUser: UserInfo): Promise<void>;
+    /**
+     * Given a content source, retrieve all content items associated with the content sources.
+     * The content items are then processed to determine if they have been modified since the last time they were processed or if they are new content items.
+     * @param contentSource
+     * @returns
+     */
+    SetContentItemsToProcess(contentSources: ContentSourceEntity[]): Promise<ContentItemEntity[]>;
+    /**
+     * Given a list of content item links, check if the content item already exists in the database.
+     * If the content item exists, check if the content item has been modified since the last time it was processed.
+     * If the content item does not exist, create a new content item and add it to the list of content items to process.
+     * @param contentItemLinks
+     * @param contentSourceParams
+     * @param contextUser
+     * @returns
+     */
+    protected SetNewAndModifiedContentItems(contentItemLinks: string[], contentSourceParams: ContentSourceParams, contextUser: UserInfo): Promise<ContentItemEntity[]>;
+    fetchPageContent(url: string): Promise<string>;
+    getTextWithLineBreaks(element: any, $: cheerio.CheerioAPI): string;
+    /**
+     * Given a URL, this function extracts text from a webpage.
+     * @param url
+     * @returns The text extracted from the webpage
+     */
+    parseWebPage(url: string): Promise<string>;
+    /**
+     * Given a root URL that corresponds to a content source, retrieve all the links in accordance to the crawl settings.
+     * If the crawl settings are set to crawl other sites in the top level domain, then all links in the top level domain will be retrieved.
+     * If the crawl settings are set to crawl sites in lower level domains, then function is recursively called to retrieve all links in the lower level domains.
+     * @param url
+     * @returns
+     */
+    protected getAllLinksFromContentSource(url: string, rootURL: string, regex: RegExp): Promise<string[]>;
+    /**
+     * For a given URL, retrieves all other links at that top level domain.
+     * @param url
+     * @param rootURL
+     * @param visitedURLs
+     * @returns
+     */
+    protected getTopLevelLinks(url: string, rootURL: string): Promise<void>;
+    /**
+     * Simple check to see if the URL is at the highest level domain.
+     * @param url
+     * @returns
+     */
+    protected isHighestDomain(url: string): boolean;
+    protected getBasePath(url: string): string;
+    protected getPathName(url: string): string;
+    protected urlIsValid(url: string): Promise<boolean>;
+    /**
+     * For a given URL, retrieves all links at lower level domains up to the specified crawl depth.
+     * @param url
+     * @param rootURL
+     * @param crawlDepth
+     * @param visitedURLs
+     * @returns
+     */
+    protected getLowerLevelLinks(url: string, rootURL: string, crawlDepth: number, scrapedURLs: Set<string>, regex: RegExp): Promise<Set<string>>;
+    protected delay(ms: number): Promise<unknown>;
+}
+//# sourceMappingURL=AutotagWebsite.d.ts.map

package/dist/Websites/generic/AutotagWebsite.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"AutotagWebsite.d.ts","sourceRoot":"","sources":["../../../src/Websites/generic/AutotagWebsite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAqB,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAEtE,OAAO,EAAE,QAAQ,EAAqB,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AACvF,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAMnC,qBACa,cAAe,SAAQ,WAAW;IAC3C,OAAO,CAAC,WAAW,CAAW;IAC9B,OAAO,CAAC,MAAM,CAAoB;IAClC,SAAS,CAAC,mBAAmB,EAAE,MAAM,CAAA;IACrC,SAAS,CAAC,+BAA+B,EAAE,OAAO,CAAC;IACnD,SAAS,CAAC,4BAA4B,EAAE,OAAO,CAAC;IAChD,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC;IAC7B,SAAS,CAAC,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;;IAQnC,SAAS,CAAC,cAAc,IAAI,QAAQ;IAIpC;;;;OAIG;IACU,OAAO,CAAC,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAS1D;;;;;OAKG;IACU,wBAAwB,CAAC,cAAc,EAAE,mBAAmB,EAAE,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;IAoD1G;;;;;;;;OAQG;cACa,6BAA6B,CAAC,gBAAgB,EAAE,MAAM,EAAE,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;IAsE3J,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAKpD,qBAAqB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,CAAC,UAAU,GAAG,MAAM;IAgBzE;;;;OAIG;IACU,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAavD;;;;;;OAMG;cACa,4BAA4B,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAa5G;;;;;;OAMG;cACa,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAmC7E;;;;OAIG;IACH,SAAS,CAAC,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAW/C,SAAS,CAAC,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAW1C,SAAS,CAAC,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;cAa1B,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAWzD;;;;;;;OAOG;cACa,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;cA+CnI,KAAK,CAAC,EAAE,EAAE,MAAM;CAGnC"}