@memberjunction/content-autotagging 5.22.0 → 5.24.0

package/dist/RSSFeed/generic/AutotagRSSFeed.d.ts

@@ -1,30 +1,61 @@
 import { UserInfo } from '@memberjunction/core';
-import { AutotagBase } from "../../Core/index.js";
-import { ContentSourceParams } from "../../Engine/index.js";
+import { AutotagBase, AutotagProgressCallback } from '../../Core/index.js';
 import { MJContentSourceEntity, MJContentItemEntity } from '@memberjunction/core-entities';
 import { RSSItem } from './RSS.types.js';
+/**
+ * Autotag provider for RSS and Atom feeds. Parses feed items, follows article
+ * links to fetch full page content via Cheerio, and creates ContentItems with
+ * the extracted article text rather than raw RSS metadata.
+ *
+ * Fixes:
+ * - Text capture: follows item.link to fetch full article text instead of
+ *   storing JSON.stringify(RSSItem)
+ * - Item naming: uses the RSS item title for ContentItem.Name, and the RSS
+ *   description for ContentItem.Description, instead of the source name
+ */
 export declare class AutotagRSSFeed extends AutotagBase {
     private contextUser;
     private engine;
     protected contentSourceTypeID: string;
     constructor();
-    protected getContextUser(): UserInfo;
+    Autotag(contextUser: UserInfo, onProgress?: AutotagProgressCallback): Promise<void>;
+    SetContentItemsToProcess(contentSources: MJContentSourceEntity[]): Promise<MJContentItemEntity[]>;
     /**
-     * Implemented abstract method from the AutotagBase class. that runs the entire autotagging process. This method is the entry point for the autotagging process.
-     * It initializes the connection, retrieves the content sources corresponding to the content source type, sets the content items that we want to process,
-     * extracts and processes the text, and sets the results in the database.
+     * Process a single content source: parse the RSS feed, detect new/modified
+     * items, fetch full article text, and create/update ContentItems.
      */
-    Autotag(contextUser: UserInfo): Promise<void>;
+    private ProcessContentSource;
     /**
-     * Implemented abstract method from the AutotagBase class. Given a list of content sources, this method should return a list
-     * of content source items that have been modified or added after the most recent process run for that content source.
-     * @param contentSources - An array of content sources to check for modified or added content source items
-     * @returns - An array of content source items that have been modified or added after the most recent process run for that content source
+     * Process a single RSS feed item: fetch full article text, compute checksum,
+     * create or update the ContentItem.
      */
-    SetContentItemsToProcess(contentSources: MJContentSourceEntity[]): Promise<MJContentItemEntity[]>;
-    SetNewAndModifiedContentItems(allRSSItems: RSSItem[], contentSourceParams: ContentSourceParams): Promise<MJContentItemEntity[]>;
-    parseRSSFeed(url: string): Promise<RSSItem[]>;
-    protected urlIsValid(url: string): Promise<boolean>;
-    getChecksumFromRSSItem(RSSContentItem: RSSItem, contextUser: UserInfo): Promise<string>;
+    private ProcessSingleFeedItem;
+    /**
+     * Fetch the full article text for an RSS item.
+     *
+     * Strategy:
+     * 1. If the RSS item has inline content (content:encoded), use that
+     * 2. Otherwise, follow the item's link URL and extract text with Cheerio
+     * 3. Fall back to the RSS description if link fetching fails
+     */
+    private FetchArticleText;
+    /**
+     * Fetch a web page and extract its main text content using Cheerio.
+     * Strips navigation, headers, footers, scripts, and styles.
+     */
+    private FetchAndParseWebPage;
+    /**
+     * Parse an RSS/Atom feed URL and return structured items.
+     * The content field is HTML-stripped via the engine's parseHTML.
+     */
+    ParseRSSFeed(url: string): Promise<RSSItem[]>;
+    /**
+     * Check if a URL is reachable via HTTP HEAD request.
+     */
+    private UrlIsValid;
+    /**
+     * Load existing ContentItems for this source, keyed by lowercase URL for upsert.
+     */
+    private LoadExistingContentItems;
 }
 //# sourceMappingURL=AutotagRSSFeed.d.ts.map

package/dist/RSSFeed/generic/AutotagRSSFeed.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"AutotagRSSFeed.d.ts","sourceRoot":"","sources":["../../../src/RSSFeed/generic/AutotagRSSFeed.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAqB,MAAM,sBAAsB,CAAC;AAEnE,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAqB,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACtE,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AAC3F,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAOtC,qBACa,cAAe,SAAQ,WAAW;IAC3C,OAAO,CAAC,WAAW,CAAW;IAC9B,OAAO,CAAC,MAAM,CAAoB;IAClC,SAAS,CAAC,mBAAmB,EAAE,MAAM,CAAA;;IAOrC,SAAS,CAAC,cAAc,IAAI,QAAQ;IAIpC;;;;OAIG;IACU,OAAO,CAAC,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ1D;;;;;OAKG;IACU,wBAAwB,CAAC,cAAc,EAAE,qBAAqB,EAAE,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAuCjG,6BAA6B,CAAC,WAAW,EAAE,OAAO,EAAE,EAAE,mBAAmB,EAAE,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAkD/H,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;cAqC1C,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAW5C,sBAAsB,CAAC,cAAc,EAAE,OAAO,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;CAIvG"}
+{"version":3,"file":"AutotagRSSFeed.d.ts","sourceRoot":"","sources":["../../../src/RSSFeed/generic/AutotagRSSFeed.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAA0C,MAAM,sBAAsB,CAAC;AAExF,OAAO,EAAE,WAAW,EAAE,uBAAuB,EAAE,MAAM,YAAY,CAAC;AAElE,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AAC3F,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAKtC;;;;;;;;;;GAUG;AACH,qBACa,cAAe,SAAQ,WAAW;IAC3C,OAAO,CAAC,WAAW,CAAY;IAC/B,OAAO,CAAC,MAAM,CAAqB;IACnC,SAAS,CAAC,mBAAmB,EAAG,MAAM,CAAC;;IAO1B,OAAO,CAAC,WAAW,EAAE,QAAQ,EAAE,UAAU,CAAC,EAAE,uBAAuB,GAAG,OAAO,CAAC,IAAI,CAAC;IA+BnF,wBAAwB,CAAC,cAAc,EAAE,qBAAqB,EAAE,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAgB9G;;;OAGG;YACW,oBAAoB;IA4ClC;;;OAGG;YACW,qBAAqB;IAmDnC;;;;;;;OAOG;YACW,gBAAgB;IA4B9B;;;OAGG;YACW,oBAAoB;IAgBlC;;;OAGG;IACU,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAwC1D;;OAEG;YACW,UAAU;IASxB;;OAEG;YACW,wBAAwB;CAkBzC"}

package/dist/RSSFeed/generic/AutotagRSSFeed.js

@@ -7,166 +7,284 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
 var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
-import { Metadata, RunView } from '@memberjunction/core';
+import { Metadata, RunView, LogStatus, LogError } from '@memberjunction/core';
 import { RegisterClass } from '@memberjunction/global';
-import { AutotagBase } from "../../Core/index.js";
-import { AutotagBaseEngine } from "../../Engine/index.js";
+import { AutotagBase } from '../../Core/index.js';
+import { AutotagBaseEngine } from '../../Engine/index.js';
 import { RSSItem } from './RSS.types.js';
 import axios from 'axios';
 import crypto from 'crypto';
 import Parser from 'rss-parser';
-import dotenv from 'dotenv';
-dotenv.config({ quiet: true });
+/**
+ * Autotag provider for RSS and Atom feeds. Parses feed items, follows article
+ * links to fetch full page content via Cheerio, and creates ContentItems with
+ * the extracted article text rather than raw RSS metadata.
+ *
+ * Fixes:
+ * - Text capture: follows item.link to fetch full article text instead of
+ *   storing JSON.stringify(RSSItem)
+ * - Item naming: uses the RSS item title for ContentItem.Name, and the RSS
+ *   description for ContentItem.Description, instead of the source name
+ */
 let AutotagRSSFeed = class AutotagRSSFeed extends AutotagBase {
     constructor() {
         super();
         this.engine = AutotagBaseEngine.Instance;
     }
-    getContextUser() {
-        return this.contextUser;
-    }
-    /**
-     * Implemented abstract method from the AutotagBase class. that runs the entire autotagging process. This method is the entry point for the autotagging process.
-     * It initializes the connection, retrieves the content sources corresponding to the content source type, sets the content items that we want to process,
-     * extracts and processes the text, and sets the results in the database.
-     */
-    async Autotag(contextUser) {
+    async Autotag(contextUser, onProgress) {
         this.contextUser = contextUser;
         this.contentSourceTypeID = this.engine.SetSubclassContentSourceType('RSS Feed');
+        LogStatus(`[RSS] Starting RSS autotag...`);
         const contentSources = await this.engine.getAllContentSources(this.contextUser, this.contentSourceTypeID);
-        const contentItemsToProcess = await this.SetContentItemsToProcess(contentSources);
-        await this.engine.ExtractTextAndProcessWithLLM(contentItemsToProcess, this.contextUser);
+        LogStatus(`[RSS] Found ${contentSources.length} RSS source(s)`);
+        let contentItemsToProcess;
+        try {
+            contentItemsToProcess = await this.SetContentItemsToProcess(contentSources);
+            LogStatus(`[RSS] SetContentItemsToProcess returned ${contentItemsToProcess.length} items`);
+        }
+        catch (e) {
+            const msg = e instanceof Error ? e.message : String(e);
+            LogError(`[RSS] SetContentItemsToProcess THREW: ${msg}`);
+            return;
+        }
+        if (contentItemsToProcess.length > 0) {
+            LogStatus(`[RSS] Calling ExtractTextAndProcessWithLLM with ${contentItemsToProcess.length} items...`);
+            try {
+                await this.engine.ExtractTextAndProcessWithLLM(contentItemsToProcess, this.contextUser, undefined, undefined, onProgress);
+                LogStatus(`[RSS] ExtractTextAndProcessWithLLM completed successfully`);
+            }
+            catch (e) {
+                const msg = e instanceof Error ? e.message : String(e);
+                LogError(`[RSS] ExtractTextAndProcessWithLLM THREW: ${msg}`);
+            }
+        }
+        else {
+            LogStatus('[RSS] No new or modified feed items to process');
+        }
     }
-    /**
-     * Implemented abstract method from the AutotagBase class. Given a list of content sources, this method should return a list
-     * of content source items that have been modified or added after the most recent process run for that content source.
-     * @param contentSources - An array of content sources to check for modified or added content source items
-     * @returns - An array of content source items that have been modified or added after the most recent process run for that content source
-     */
     async SetContentItemsToProcess(contentSources) {
         const contentItemsToProcess = [];
         for (const contentSource of contentSources) {
-            // If content source parameters were provided, set them. Otherwise, use the default values.
-            const contentSourceParamsMap = await this.engine.getContentSourceParams(contentSource, this.contextUser);
-            if (contentSourceParamsMap) {
-                // Override defaults with content source specific params
-                contentSourceParamsMap.forEach((value, key) => {
-                    if (key in this) {
-                        this[key] = value;
-                    }
-                });
-            }
-            const contentSourceParams = {
-                contentSourceID: contentSource.ID,
-                name: contentSource.Name,
-                ContentTypeID: contentSource.ContentTypeID,
-                ContentFileTypeID: contentSource.ContentFileTypeID,
-                ContentSourceTypeID: contentSource.ContentSourceTypeID,
-                URL: contentSource.URL
-            };
-            const allRSSItems = await this.parseRSSFeed(contentSourceParams.URL);
-            const contentItems = await this.SetNewAndModifiedContentItems(allRSSItems, contentSourceParams);
-            if (contentItems && contentItems.length > 0) {
-                contentItemsToProcess.push(...contentItems);
+            try {
+                const items = await this.ProcessContentSource(contentSource);
+                contentItemsToProcess.push(...items);
             }
-            else {
-                // No content items found to process
-                console.log(`No content items found to process for content source: ${contentSource.Get('Name')}`);
+            catch (e) {
+                const msg = e instanceof Error ? e.message : String(e);
+                LogError(`AutotagRSSFeed: failed to process source "${contentSource.Name}": ${msg}`);
             }
         }
         return contentItemsToProcess;
     }
-    async SetNewAndModifiedContentItems(allRSSItems, contentSourceParams) {
-        const contentItemsToProcess = [];
-        for (const RSSContentItem of allRSSItems) {
-            const rv = new RunView();
-            const results = await rv.RunView({
-                EntityName: 'MJ: Content Items',
-                ExtraFilter: `ContentSourceID = '${contentSourceParams.contentSourceID}' AND (URL = '${RSSContentItem.link}' OR Description = '${RSSContentItem.description}')`, // According to the RSS spec, all items must contain either a title or a description.
-                ResultType: 'entity_object',
-            }, this.contextUser);
-            if (results.Success && results.Results.length) {
-                const contentItemResult = results.Results[0];
-                // This content item already exists, check the last hash to see if it has been modified
-                const lastStoredHash = contentItemResult.Checksum;
-                const newHash = await this.getChecksumFromRSSItem(RSSContentItem, this.contextUser);
-                if (lastStoredHash !== newHash) {
-                    // This content item has been modified
-                    const md = new Metadata();
-                    const contentItem = await md.GetEntityObject('MJ: Content Items', this.contextUser);
-                    contentItem.Load(contentItemResult.ID);
-                    contentItem.Checksum = newHash;
-                    contentItem.Text = JSON.stringify(RSSContentItem);
-                    await contentItem.Save();
-                    contentItemsToProcess.push(contentItem); // Content item was modified, add to list
+    /**
+     * Process a single content source: parse the RSS feed, detect new/modified
+     * items, fetch full article text, and create/update ContentItems.
+     */
+    async ProcessContentSource(contentSource) {
+        const contentSourceParams = {
+            contentSourceID: contentSource.ID,
+            name: contentSource.Name ?? '',
+            ContentTypeID: contentSource.ContentTypeID,
+            ContentFileTypeID: contentSource.ContentFileTypeID,
+            ContentSourceTypeID: contentSource.ContentSourceTypeID,
+            URL: contentSource.URL
+        };
+        LogStatus(`[RSS] Parsing feed "${contentSource.Name}" at ${contentSourceParams.URL}...`);
+        const allRSSItems = await this.ParseRSSFeed(contentSourceParams.URL);
+        if (allRSSItems.length === 0) {
+            LogStatus(`AutotagRSSFeed: no items in feed "${contentSource.Name}"`);
+            return [];
+        }
+        LogStatus(`[RSS] Parsed ${allRSSItems.length} items from "${contentSource.Name}"`);
+        // Load existing content items for upsert by URL
+        const existingItems = await this.LoadExistingContentItems(contentSourceParams.contentSourceID);
+        LogStatus(`[RSS] ${existingItems.size} existing items for "${contentSource.Name}"`);
+        const items = [];
+        for (let idx = 0; idx < allRSSItems.length; idx++) {
+            const rssItem = allRSSItems[idx];
+            try {
+                LogStatus(`[RSS] Processing item ${idx + 1}/${allRSSItems.length}: "${rssItem.title?.substring(0, 60) ?? 'untitled'}"...`);
+                const item = await this.ProcessSingleFeedItem(rssItem, contentSourceParams, existingItems);
+                if (item) {
+                    items.push(item);
+                    LogStatus(`[RSS] Item ${idx + 1} created/updated (text: ${item.Text?.length ?? 0} chars)`);
+                }
+                else {
+                    LogStatus(`[RSS] Item ${idx + 1} skipped (unchanged or empty)`);
                 }
             }
-            else {
-                // This content item does not exist, add it
-                const md = new Metadata();
-                const contentItem = await md.GetEntityObject('MJ: Content Items', this.contextUser);
-                contentItem.ContentSourceID = contentSourceParams.contentSourceID;
-                contentItem.Name = contentSourceParams.name;
-                contentItem.Description = RSSContentItem.description || this.engine.GetContentItemDescription(contentSourceParams);
-                contentItem.ContentTypeID = contentSourceParams.ContentTypeID;
-                contentItem.ContentFileTypeID = contentSourceParams.ContentFileTypeID;
-                contentItem.ContentSourceTypeID = contentSourceParams.ContentSourceTypeID;
-                contentItem.Checksum = await this.getChecksumFromRSSItem(RSSContentItem, this.contextUser);
-                contentItem.URL = RSSContentItem.link || contentSourceParams.URL;
-                contentItem.Text = JSON.stringify(RSSContentItem);
-                await contentItem.Save();
-                contentItemsToProcess.push(contentItem); // Content item was added, add to list
+            catch (e) {
+                const msg = e instanceof Error ? e.message : String(e);
+                LogError(`[RSS] Item ${idx + 1} FAILED "${rssItem.title ?? rssItem.link}": ${msg}`);
             }
         }
-        return contentItemsToProcess;
+        LogStatus(`AutotagRSSFeed: ${items.length} new/modified items from "${contentSource.Name}"`);
+        return items;
     }
-    async parseRSSFeed(url) {
-        try {
-            if (await this.urlIsValid(url)) {
-                const RSSItems = [];
-                const parser = new Parser();
-                const feed = await parser.parseURL(url);
-                const items = feed.items;
-                // Map each item to an RSSItem object and add it to the RSSItems array
-                items.forEach(async (item) => {
-                    const rssItem = new RSSItem();
-                    rssItem.title = item.title ?? '';
-                    rssItem.link = item.link ?? '';
-                    rssItem.description = item.description ?? '';
-                    rssItem.pubDate = item.pubDate ?? '';
-                    rssItem.guid = item.guid ?? '';
-                    rssItem.category = item.category ?? '';
-                    const content = item['content:encoded'] ?? item['content'] ?? '';
-                    rssItem.content = await this.engine.parseHTML(content);
-                    rssItem.author = item.author ?? '';
-                    rssItem.comments = item.comments ?? '';
-                    rssItem.source = item.source ?? '';
-                    RSSItems.push(rssItem);
-                });
-                return RSSItems;
+    /**
+     * Process a single RSS feed item: fetch full article text, compute checksum,
+     * create or update the ContentItem.
+     */
+    async ProcessSingleFeedItem(rssItem, contentSourceParams, existingItems) {
+        // Fetch full article text from the link URL
+        const articleText = await this.FetchArticleText(rssItem);
+        if (!articleText || articleText.trim().length === 0) {
+            return null;
+        }
+        const checksum = crypto.createHash('sha256').update(articleText).digest('hex');
+        const itemUrl = rssItem.link ?? '';
+        const urlKey = itemUrl.toLowerCase();
+        // Check for existing content item (skip if unchanged, unless force reprocess)
+        const existing = existingItems.get(urlKey);
+        if (existing && existing.Checksum === checksum && !this.engine.ForceReprocess) {
+            return null; // Content unchanged
+        }
+        const md = new Metadata();
+        let contentItem;
+        if (existing) {
+            contentItem = existing;
+        }
+        else {
+            contentItem = await md.GetEntityObject('MJ: Content Items', this.contextUser);
+            contentItem.NewRecord();
+            contentItem.ContentSourceID = contentSourceParams.contentSourceID;
+            contentItem.ContentTypeID = contentSourceParams.ContentTypeID;
+            contentItem.ContentFileTypeID = contentSourceParams.ContentFileTypeID;
+            contentItem.ContentSourceTypeID = contentSourceParams.ContentSourceTypeID;
+        }
+        // Fix #6: Use RSS item title and description, not the source name
+        contentItem.Name = rssItem.title ?? contentSourceParams.name;
+        contentItem.Description = rssItem.description ?? this.engine.GetContentItemDescription(contentSourceParams);
+        contentItem.URL = itemUrl;
+        contentItem.Text = articleText;
+        contentItem.Checksum = checksum;
+        const saved = await contentItem.Save();
+        if (!saved) {
+            throw new Error(`Failed to save ContentItem for "${itemUrl}"`);
+        }
+        existingItems.set(urlKey, contentItem);
+        return contentItem;
+    }
+    /**
+     * Fetch the full article text for an RSS item.
+     *
+     * Strategy:
+     * 1. If the RSS item has inline content (content:encoded), use that
+     * 2. Otherwise, follow the item's link URL and extract text with Cheerio
+     * 3. Fall back to the RSS description if link fetching fails
+     */
+    async FetchArticleText(rssItem) {
+        // 1. Prefer inline content if it's substantial (> 200 chars after HTML stripping)
+        if (rssItem.content && rssItem.content.trim().length > 200) {
+            return rssItem.content; // Already HTML-parsed by parseRSSFeed
+        }
+        // 2. Follow the link URL to get the full article
+        if (rssItem.link) {
+            try {
+                const fullText = await this.FetchAndParseWebPage(rssItem.link);
+                if (fullText && fullText.trim().length > 100) {
+                    return fullText;
+                }
+            }
+            catch (e) {
+                const msg = e instanceof Error ? e.message : String(e);
+                LogStatus(`AutotagRSSFeed: failed to fetch article from "${rssItem.link}": ${msg}`);
+            }
+        }
+        // 3. Fall back to whatever content we have
+        if (rssItem.content && rssItem.content.trim().length > 0) {
+            return rssItem.content;
+        }
+        // 4. Last resort: use description (usually just a summary)
+        return rssItem.description ?? '';
+    }
+    /**
+     * Fetch a web page and extract its main text content using Cheerio.
+     * Strips navigation, headers, footers, scripts, and styles.
+     */
+    async FetchAndParseWebPage(url) {
+        const response = await axios.get(url, {
+            timeout: 8000,
+            headers: {
+                'User-Agent': 'Mozilla/5.0 (compatible; MemberJunction/1.0)',
+                'Accept': 'text/html,application/xhtml+xml'
             }
-            else {
-                throw new Error(`Invalid URL: ${url}`);
+        });
+        if (typeof response.data !== 'string') {
+            return '';
+        }
+        return this.engine.parseHTML(response.data);
+    }
+    /**
+     * Parse an RSS/Atom feed URL and return structured items.
+     * The content field is HTML-stripped via the engine's parseHTML.
+     */
+    async ParseRSSFeed(url) {
+        if (!await this.UrlIsValid(url)) {
+            LogError(`AutotagRSSFeed: invalid feed URL: ${url}`);
+            return [];
+        }
+        try {
+            const parser = new Parser();
+            const feed = await parser.parseURL(url);
+            const items = [];
+            for (const item of feed.items) {
+                const rssItem = new RSSItem();
+                rssItem.title = item.title ?? '';
+                rssItem.link = item.link ?? '';
+                rssItem.description = item.contentSnippet ?? item.description ?? '';
+                rssItem.pubDate = item.pubDate ?? '';
+                rssItem.guid = item.guid ?? '';
+                rssItem.category = item.categories?.join(', ') ?? '';
+                rssItem.author = item.creator ?? item.author ?? '';
+                rssItem.comments = item['comments'] ?? '';
+                rssItem.source = item['source'] ?? '';
+                // Parse inline content (content:encoded or content)
+                const rawContent = item['content:encoded'] ?? item.content ?? '';
+                if (rawContent) {
+                    rssItem.content = await this.engine.parseHTML(rawContent);
+                }
+                items.push(rssItem);
             }
+            return items;
         }
         catch (error) {
-            console.error('Error fetching RSS feed:', error);
+            const msg = error instanceof Error ? error.message : String(error);
+            LogError(`AutotagRSSFeed: error parsing feed "${url}": ${msg}`);
             return [];
         }
     }
-    async urlIsValid(url) {
+    /**
+     * Check if a URL is reachable via HTTP HEAD request.
+     */
+    async UrlIsValid(url) {
         try {
-            const response = await axios.head(url);
-            return response.status === 200;
+            const response = await axios.head(url, { timeout: 10000 });
+            return response.status >= 200 && response.status < 400;
         }
-        catch (e) {
-            console.error(`Invalid URL: ${url}`);
+        catch {
             return false;
         }
     }
-    async getChecksumFromRSSItem(RSSContentItem, contextUser) {
-        const hash = crypto.createHash('sha256').update(JSON.stringify(RSSContentItem)).digest('hex');
-        return hash;
+    /**
+     * Load existing ContentItems for this source, keyed by lowercase URL for upsert.
+     */
+    async LoadExistingContentItems(contentSourceID) {
+        const rv = new RunView();
+        const result = await rv.RunView({
+            EntityName: 'MJ: Content Items',
+            ExtraFilter: `ContentSourceID='${contentSourceID}'`,
+            ResultType: 'entity_object'
+        }, this.contextUser);
+        const map = new Map();
+        if (result.Success) {
+            for (const ci of result.Results) {
+                if (ci.URL) {
+                    map.set(ci.URL.toLowerCase(), ci);
+                }
+            }
+        }
+        return map;
     }
 };
 AutotagRSSFeed = __decorate([

package/dist/RSSFeed/generic/AutotagRSSFeed.js.map

@@ -1 +1 @@
-{"version":3,"file":"AutotagRSSFeed.js","sourceRoot":"","sources":["../../../src/RSSFeed/generic/AutotagRSSFeed.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAY,QAAQ,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAuB,MAAM,cAAc,CAAC;AAEtE,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,MAAM,OAAO,CAAA;AACzB,OAAO,MAAM,MAAM,QAAQ,CAAA;AAC3B,OAAO,MAAM,MAAM,YAAY,CAAA;AAC/B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;AAGvB,IAAM,cAAc,GAApB,MAAM,cAAe,SAAQ,WAAW;IAK3C;QACI,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,iBAAiB,CAAC,QAAQ,CAAC;IAC7C,CAAC;IAES,cAAc;QACpB,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,OAAO,CAAC,WAAqB;QACtC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,UAAU,CAAC,CAAC;QAChF,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC1G,MAAM,qBAAqB,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,cAAc,CAAC,CAAC;QAClF,MAAM,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,qBAAqB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAC5F,CAAC;IAED;;;;;OAKG;IACI,KAAK,CAAC,wBAAwB,CAAC,cAAuC;QACzE,MAAM,qBAAqB,GAA0B,EAAE,CAAA;QACvD,KAAK,MAAM,aAAa,IAAI,cAAc,EAAE,CAAC;YAEzC,2FAA2F;YAC3F,MAAM,sBAAsB,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,aAAa,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YACzG,IAAI,sBAAsB,EAAE,CAAC;gBACzB,wDAAwD;gBACxD,sBAAsB,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;oBAC1C,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;wBACb,IAAY,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;oBAC/B,CAAC;gBACL,CAAC,CAAC,CAAA;YACN,CAAC;YAED,MAAM,mBAAmB,GAAwB;gBAC7C,eAAe,EAAE,aAAa,CAAC,EAAE;gBACjC,IAAI,EAAE,aAAa,CAAC,IAAI;gBACxB,aAAa,EAAE,aAAa,CAAC,aAAa;gBAC1C,iBAAiB,EAAE,aAAa,CAAC,iBAAiB;gBAClD,mBAAmB,EAAE,aAAa,CAAC,mBAAmB;gBACtD,GAAG,EAAE,aAAa,CAAC,GAAG;aACzB,CAAA;YAED,MAAM,WAAW,GAAc,MAAM,IAAI,CAAC,YAAY,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;YAEhF,MAAM,YAAY,GAA0B,MAAM,IAAI,CAAC,6BAA6B,CAAC,WAAW,EAAE,mBAAmB,CAAC,CAAA;YAEtH,IAAI,YAAY,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1C,qBAAqB,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;YAChD,CAAC;iBACI,CAAC;gBACF,oCAAoC;gBACpC,OAAO,CAAC,GAAG,CAAC,yDAAyD,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACtG,CAAC;QACL,CAAC;QACD,OAAO,qBAAqB,CAAA;IAChC,CAAC;IAEM,KAAK,CAAC,6BAA6B,CAAC,WAAsB,EAAE,mBAAwC;QACvG,MAAM,qBAAqB,GAA0B,EAAE,CAAC;QACxD,KAAK,MAAM,cAAc,IAAI,WAAW,EAAE,CAAC;YACvC,MAAM,EAAE,GAAG,IAAI,OAAO,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC;gBAC7B,UAAU,EAAE,mBAAmB;gBAC/B,WAAW,EAAE,sBAAsB,mBAAmB,CAAC,eAAe,iBAAiB,cAAc,CAAC,IAAI,uBAAuB,cAAc,CAAC,WAAW,IAAI,EAAE,qFAAqF;gBACtP,UAAU,EAAE,eAAe;aAC9B,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;YAEpB,IAAI,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC5C,MAAM,iBAAiB,GAAyB,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBACnE,uFAAuF;gBACvF,MAAM,cAAc,GAAW,iBAAiB,CAAC,QAAQ,CAAA;gBACzD,MAAM,OAAO,GAAW,MAAM,IAAI,CAAC,sBAAsB,CAAC,cAAc,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;gBAE3F,IAAI,cAAc,KAAK,OAAO,EAAE,CAAC;oBAC7B,sCAAsC;oBACtC,MAAM,EAAE,GAAG,IAAI,QAAQ,EAAE,CAAC;oBAC1B,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,eAAe,CAAsB,mBAAmB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;oBACzG,WAAW,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;oBACvC,WAAW,CAAC,QAAQ,GAAG,OAAO,CAAA;oBAC9B,WAAW,CAAC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAA;oBAEjD,MAAM,WAAW,CAAC,IAAI,EAAE,CAAC;oBACzB,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,yCAAyC;gBACtF,CAAC;YACL,CAAC;iBACI,CAAC;gBACF,2CAA2C;gBAC3C,MAAM,EAAE,GAAG,IAAI,QAAQ,EAAE,CAAC;gBAC1B,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,eAAe,CAAsB,mBAAmB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;gBACzG,WAAW,CAAC,eAAe,GAAG,mBAAmB,CAAC,eAAe,CAAA;gBACjE,WAAW,CAAC,IAAI,GAAG,mBAAmB,CAAC,IAAI,CAAA;gBAC3C,WAAW,CAAC,WAAW,GAAG,cAAc,CAAC,WAAW,IAAI,IAAI,CAAC,MAAM,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAA;gBAClH,WAAW,CAAC,aAAa,GAAG,mBAAmB,CAAC,aAAa,CAAA;gBAC7D,WAAW,CAAC,iBAAiB,GAAG,mBAAmB,CAAC,iBAAiB,CAAA;gBACrE,WAAW,CAAC,mBAAmB,GAAG,mBAAmB,CAAC,mBAAmB,CAAA;gBACzE,WAAW,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,cAAc,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;gBAC1F,WAAW,CAAC,GAAG,GAAG,cAAc,CAAC,IAAI,IAAI,mBAAmB,CAAC,GAAG,CAAA;gBAChE,WAAW,CAAC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAA;gBAEjD,MAAM,WAAW,CAAC,IAAI,EAAE,CAAC;gBACzB,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,sCAAsC;YAEnF,CAAC;QACL,CAAC;QACD,OAAO,qBAAqB,CAAA;IAChC,CAAC;IAEM,KAAK,CAAC,YAAY,CAAC,GAAW;QACjC,IAAI,CAAC;YACD,IAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,QAAQ,GAAc,EAAE,CAAA;gBAC9B,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;gBAC5B,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;gBACxC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;gBAEzB,sEAAsE;gBACtE,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,IAAS,EAAE,EAAE;oBAC9B,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;oBAC9B,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;oBACjC,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;oBAC/B,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;oBAC7C,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;oBACrC,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;oBAC/B,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;oBACvC,MAAM,OAAO,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;oBACjE,OAAO,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;oBACvD,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;oBACnC,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;oBACvC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;oBACnC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC3B,CAAC,CAAC,CAAC;gBAEH,OAAO,QAAQ,CAAA;YACnB,CAAC;iBACI,CAAC;gBACF,MAAM,IAAI,KAAK,CAAC,gBAAgB,GAAG,EAAE,CAAC,CAAC;YAC3C,CAAC;QACL,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,0BAA0B,EAAE,KAAK,CAAC,CAAC;YACjD,OAAO,EAAE,CAAC;QACZ,CAAC;IACP,CAAC;IAES,KAAK,CAAC,UAAU,CAAC,GAAW;QAClC,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvC,OAAO,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC;QACnC,CAAC;QACD,OAAO,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,KAAK,CAAC,gBAAgB,GAAG,EAAE,CAAC,CAAC;YACrC,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,sBAAsB,CAAC,cAAuB,EAAE,WAAqB;QAC9E,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAC7F,OAAO,IAAI,CAAA;IACf,CAAC;CACJ,CAAA;AA9KY,cAAc;IAD1B,aAAa,CAAC,WAAW,EAAE,gBAAgB,CAAC;;GAChC,cAAc,CA8K1B"}
+{"version":3,"file":"AutotagRSSFeed.js","sourceRoot":"","sources":["../../../src/RSSFeed/generic/AutotagRSSFeed.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAY,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACxF,OAAO,EAAE,aAAa,EAAiB,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAE,WAAW,EAA2B,MAAM,YAAY,CAAC;AAClE,OAAO,EAAE,iBAAiB,EAAuB,MAAM,cAAc,CAAC;AAEtE,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,MAAM,MAAM,YAAY,CAAC;AAEhC;;;;;;;;;;GAUG;AAEI,IAAM,cAAc,GAApB,MAAM,cAAe,SAAQ,WAAW;IAK3C;QACI,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,iBAAiB,CAAC,QAAQ,CAAC;IAC7C,CAAC;IAEM,KAAK,CAAC,OAAO,CAAC,WAAqB,EAAE,UAAoC;QAC5E,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,UAAU,CAAC,CAAC;QAChF,SAAS,CAAC,+BAA+B,CAAC,CAAC;QAC3C,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC1G,SAAS,CAAC,eAAe,cAAc,CAAC,MAAM,gBAAgB,CAAC,CAAC;QAEhE,IAAI,qBAA4C,CAAC;QACjD,IAAI,CAAC;YACD,qBAAqB,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,cAAc,CAAC,CAAC;YAC5E,SAAS,CAAC,2CAA2C,qBAAqB,CAAC,MAAM,QAAQ,CAAC,CAAC;QAC/F,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,MAAM,GAAG,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACvD,QAAQ,CAAC,yCAAyC,GAAG,EAAE,CAAC,CAAC;YACzD,OAAO;QACX,CAAC;QAED,IAAI,qBAAqB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,SAAS,CAAC,mDAAmD,qBAAqB,CAAC,MAAM,WAAW,CAAC,CAAC;YACtG,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,qBAAqB,EAAE,IAAI,CAAC,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;gBAC1H,SAAS,CAAC,2DAA2D,CAAC,CAAC;YAC3E,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,MAAM,GAAG,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBACvD,QAAQ,CAAC,6CAA6C,GAAG,EAAE,CAAC,CAAC;YACjE,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,SAAS,CAAC,gDAAgD,CAAC,CAAC;QAChE,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,wBAAwB,CAAC,cAAuC;QACzE,MAAM,qBAAqB,GAA0B,EAAE,CAAC;QAExD,KAAK,MAAM,aAAa,IAAI,cAAc,EAAE,CAAC;YACzC,IAAI,CAAC;gBACD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC,CAAC;gBAC7D,qBAAqB,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;YACzC,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,MAAM,GAAG,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBACvD,QAAQ,CAAC,6CAA6C,aAAa,CAAC,IAAI,MAAM,GAAG,EAAE,CAAC,CAAC;YACzF,CAAC;QACL,CAAC;QAED,OAAO,qBAAqB,CAAC;IACjC,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,oBAAoB,CAAC,aAAoC;QACnE,MAAM,mBAAmB,GAAwB;YAC7C,eAAe,EAAE,aAAa,CAAC,EAAE;YACjC,IAAI,EAAE,aAAa,CAAC,IAAI,IAAI,EAAE;YAC9B,aAAa,EAAE,aAAa,CAAC,aAAa;YAC1C,iBAAiB,EAAE,aAAa,CAAC,iBAAiB;YAClD,mBAAmB,EAAE,aAAa,CAAC,mBAAmB;YACtD,GAAG,EAAE,aAAa,CAAC,GAAG;SACzB,CAAC;QAEF,SAAS,CAAC,uBAAuB,aAAa,CAAC,IAAI,QAAQ,mBAAmB,CAAC,GAAG,KAAK,CAAC,CAAC;QACzF,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;QACrE,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,SAAS,CAAC,qCAAqC,aAAa,CAAC,IAAI,GAAG,CAAC,CAAC;YACtE,OAAO,EAAE,CAAC;QACd,CAAC;QACD,SAAS,CAAC,gBAAgB,WAAW,CAAC,MAAM,gBAAgB,aAAa,CAAC,IAAI,GAAG,CAAC,CAAC;QAEnF,gDAAgD;QAChD,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,mBAAmB,CAAC,eAAe,CAAC,CAAC;QAC/F,SAAS,CAAC,SAAS,aAAa,CAAC,IAAI,wBAAwB,aAAa,CAAC,IAAI,GAAG,CAAC,CAAC;QAEpF,MAAM,KAAK,GAA0B,EAAE,CAAC;QACxC,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,WAAW,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;YAChD,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;YACjC,IAAI,CAAC;gBACD,SAAS,CAAC,yBAAyB,GAAG,GAAG,CAAC,IAAI,WAAW,CAAC,MAAM,MAAM,OAAO,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,UAAU,MAAM,CAAC,CAAC;gBAC3H,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,OAAO,EAAE,mBAAmB,EAAE,aAAa,CAAC,CAAC;gBAC3F,IAAI,IAAI,EAAE,CAAC;oBACP,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACjB,SAAS,CAAC,cAAc,GAAG,GAAG,CAAC,2BAA2B,IAAI,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC/F,CAAC;qBAAM,CAAC;oBACJ,SAAS,CAAC,cAAc,GAAG,GAAG,CAAC,+BAA+B,CAAC,CAAC;gBACpE,CAAC;YACL,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,MAAM,GAAG,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBACvD,QAAQ,CAAC,cAAc,GAAG,GAAG,CAAC,YAAY,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,IAAI,MAAM,GAAG,EAAE,CAAC,CAAC;YACxF,CAAC;QACL,CAAC;QAED,SAAS,CAAC,mBAAmB,KAAK,CAAC,MAAM,6BAA6B,aAAa,CAAC,IAAI,GAAG,CAAC,CAAC;QAC7F,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,qBAAqB,CAC/B,OAAgB,EAChB,mBAAwC,EACxC,aAA+C;QAE/C,4CAA4C;QAC5C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;QACzD,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClD,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/E,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;QAErC,8EAA8E;QAC9E,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC3C,IAAI,QAAQ,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;YAC5E,OAAO,IAAI,CAAC,CAAC,oBAAoB;QACrC,CAAC;QAED,MAAM,EAAE,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC1B,IAAI,WAAgC,CAAC;QAErC,IAAI,QAAQ,EAAE,CAAC;YACX,WAAW,GAAG,QAAQ,CAAC;QAC3B,CAAC;aAAM,CAAC;YACJ,WAAW,GAAG,MAAM,EAAE,CAAC,eAAe,CAAsB,mBAAmB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YACnG,WAAW,CAAC,SAAS,EAAE,CAAC;YACxB,WAAW,CAAC,eAAe,GAAG,mBAAmB,CAAC,eAAe,CAAC;YAClE,WAAW,CAAC,aAAa,GAAG,mBAAmB,CAAC,aAAa,CAAC;YAC9D,WAAW,CAAC,iBAAiB,GAAG,mBAAmB,CAAC,iBAAiB,CAAC;YACtE,WAAW,CAAC,mBAAmB,GAAG,mBAAmB,CAAC,mBAAmB,CAAC;QAC9E,CAAC;QAED,kEAAkE;QAClE,WAAW,CAAC,IAAI,GAAG,OAAO,CAAC,KAAK,IAAI,mBAAmB,CAAC,IAAI,CAAC;QAC7D,WAAW,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,MAAM,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;QAC5G,WAAW,CAAC,GAAG,GAAG,OAAO,CAAC;QAC1B,WAAW,CAAC,IAAI,GAAG,WAAW,CAAC;QAC/B,WAAW,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAEhC,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,CAAC;QACvC,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,mCAAmC,OAAO,GAAG,CAAC,CAAC;QACnE,CAAC;QAED,aAAa,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QACvC,OAAO,WAAW,CAAC;IACvB,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,gBAAgB,CAAC,OAAgB;QAC3C,kFAAkF;QAClF,IAAI,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YACzD,OAAO,OAAO,CAAC,OAAO,CAAC,CAAC,sCAAsC;QAClE,CAAC;QAED,iDAAiD;QACjD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACf,IAAI,CAAC;gBACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;gBAC/D,IAAI,QAAQ,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;oBAC3C,OAAO,QAAQ,CAAC;gBACpB,CAAC;YACL,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,MAAM,GAAG,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBACvD,SAAS,CAAC,iDAAiD,OAAO,CAAC,IAAI,MAAM,GAAG,EAAE,CAAC,CAAC;YACxF,CAAC;QACL,CAAC;QAED,2CAA2C;QAC3C,IAAI,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvD,OAAO,OAAO,CAAC,OAAO,CAAC;QAC3B,CAAC;QAED,2DAA2D;QAC3D,OAAO,OAAO,CAAC,WAAW,IAAI,EAAE,CAAC;IACrC,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,oBAAoB,CAAC,GAAW;QAC1C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE;YAClC,OAAO,EAAE,IAAI;YACb,OAAO,EAAE;gBACL,YAAY,EAAE,8CAA8C;gBAC5D,QAAQ,EAAE,iCAAiC;aAC9C;SACJ,CAAC,CAAC;QAEH,IAAI,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACpC,OAAO,EAAE,CAAC;QACd,CAAC;QAED,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAChD,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,YAAY,CAAC,GAAW;QACjC,IAAI,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC9B,QAAQ,CAAC,qCAAqC,GAAG,EAAE,CAAC,CAAC;YACrD,OAAO,EAAE,CAAC;QACd,CAAC;QAED,IAAI,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YAExC,MAAM,KAAK,GAAc,EAAE,CAAC;YAC5B,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC5B,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;gBAC9B,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;gBACjC,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;gBAC/B,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;gBACpE,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;gBACrC,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;gBAC/B,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACrD,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;gBACnD,OAAO,CAAC,QAAQ,GAAI,IAAgC,CAAC,UAAU,CAAW,IAAI,EAAE,CAAC;gBACjF,OAAO,CAAC,MAAM,GAAI,IAAgC,CAAC,QAAQ,CAAW,IAAI,EAAE,CAAC;gBAE7E,oDAAoD;gBACpD,MAAM,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;gBACjE,IAAI,UAAU,EAAE,CAAC;oBACb,OAAO,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;gBAC9D,CAAC;gBAED,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;YAED,OAAO,KAAK,CAAC;QACjB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACnE,QAAQ,CAAC,uCAAuC,GAAG,MAAM,GAAG,EAAE,CAAC,CAAC;YAChE,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,UAAU,CAAC,GAAW;QAChC,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YAC3D,OAAO,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,CAAC;QAC3D,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,wBAAwB,CAAC,eAAuB;QAC1D,MAAM,EAAE,GAAG,IAAI,OAAO,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAsB;YACjD,UAAU,EAAE,mBAAmB;YAC/B,WAAW,EAAE,oBAAoB,eAAe,GAAG;YACnD,UAAU,EAAE,eAAe;SAC9B,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAErB,MAAM,GAAG,GAAG,IAAI,GAAG,EAA+B,CAAC;QACnD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACjB,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBAC9B,IAAI,EAAE,CAAC,GAAG,EAAE,CAAC;oBACT,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtC,CAAC;YACL,CAAC;QACL,CAAC;QACD,OAAO,GAAG,CAAC;IACf,CAAC;CACJ,CAAA;AArSY,cAAc;IAD1B,aAAa,CAAC,WAAW,EAAE,gBAAgB,CAAC;;GAChC,cAAc,CAqS1B"}

package/dist/Websites/generic/AutotagWebsite.d.ts

@@ -1,4 +1,4 @@
-import { AutotagBase } from '../../Core/index.js';
+import { AutotagBase, AutotagProgressCallback } from '../../Core/index.js';
 import { ContentSourceParams } from '../../Engine/index.js';
 import { UserInfo } from '@memberjunction/core';
 import { MJContentSourceEntity, MJContentItemEntity } from '@memberjunction/core-entities';
@@ -20,7 +20,7 @@ export declare class AutotagWebsite extends AutotagBase {
      * It initializes the connection, retrieves the content sources corresponding to the content source type, sets the content items that we want to process,
      * extracts and processes the text, and sets the results in the database.
      */
-    Autotag(contextUser: UserInfo): Promise<void>;
+    Autotag(contextUser: UserInfo, onProgress?: AutotagProgressCallback): Promise<void>;
     /**
      * Given a content source, retrieve all content items associated with the content sources.
      * The content items are then processed to determine if they have been modified since the last time they were processed or if they are new content items.

package/dist/Websites/generic/AutotagWebsite.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"AutotagWebsite.d.ts","sourceRoot":"","sources":["../../../src/Websites/generic/AutotagWebsite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAqB,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAEtE,OAAO,EAAE,QAAQ,EAAqB,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AAC3F,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAMnC,qBACa,cAAe,SAAQ,WAAW;IAC3C,OAAO,CAAC,WAAW,CAAW;IAC9B,OAAO,CAAC,MAAM,CAAoB;IAClC,SAAS,CAAC,mBAAmB,EAAE,MAAM,CAAA;IACrC,SAAS,CAAC,+BAA+B,EAAE,OAAO,CAAC;IACnD,SAAS,CAAC,4BAA4B,EAAE,OAAO,CAAC;IAChD,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC;IAC7B,SAAS,CAAC,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;;IAQnC,SAAS,CAAC,cAAc,IAAI,QAAQ;IAIpC;;;;OAIG;IACU,OAAO,CAAC,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAS1D;;;;;OAKG;IACU,wBAAwB,CAAC,cAAc,EAAE,qBAAqB,EAAE,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAoD9G;;;;;;;;OAQG;cACa,6BAA6B,CAAC,gBAAgB,EAAE,MAAM,EAAE,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAsE7J,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAKpD,qBAAqB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,CAAC,UAAU,GAAG,MAAM;IAgBzE;;;;OAIG;IACU,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAavD;;;;;;OAMG;cACa,4BAA4B,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAa5G;;;;;;OAMG;cACa,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAmC7E;;;;OAIG;IACH,SAAS,CAAC,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAW/C,SAAS,CAAC,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAW1C,SAAS,CAAC,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;cAa1B,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAWzD;;;;;;;OAOG;cACa,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;cA+CnI,KAAK,CAAC,EAAE,EAAE,MAAM;CAGnC"}
+{"version":3,"file":"AutotagWebsite.d.ts","sourceRoot":"","sources":["../../../src/Websites/generic/AutotagWebsite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,uBAAuB,EAAE,MAAM,YAAY,CAAC;AAClE,OAAO,EAAqB,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAEtE,OAAO,EAAE,QAAQ,EAAqB,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AAC3F,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAMnC,qBACa,cAAe,SAAQ,WAAW;IAC3C,OAAO,CAAC,WAAW,CAAW;IAC9B,OAAO,CAAC,MAAM,CAAoB;IAClC,SAAS,CAAC,mBAAmB,EAAE,MAAM,CAAA;IACrC,SAAS,CAAC,+BAA+B,EAAE,OAAO,CAAC;IACnD,SAAS,CAAC,4BAA4B,EAAE,OAAO,CAAC;IAChD,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC;IAC7B,SAAS,CAAC,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;;IAQnC,SAAS,CAAC,cAAc,IAAI,QAAQ;IAIpC;;;;OAIG;IACU,OAAO,CAAC,WAAW,EAAE,QAAQ,EAAE,UAAU,CAAC,EAAE,uBAAuB,GAAG,OAAO,CAAC,IAAI,CAAC;IAShG;;;;;OAKG;IACU,wBAAwB,CAAC,cAAc,EAAE,qBAAqB,EAAE,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAoD9G;;;;;;;;OAQG;cACa,6BAA6B,CAAC,gBAAgB,EAAE,MAAM,EAAE,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAsE7J,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAKpD,qBAAqB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,CAAC,UAAU,GAAG,MAAM;IAgBzE;;;;OAIG;IACU,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAavD;;;;;;OAMG;cACa,4BAA4B,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAa5G;;;;;;OAMG;cACa,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAmC7E;;;;OAIG;IACH,SAAS,CAAC,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAW/C,SAAS,CAAC,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAW1C,SAAS,CAAC,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;cAa1B,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAWzD;;;;;;;OAOG;cACa,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;cA+CnI,KAAK,CAAC,EAAE,EAAE,MAAM;CAGnC"}

package/dist/Websites/generic/AutotagWebsite.js

@@ -30,12 +30,12 @@ let AutotagWebsite = class AutotagWebsite extends AutotagBase {
      * It initializes the connection, retrieves the content sources corresponding to the content source type, sets the content items that we want to process,
      * extracts and processes the text, and sets the results in the database.
      */
-    async Autotag(contextUser) {
+    async Autotag(contextUser, onProgress) {
         this.contextUser = contextUser;
         this.contentSourceTypeID = this.engine.SetSubclassContentSourceType('Website');
         const contentSources = await this.engine.getAllContentSources(this.contextUser, this.contentSourceTypeID);
         const contentItemsToProcess = await this.SetContentItemsToProcess(contentSources);
-        await this.engine.ExtractTextAndProcessWithLLM(contentItemsToProcess, this.contextUser);
+        await this.engine.ExtractTextAndProcessWithLLM(contentItemsToProcess, this.contextUser, undefined, undefined, onProgress);
     }
     /**
      * Given a content source, retrieve all content items associated with the content sources.