@memberjunction/content-autotagging 5.22.0 → 5.24.0

package/README.md

@@ -1,10 +1,10 @@
 # @memberjunction/content-autotagging
 
-AI-powered content ingestion and autotagging engine for MemberJunction. Scans content from multiple sources (local files, websites, RSS feeds, cloud storage), extracts text from documents, and uses LLMs to generate tags, summaries, and metadata attributes.
+AI-powered content ingestion, autotagging, and vectorization engine for MemberJunction. Scans content from multiple sources (local files, websites, RSS feeds, cloud storage), extracts text from documents, uses LLMs to generate weighted tags and metadata attributes, and vectorizes content for semantic search.
 
 ## Overview
 
-The `@memberjunction/content-autotagging` package provides an extensible framework for ingesting content from diverse sources and leveraging AI models to extract meaningful tags, summaries, and metadata. Built on the MemberJunction platform, it helps organizations automatically organize and categorize their content.
+The `@memberjunction/content-autotagging` package provides an extensible framework for ingesting content from diverse sources and leveraging AI models to extract meaningful tags, summaries, and metadata. Built on the MemberJunction platform, it helps organizations automatically organize and categorize their content. The engine uses the managed **"Content Autotagging"** AI prompt via `AIPromptRunner` (rather than direct `BaseLLM` calls), enabling prompt versioning, model routing, and centralized prompt management.
 
 ```mermaid
 graph TD
@@ -19,10 +19,14 @@ graph TD
     G --> I["Office Parser"]
     G --> J["HTML Parser<br/>(Cheerio)"]
 
-    A --> K["LLM Processing"]
-    K --> L["Tag Generation"]
+    A --> K["AIPromptRunner<br/>(Content Autotagging prompt)"]
+    K --> L["Tag Generation<br/>(with weights)"]
     K --> M["Attribute Extraction"]
 
+    A --> V["Vectorization"]
+    V --> W["Embedding Model"]
+    V --> X["Vector DB Upsert"]
+
     A --> N["Content Items<br/>(Database)"]
     A --> O["Content Item Attributes<br/>(Database)"]
 
@@ -34,10 +38,21 @@ graph TD
     style F fill:#2d8659,stroke:#1a5c3a,color:#fff
     style G fill:#b8762f,stroke:#8a5722,color:#fff
     style K fill:#7c5295,stroke:#563a6b,color:#fff
+    style V fill:#2d6a9f,stroke:#1a4971,color:#fff
     style N fill:#2d6a9f,stroke:#1a4971,color:#fff
     style O fill:#2d6a9f,stroke:#1a4971,color:#fff
 ```
 
+## Key Features
+
+- **AIPromptRunner integration**: Uses the managed "Content Autotagging" prompt, enabling prompt versioning and model routing through MJ's prompt management system (no direct `BaseLLM` calls)
+- **Tag weights**: Each generated tag includes a relevance weight (0.0--1.0) indicating how strongly the tag relates to the content
+- **Batch processing**: Configurable batch size (default: 20) with concurrent processing within each batch
+- **Parallel tagging + vectorization**: Tagging and vectorization run in parallel for maximum throughput
+- **Per-source/type embedding model selection**: Cascade resolution for embedding model and vector index -- source override, then content type default, then global fallback (first active vector index)
+- **Real-time progress reporting**: `AutotagProgressCallback` provides per-item progress updates during processing
+- **Graceful provider skip**: Providers skip gracefully when no content sources are configured for their type
+
 ## Installation
 
 ```bash
@@ -51,7 +66,8 @@ sequenceDiagram
     participant Source as Content Source
     participant Engine as AutotagBaseEngine
     participant Extract as Text Extractor
-    participant AI as LLM
+    participant Prompt as AIPromptRunner
+    participant Vec as Embedding + VectorDB
     participant DB as Database
 
     Source->>Engine: Provide content items
@@ -59,9 +75,14 @@ sequenceDiagram
     Engine->>Extract: Extract text (PDF/Office/HTML)
     Extract-->>Engine: Raw text
     Engine->>Engine: Chunk text for token limits
-    Engine->>AI: Generate tags and attributes
-    AI-->>Engine: Structured metadata
-    Engine->>DB: Save ContentItem + Attributes
+    par Tagging
+        Engine->>Prompt: Run "Content Autotagging" prompt
+        Prompt-->>Engine: Tags (with weights) + Attributes
+    and Vectorization
+        Engine->>Vec: Embed text + upsert to vector DB
+        Vec-->>Engine: Vectorization result
+    end
+    Engine->>DB: Save ContentItem + Tags + Attributes
     Engine->>DB: Create ProcessRun record
 ```
 
@@ -74,7 +95,7 @@ sequenceDiagram
 | RSS Feeds | `AutotagRSSFeed` | Parses RSS/Atom feeds for articles |
 | Azure Blob | `AutotagAzureBlob` | Processes files from Azure Blob Storage |
 
-All sources extend `AutotagBase`, which provides the common interface for content discovery and ingestion.
+All sources extend `AutotagBase`, which provides the common interface for content discovery and ingestion. Each source's `Autotag()` method accepts an optional `AutotagProgressCallback` for real-time progress reporting. Sources skip gracefully when no content sources of their type are configured in the database.
 
 ## Supported File Formats
 
@@ -85,6 +106,32 @@ All sources extend `AutotagBase`, which provides the common interface for conten
 | HTML/Web Pages | `cheerio` | .html, .htm |
 | Plain Text | Native | .txt, .md, .csv |
 
+## Tag Weights
+
+The LLM prompt returns tags with relevance weights between 0.0 and 1.0 indicating how strongly each tag relates to the content. Both old-style (plain string array) and new-style (object with `tag` + `weight`) responses are supported:
+
+```json
+// New format (preferred) — returned by the "Content Autotagging" prompt
+[
+  { "tag": "machine learning", "weight": 0.95 },
+  { "tag": "neural networks", "weight": 0.82 },
+  { "tag": "data science", "weight": 0.70 }
+]
+
+// Legacy format — auto-normalized with weight 1.0
+["machine learning", "neural networks", "data science"]
+```
+
+## Embedding Model and Vector Index Resolution
+
+The engine resolves the embedding model and vector index for each content item using a three-level cascade:
+
+1. **Content Source override**: If the source has `EmbeddingModelID` and `VectorIndexID` set, those are used
+2. **Content Type default**: If the source has no override, the content type's defaults are used
+3. **Global fallback**: If neither source nor type specifies, the first active vector index in the system is used
+
+Items sharing the same (embeddingModel, vectorIndex) pair are grouped and processed together for efficient batching.
+
 ## Usage
 
 ### RSS Feed Processing
@@ -93,7 +140,9 @@ All sources extend `AutotagBase`, which provides the common interface for conten
 import { AutotagRSSFeed } from '@memberjunction/content-autotagging';
 
 const rssTagger = new AutotagRSSFeed();
-await rssTagger.Autotag(contextUser);
+await rssTagger.Autotag(contextUser, (processed, total, currentItem) => {
+    console.log(`[${processed}/${total}] Processing: ${currentItem}`);
+});
 ```
 
 ### Website Content Processing
@@ -133,13 +182,19 @@ await blobTagger.Autotag(contextUser);
 import { AutotagBaseEngine } from '@memberjunction/content-autotagging';
 
 const engine = AutotagBaseEngine.Instance;
-await engine.ExtractTextAndProcessWithLLM(contentItems, contextUser);
+
+// Process content items with custom batch size
+await engine.ExtractTextAndProcessWithLLM(contentItems, contextUser, batchSize);
+
+// Vectorize content items (runs in parallel with tagging)
+const result = await engine.VectorizeContentItems(contentItems, tagMap, contextUser, batchSize);
+console.log(`Vectorized: ${result.vectorized}, Skipped: ${result.skipped}`);
 ```
 
 ## Creating a Custom Content Source
 
 ```typescript
-import { AutotagBase } from '@memberjunction/content-autotagging';
+import { AutotagBase, AutotagProgressCallback } from '@memberjunction/content-autotagging';
 import { RegisterClass } from '@memberjunction/global';
 
 @RegisterClass(AutotagBase, 'AutotagCustomSource')
@@ -149,13 +204,14 @@ export class AutotagCustomSource extends AutotagBase {
     return contentItems;
   }
 
-  public async Autotag(contextUser) {
+  public async Autotag(contextUser, onProgress?: AutotagProgressCallback) {
     const contentSourceTypeID = await this.engine.setSubclassContentSourceType(
       'Custom Source', contextUser
     );
     const contentSources = await this.engine.getAllContentSources(
       contextUser, contentSourceTypeID
     );
+    if (contentSources.length === 0) return; // Skip gracefully
     const contentItems = await this.SetContentItemsToProcess(contentSources);
     await this.engine.ExtractTextAndProcessWithLLM(contentItems, contextUser);
   }
@@ -166,12 +222,12 @@ export class AutotagCustomSource extends AutotagBase {
 
 | Entity | Purpose |
 |--------|---------|
-| Content Sources | Configuration for each content source |
+| Content Sources | Configuration for each content source (with optional EmbeddingModelID/VectorIndexID overrides) |
 | Content Items | Individual pieces of content with extracted text |
-| Content Item Tags | AI-generated tags |
+| Content Item Tags | AI-generated tags with relevance weights (0.0--1.0) |
 | Content Item Attributes | Additional extracted metadata |
 | Content Process Runs | Processing history and audit trail |
-| Content Types | Content categorization definitions |
+| Content Types | Content categorization definitions (with default EmbeddingModelID/VectorIndexID) |
 | Content Source Types | Source type definitions |
 | Content File Types | Supported file format definitions |
 
@@ -182,8 +238,12 @@ export class AutotagCustomSource extends AutotagBase {
 | `@memberjunction/core` | Entity system and metadata |
 | `@memberjunction/global` | Class registration |
 | `@memberjunction/core-entities` | Content entity types |
-| `@memberjunction/ai` | LLM integration |
-| `@memberjunction/aiengine` | AI Engine base class |
+| `@memberjunction/ai` | Embedding model integration |
+| `@memberjunction/aiengine` | AI Engine for prompt cache access |
+| `@memberjunction/ai-prompts` | AIPromptRunner for managed prompt execution |
+| `@memberjunction/ai-core-plus` | AIPromptParams types |
+| `@memberjunction/ai-vectors` | TextChunker for content chunking |
+| `@memberjunction/ai-vectordb` | VectorDBBase for vector storage |
 | `pdf-parse` | PDF text extraction |
 | `officeparser` | Office document parsing |
 | `cheerio` | HTML parsing |

package/dist/CloudStorage/generic/CloudStorageBase.d.ts

@@ -1,4 +1,4 @@
-import { AutotagBase } from "../../Core/index.js";
+import { AutotagBase, AutotagProgressCallback } from "../../Core/index.js";
 import { AutotagBaseEngine } from "../../Engine/index.js";
 import { ContentSourceParams } from "../../Engine/index.js";
 import { UserInfo } from "@memberjunction/core";
@@ -22,7 +22,7 @@ export declare abstract class CloudStorageBase extends AutotagBase {
      * @returns - An array of content source items that have been modified or added after the most recent process run for that content source
     */
     abstract SetNewAndModifiedContentItems(contentSourceParams: ContentSourceParams, lastRunDate: Date, contextUser: UserInfo): Promise<MJContentItemEntity[]>;
-    Autotag(contextUser: UserInfo): Promise<void>;
+    Autotag(contextUser: UserInfo, onProgress?: AutotagProgressCallback): Promise<void>;
     SetContentItemsToProcess(contentSources: MJContentSourceEntity[]): Promise<MJContentItemEntity[]>;
 }
 //# sourceMappingURL=CloudStorageBase.d.ts.map

package/dist/CloudStorage/generic/CloudStorageBase.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"CloudStorageBase.d.ts","sourceRoot":"","sources":["../../../src/CloudStorage/generic/CloudStorageBase.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AACjD,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AAI3F,8BAAsB,gBAAiB,SAAQ,WAAW;IACtD,SAAS,CAAC,WAAW,EAAE,QAAQ,CAAC;IAChC,SAAS,CAAC,MAAM,EAAE,iBAAiB,CAAC;IACpC,SAAS,CAAC,mBAAmB,EAAE,MAAM,CAAA;;IAOrC;;MAEE;aACc,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAE7C;;;;;;;;MAQE;aACc,6BAA6B,CAAC,mBAAmB,EAAE,mBAAmB,EAAE,WAAW,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAEpJ,OAAO,CAAC,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ7C,wBAAwB,CAAC,cAAc,EAAE,qBAAqB,EAAE,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;CAyBjH"}
+{"version":3,"file":"CloudStorageBase.d.ts","sourceRoot":"","sources":["../../../src/CloudStorage/generic/CloudStorageBase.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,uBAAuB,EAAE,MAAM,YAAY,CAAC;AAClE,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AACjD,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AAI3F,8BAAsB,gBAAiB,SAAQ,WAAW;IACtD,SAAS,CAAC,WAAW,EAAE,QAAQ,CAAC;IAChC,SAAS,CAAC,MAAM,EAAE,iBAAiB,CAAC;IACpC,SAAS,CAAC,mBAAmB,EAAE,MAAM,CAAA;;IAOrC;;MAEE;aACc,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAE7C;;;;;;;;MAQE;aACc,6BAA6B,CAAC,mBAAmB,EAAE,mBAAmB,EAAE,WAAW,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAEpJ,OAAO,CAAC,WAAW,EAAE,QAAQ,EAAE,UAAU,CAAC,EAAE,uBAAuB,GAAG,OAAO,CAAC,IAAI,CAAC;IAQnF,wBAAwB,CAAC,cAAc,EAAE,qBAAqB,EAAE,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;CAyBjH"}

package/dist/CloudStorage/generic/CloudStorageBase.js

@@ -7,12 +7,12 @@ export class CloudStorageBase extends AutotagBase {
         super();
         this.engine = AutotagBaseEngine.Instance;
     }
-    async Autotag(contextUser) {
+    async Autotag(contextUser, onProgress) {
         this.contextUser = contextUser;
         this.contentSourceTypeID = this.engine.SetSubclassContentSourceType('Cloud Storage');
         const contentSources = await this.engine.getAllContentSources(this.contextUser, this.contentSourceTypeID) || [];
         const contentItemsToProcess = await this.SetContentItemsToProcess(contentSources);
-        await this.engine.ExtractTextAndProcessWithLLM(contentItemsToProcess, this.contextUser);
+        await this.engine.ExtractTextAndProcessWithLLM(contentItemsToProcess, this.contextUser, undefined, undefined, onProgress);
     }
     async SetContentItemsToProcess(contentSources) {
         const contentItemsToProcess = [];

package/dist/CloudStorage/generic/CloudStorageBase.js.map

@@ -1 +1 @@
-{"version":3,"file":"CloudStorageBase.js","sourceRoot":"","sources":["../../../src/CloudStorage/generic/CloudStorageBase.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAIjD,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;AAE9B,MAAM,OAAgB,gBAAiB,SAAQ,WAAW;IAKtD;QACI,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,iBAAiB,CAAC,QAAQ,CAAC;IAC7C,CAAC;IAkBM,KAAK,CAAC,OAAO,CAAC,WAAqB;QACtC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,eAAe,CAAC,CAAC;QACrF,MAAM,cAAc,GAA4B,MAAM,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC;QACzI,MAAM,qBAAqB,GAA0B,MAAM,IAAI,CAAC,wBAAwB,CAAC,cAAc,CAAC,CAAA;QACxG,MAAM,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,qBAAqB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAC5F,CAAC;IAEM,KAAK,CAAC,wBAAwB,CAAC,cAAuC;QACzE,MAAM,qBAAqB,GAA0B,EAAE,CAAA;QAEvD,KAAK,MAAM,aAAa,IAAI,cAAc,EAAE,CAAC;YACzC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAE1B,MAAM,mBAAmB,GAAwB;gBAC7C,eAAe,EAAE,aAAa,CAAC,EAAE;gBACjC,IAAI,EAAE,aAAa,CAAC,IAAI;gBACxB,aAAa,EAAE,aAAa,CAAC,aAAa;gBAC1C,mBAAmB,EAAE,aAAa,CAAC,mBAAmB;gBACtD,iBAAiB,EAAE,aAAa,CAAC,iBAAiB;gBAClD,GAAG,EAAE,aAAa,CAAC,GAAG;aACzB,CAAA;YAED,MAAM,WAAW,GAAS,MAAM,IAAI,CAAC,MAAM,CAAC,2BAA2B,CAAC,mBAAmB,CAAC,eAAe,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YAE/H,IAAI,WAAW,EAAE,CAAC;gBACd,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,mBAAmB,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;gBAClH,qBAAqB,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;YAChD,CAAC;QACL,CAAC;QAED,OAAO,qBAAqB,CAAC;IACjC,CAAC;CACJ"}
+{"version":3,"file":"CloudStorageBase.js","sourceRoot":"","sources":["../../../src/CloudStorage/generic/CloudStorageBase.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAA2B,MAAM,YAAY,CAAC;AAClE,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAIjD,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;AAE9B,MAAM,OAAgB,gBAAiB,SAAQ,WAAW;IAKtD;QACI,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,iBAAiB,CAAC,QAAQ,CAAC;IAC7C,CAAC;IAkBM,KAAK,CAAC,OAAO,CAAC,WAAqB,EAAE,UAAoC;QAC5E,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,eAAe,CAAC,CAAC;QACrF,MAAM,cAAc,GAA4B,MAAM,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC;QACzI,MAAM,qBAAqB,GAA0B,MAAM,IAAI,CAAC,wBAAwB,CAAC,cAAc,CAAC,CAAA;QACxG,MAAM,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,qBAAqB,EAAE,IAAI,CAAC,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;IAC9H,CAAC;IAEM,KAAK,CAAC,wBAAwB,CAAC,cAAuC;QACzE,MAAM,qBAAqB,GAA0B,EAAE,CAAA;QAEvD,KAAK,MAAM,aAAa,IAAI,cAAc,EAAE,CAAC;YACzC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAE1B,MAAM,mBAAmB,GAAwB;gBAC7C,eAAe,EAAE,aAAa,CAAC,EAAE;gBACjC,IAAI,EAAE,aAAa,CAAC,IAAI;gBACxB,aAAa,EAAE,aAAa,CAAC,aAAa;gBAC1C,mBAAmB,EAAE,aAAa,CAAC,mBAAmB;gBACtD,iBAAiB,EAAE,aAAa,CAAC,iBAAiB;gBAClD,GAAG,EAAE,aAAa,CAAC,GAAG;aACzB,CAAA;YAED,MAAM,WAAW,GAAS,MAAM,IAAI,CAAC,MAAM,CAAC,2BAA2B,CAAC,mBAAmB,CAAC,eAAe,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YAE/H,IAAI,WAAW,EAAE,CAAC;gBACd,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,mBAAmB,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;gBAClH,qBAAqB,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;YAChD,CAAC;QACL,CAAC;QAED,OAAO,qBAAqB,CAAC;IACjC,CAAC;CACJ"}

package/dist/CloudStorage/index.d.ts

@@ -1,3 +1,8 @@
 export * from './generic/CloudStorageBase.js';
+export * from './providers/AutotagCloudStorage.js';
+/**
+ * @deprecated Use AutotagCloudStorage instead, which works with any MJ Storage provider.
+ * AutotagAzureBlob is retained for backward compatibility but will be removed in a future version.
+ */
 export * from './providers/AutotagAzureBlob.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/CloudStorage/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/CloudStorage/index.ts"],"names":[],"mappings":"AAAA,cAAc,4BAA4B,CAAA;AAC1C,cAAc,8BAA8B,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/CloudStorage/index.ts"],"names":[],"mappings":"AAAA,cAAc,4BAA4B,CAAA;AAC1C,cAAc,iCAAiC,CAAA;AAE/C;;;GAGG;AACH,cAAc,8BAA8B,CAAA"}

package/dist/CloudStorage/index.js

@@ -1,3 +1,8 @@
 export * from './generic/CloudStorageBase.js';
+export * from './providers/AutotagCloudStorage.js';
+/**
+ * @deprecated Use AutotagCloudStorage instead, which works with any MJ Storage provider.
+ * AutotagAzureBlob is retained for backward compatibility but will be removed in a future version.
+ */
 export * from './providers/AutotagAzureBlob.js';
 //# sourceMappingURL=index.js.map

package/dist/CloudStorage/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/CloudStorage/index.ts"],"names":[],"mappings":"AAAA,cAAc,4BAA4B,CAAA;AAC1C,cAAc,8BAA8B,CAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/CloudStorage/index.ts"],"names":[],"mappings":"AAAA,cAAc,4BAA4B,CAAA;AAC1C,cAAc,iCAAiC,CAAA;AAE/C;;;GAGG;AACH,cAAc,8BAA8B,CAAA"}

package/dist/CloudStorage/providers/AutotagCloudStorage.d.ts

@@ -0,0 +1,61 @@
+import { UserInfo } from '@memberjunction/core';
+import { MJContentItemEntity } from '@memberjunction/core-entities';
+import { AutotagBase, AutotagProgressCallback } from '../../Core/index.js';
+import { MJContentSourceEntity } from '@memberjunction/core-entities';
+/**
+ * Generic cloud storage autotag provider that works with ANY MJ Storage driver.
+ *
+ * Replaces the previous Azure-specific AutotagAzureBlob by delegating file listing
+ * and reading to the MJ Storage abstraction layer (FileStorageBase). This means it
+ * works with Azure Blob, AWS S3, Google Cloud Storage, SharePoint, Dropbox, Box, etc.
+ *
+ * Configuration is driven by ContentSource.Configuration JSON which must include
+ * a `FileStorageProviderKey` matching a registered MJ Storage driver.
+ *
+ * The storage driver is initialized via environment-based config (e.g.,
+ * STORAGE_AZURE_ACCOUNT_NAME, STORAGE_AZURE_ACCOUNT_KEY) which the MJ Storage
+ * drivers read automatically on construction.
+ */
+export declare class AutotagCloudStorage extends AutotagBase {
+    private contextUser;
+    private engine;
+    protected contentSourceTypeID: string;
+    Autotag(contextUser: UserInfo, onProgress?: AutotagProgressCallback): Promise<void>;
+    SetContentItemsToProcess(contentSources: MJContentSourceEntity[]): Promise<MJContentItemEntity[]>;
+    /**
+     * Process a single content source: initialize the storage driver, list files,
+     * detect new/modified files, download and extract text, create ContentItems.
+     */
+    private ProcessContentSource;
+    /**
+     * Parse the ContentSource.Configuration JSON to extract cloud storage config.
+     */
+    private ParseSourceConfig;
+    /**
+     * Create and return a storage driver via ClassFactory using the provider key.
+     */
+    private CreateStorageDriver;
+    /**
+     * List all objects in the storage driver that were modified after lastRunDate.
+     * Optionally filter by file extension.
+     */
+    private ListModifiedObjects;
+    /**
+     * Download a file, extract text, and create/update a ContentItem.
+     */
+    private ProcessSingleFile;
+    /**
+     * Load existing ContentItems for a source, keyed by lowercase URL for upsert lookups.
+     */
+    private LoadExistingContentItems;
+    /**
+     * Extract text from a file buffer based on file extension.
+     * Delegates to the engine's built-in parsers for PDF and Office documents.
+     */
+    private ExtractTextFromBuffer;
+    /**
+     * Get the lowercase file extension including the dot (e.g., '.pdf').
+     */
+    private GetFileExtension;
+}
+//# sourceMappingURL=AutotagCloudStorage.d.ts.map

package/dist/CloudStorage/providers/AutotagCloudStorage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"AutotagCloudStorage.d.ts","sourceRoot":"","sources":["../../../src/CloudStorage/providers/AutotagCloudStorage.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAA0C,MAAM,sBAAsB,CAAC;AACxF,OAAO,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AAEpE,OAAO,EAAE,WAAW,EAAE,uBAAuB,EAAE,MAAM,YAAY,CAAC;AAElE,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAgBtE;;;;;;;;;;;;;GAaG;AACH,qBACa,mBAAoB,SAAQ,WAAW;IAChD,OAAO,CAAC,WAAW,CAAY;IAC/B,OAAO,CAAC,MAAM,CAAqB;IACnC,SAAS,CAAC,mBAAmB,EAAG,MAAM,CAAC;IAE1B,OAAO,CAAC,WAAW,EAAE,QAAQ,EAAE,UAAU,CAAC,EAAE,uBAAuB,GAAG,OAAO,CAAC,IAAI,CAAC;IAenF,wBAAwB,CAAC,cAAc,EAAE,qBAAqB,EAAE,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAgB9G;;;OAGG;YACW,oBAAoB;IA4ClC;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAsBzB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAgB3B;;;OAGG;YACW,mBAAmB;IAsBjC;;OAEG;YACW,iBAAiB;IAsD/B;;OAEG;YACW,wBAAwB;IAmBtC;;;OAGG;YACW,qBAAqB;IAoBnC;;OAEG;IACH,OAAO,CAAC,gBAAgB;CAK3B"}

package/dist/CloudStorage/providers/AutotagCloudStorage.js

@@ -0,0 +1,256 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+import { RegisterClass } from '@memberjunction/global';
+import { MJGlobal } from '@memberjunction/global';
+import { Metadata, RunView, LogStatus, LogError } from '@memberjunction/core';
+import { FileStorageBase } from '@memberjunction/storage';
+import { AutotagBase } from '../../Core/index.js';
+import { AutotagBaseEngine } from '../../Engine/index.js';
+/**
+ * Generic cloud storage autotag provider that works with ANY MJ Storage driver.
+ *
+ * Replaces the previous Azure-specific AutotagAzureBlob by delegating file listing
+ * and reading to the MJ Storage abstraction layer (FileStorageBase). This means it
+ * works with Azure Blob, AWS S3, Google Cloud Storage, SharePoint, Dropbox, Box, etc.
+ *
+ * Configuration is driven by ContentSource.Configuration JSON which must include
+ * a `FileStorageProviderKey` matching a registered MJ Storage driver.
+ *
+ * The storage driver is initialized via environment-based config (e.g.,
+ * STORAGE_AZURE_ACCOUNT_NAME, STORAGE_AZURE_ACCOUNT_KEY) which the MJ Storage
+ * drivers read automatically on construction.
+ */
+let AutotagCloudStorage = class AutotagCloudStorage extends AutotagBase {
+    async Autotag(contextUser, onProgress) {
+        this.contextUser = contextUser;
+        this.engine = AutotagBaseEngine.Instance;
+        this.contentSourceTypeID = this.engine.SetSubclassContentSourceType('Cloud Storage');
+        const contentSources = await this.engine.getAllContentSources(this.contextUser, this.contentSourceTypeID);
+        const contentItemsToProcess = await this.SetContentItemsToProcess(contentSources);
+        if (contentItemsToProcess.length > 0) {
+            await this.engine.ExtractTextAndProcessWithLLM(contentItemsToProcess, this.contextUser, undefined, undefined, onProgress);
+        }
+        else {
+            LogStatus('AutotagCloudStorage: no new or modified files to process');
+        }
+    }
+    async SetContentItemsToProcess(contentSources) {
+        const contentItemsToProcess = [];
+        for (const contentSource of contentSources) {
+            try {
+                const items = await this.ProcessContentSource(contentSource);
+                contentItemsToProcess.push(...items);
+            }
+            catch (e) {
+                const msg = e instanceof Error ? e.message : String(e);
+                LogError(`AutotagCloudStorage: failed to process source "${contentSource.Name}": ${msg}`);
+            }
+        }
+        return contentItemsToProcess;
+    }
+    /**
+     * Process a single content source: initialize the storage driver, list files,
+     * detect new/modified files, download and extract text, create ContentItems.
+     */
+    async ProcessContentSource(contentSource) {
+        const config = this.ParseSourceConfig(contentSource);
+        if (!config)
+            return [];
+        const driver = this.CreateStorageDriver(config.FileStorageProviderKey);
+        if (!driver)
+            return [];
+        const lastRunDate = await this.engine.getContentSourceLastRunDate(contentSource.ID, this.contextUser);
+        const prefix = config.PathPrefix ?? '';
+        const objects = await this.ListModifiedObjects(driver, prefix, lastRunDate, config.IncludeExtensions);
+        if (objects.length === 0) {
+            LogStatus(`AutotagCloudStorage: no modified files in source "${contentSource.Name}" since ${lastRunDate.toISOString()}`);
+            return [];
+        }
+        LogStatus(`AutotagCloudStorage: found ${objects.length} new/modified files in source "${contentSource.Name}"`);
+        // Load existing content items for this source to enable upsert by URL
+        const existingItems = await this.LoadExistingContentItems(contentSource.ID);
+        const contentSourceParams = {
+            contentSourceID: contentSource.ID,
+            name: contentSource.Name ?? '',
+            ContentTypeID: contentSource.ContentTypeID,
+            ContentSourceTypeID: contentSource.ContentSourceTypeID,
+            ContentFileTypeID: contentSource.ContentFileTypeID,
+            URL: contentSource.URL
+        };
+        const items = [];
+        for (const obj of objects) {
+            try {
+                const item = await this.ProcessSingleFile(driver, obj, contentSourceParams, existingItems);
+                if (item)
+                    items.push(item);
+            }
+            catch (e) {
+                const msg = e instanceof Error ? e.message : String(e);
+                LogError(`AutotagCloudStorage: failed to process file "${obj.fullPath}": ${msg}`);
+            }
+        }
+        return items;
+    }
+    /**
+     * Parse the ContentSource.Configuration JSON to extract cloud storage config.
+     */
+    ParseSourceConfig(contentSource) {
+        const configObj = contentSource.ConfigurationObject;
+        if (!configObj) {
+            // Fall back: try to infer from URL or legacy setup
+            LogError(`AutotagCloudStorage: source "${contentSource.Name}" has no Configuration JSON. Set FileStorageProviderKey in the Configuration field.`);
+            return null;
+        }
+        const raw = configObj;
+        const providerKey = raw['FileStorageProviderKey'];
+        if (!providerKey) {
+            LogError(`AutotagCloudStorage: source "${contentSource.Name}" Configuration is missing FileStorageProviderKey`);
+            return null;
+        }
+        return {
+            FileStorageProviderKey: providerKey,
+            PathPrefix: raw['PathPrefix'] ?? undefined,
+            IncludeExtensions: raw['IncludeExtensions'] ?? undefined,
+        };
+    }
+    /**
+     * Create and return a storage driver via ClassFactory using the provider key.
+     */
+    CreateStorageDriver(providerKey) {
+        const driver = MJGlobal.Instance.ClassFactory.CreateInstance(FileStorageBase, providerKey);
+        if (!driver) {
+            LogError(`AutotagCloudStorage: no storage driver registered for key "${providerKey}". Ensure the driver is loaded (e.g., import '@memberjunction/storage').`);
+            return null;
+        }
+        if (!driver.IsConfigured) {
+            LogError(`AutotagCloudStorage: storage driver "${providerKey}" is not configured. Check environment variables or mj.config.cjs storage settings.`);
+            return null;
+        }
+        return driver;
+    }
+    /**
+     * List all objects in the storage driver that were modified after lastRunDate.
+     * Optionally filter by file extension.
+     */
+    async ListModifiedObjects(driver, prefix, lastRunDate, includeExtensions) {
+        const result = await driver.ListObjects(prefix);
+        const extSet = includeExtensions?.length
+            ? new Set(includeExtensions.map(ext => ext.toLowerCase()))
+            : null;
+        return result.objects.filter(obj => {
+            if (obj.isDirectory)
+                return false;
+            if (obj.lastModified <= lastRunDate)
+                return false;
+            if (extSet) {
+                const ext = this.GetFileExtension(obj.name);
+                if (!extSet.has(ext))
+                    return false;
+            }
+            return true;
+        });
+    }
+    /**
+     * Download a file, extract text, and create/update a ContentItem.
+     */
+    async ProcessSingleFile(driver, obj, contentSourceParams, existingItems) {
+        // Download the file content
+        const buffer = await driver.GetObject({ fullPath: obj.fullPath });
+        // Extract text using the engine's built-in parsers (PDF, DOCX, etc.)
+        const text = await this.ExtractTextFromBuffer(buffer, obj.name);
+        if (!text || text.trim().length === 0) {
+            LogStatus(`AutotagCloudStorage: no extractable text from "${obj.fullPath}", skipping`);
+            return null;
+        }
+        const checksum = await this.engine.getChecksumFromText(text);
+        const urlKey = obj.fullPath.toLowerCase();
+        // Check for existing content item by URL
+        const existing = existingItems.get(urlKey);
+        if (existing && existing.Checksum === checksum) {
+            return null; // Content unchanged
+        }
+        const md = new Metadata();
+        let contentItem;
+        if (existing) {
+            contentItem = existing;
+        }
+        else {
+            contentItem = await md.GetEntityObject('MJ: Content Items', this.contextUser);
+            contentItem.NewRecord();
+            contentItem.ContentSourceID = contentSourceParams.contentSourceID;
+            contentItem.ContentTypeID = contentSourceParams.ContentTypeID;
+            contentItem.ContentSourceTypeID = contentSourceParams.ContentSourceTypeID;
+            contentItem.ContentFileTypeID = contentSourceParams.ContentFileTypeID;
+        }
+        contentItem.Name = obj.name;
+        contentItem.Description = this.engine.GetContentItemDescription(contentSourceParams);
+        contentItem.URL = obj.fullPath;
+        contentItem.Text = text;
+        contentItem.Checksum = checksum;
+        const saved = await contentItem.Save();
+        if (!saved) {
+            throw new Error(`Failed to save ContentItem for "${obj.fullPath}"`);
+        }
+        existingItems.set(urlKey, contentItem);
+        return contentItem;
+    }
+    /**
+     * Load existing ContentItems for a source, keyed by lowercase URL for upsert lookups.
+     */
+    async LoadExistingContentItems(contentSourceID) {
+        const rv = new RunView();
+        const result = await rv.RunView({
+            EntityName: 'MJ: Content Items',
+            ExtraFilter: `ContentSourceID='${contentSourceID}'`,
+            ResultType: 'entity_object'
+        }, this.contextUser);
+        const map = new Map();
+        if (result.Success) {
+            for (const ci of result.Results) {
+                if (ci.URL) {
+                    map.set(ci.URL.toLowerCase(), ci);
+                }
+            }
+        }
+        return map;
+    }
+    /**
+     * Extract text from a file buffer based on file extension.
+     * Delegates to the engine's built-in parsers for PDF and Office documents.
+     */
+    async ExtractTextFromBuffer(buffer, fileName) {
+        const ext = this.GetFileExtension(fileName);
+        if (ext === '.pdf' || ext === '.docx' || ext === '.doc' || ext === '.pptx' || ext === '.xlsx') {
+            return this.engine.parsePDF(buffer);
+        }
+        if (ext === '.txt' || ext === '.md' || ext === '.csv' || ext === '.json' || ext === '.xml' || ext === '.html') {
+            return buffer.toString('utf-8');
+        }
+        // Attempt parsePDF as fallback for unknown formats (officeparser handles many formats)
+        try {
+            return await this.engine.parsePDF(buffer);
+        }
+        catch {
+            LogStatus(`AutotagCloudStorage: unsupported file format "${ext}" for "${fileName}"`);
+            return '';
+        }
+    }
+    /**
+     * Get the lowercase file extension including the dot (e.g., '.pdf').
+     */
+    GetFileExtension(fileName) {
+        const lastDot = fileName.lastIndexOf('.');
+        if (lastDot < 0)
+            return '';
+        return fileName.substring(lastDot).toLowerCase();
+    }
+};
+AutotagCloudStorage = __decorate([
+    RegisterClass(AutotagBase, 'AutotagCloudStorage')
+], AutotagCloudStorage);
+export { AutotagCloudStorage };
+//# sourceMappingURL=AutotagCloudStorage.js.map

package/dist/CloudStorage/providers/AutotagCloudStorage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AutotagCloudStorage.js","sourceRoot":"","sources":["../../../src/CloudStorage/providers/AutotagCloudStorage.ts"],"names":[],"mappings":";;;;;;AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAClD,OAAO,EAAY,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAExF,OAAO,EAAE,eAAe,EAAyB,MAAM,yBAAyB,CAAC;AACjF,OAAO,EAAE,WAAW,EAA2B,MAAM,YAAY,CAAC;AAClE,OAAO,EAAE,iBAAiB,EAAuB,MAAM,cAAc,CAAC;AAiBtE;;;;;;;;;;;;;GAaG;AAEI,IAAM,mBAAmB,GAAzB,MAAM,mBAAoB,SAAQ,WAAW;IAKzC,KAAK,CAAC,OAAO,CAAC,WAAqB,EAAE,UAAoC;QAC5E,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,MAAM,GAAG,iBAAiB,CAAC,QAAQ,CAAC;QACzC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,eAAe,CAAC,CAAC;QAErF,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC1G,MAAM,qBAAqB,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,cAAc,CAAC,CAAC;QAElF,IAAI,qBAAqB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,MAAM,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,qBAAqB,EAAE,IAAI,CAAC,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;QAC9H,CAAC;aAAM,CAAC;YACJ,SAAS,CAAC,0DAA0D,CAAC,CAAC;QAC1E,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,wBAAwB,CAAC,cAAuC;QACzE,MAAM,qBAAqB,GAA0B,EAAE,CAAC;QAExD,KAAK,MAAM,aAAa,IAAI,cAAc,EAAE,CAAC;YACzC,IAAI,CAAC;gBACD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC,CAAC;gBAC7D,qBAAqB,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;YACzC,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,MAAM,GAAG,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBACvD,QAAQ,CAAC,kDAAkD,aAAa,CAAC,IAAI,MAAM,GAAG,EAAE,CAAC,CAAC;YAC9F,CAAC;QACL,CAAC;QAED,OAAO,qBAAqB,CAAC;IACjC,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,oBAAoB,CAAC,aAAoC;QACnE,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;QACrD,IAAI,CAAC,MAAM;YAAE,OAAO,EAAE,CAAC;QAEvB,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC;QACvE,IAAI,CAAC,MAAM;YAAE,OAAO,EAAE,CAAC;QAEvB,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,2BAA2B,CAAC,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACtG,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC;QACvC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,CAAC,iBAAiB,CAAC,CAAC;QAEtG,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,SAAS,CAAC,qDAAqD,aAAa,CAAC,IAAI,WAAW,WAAW,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;YACzH,OAAO,EAAE,CAAC;QACd,CAAC;QAED,SAAS,CAAC,8BAA8B,OAAO,CAAC,MAAM,kCAAkC,aAAa,CAAC,IAAI,GAAG,CAAC,CAAC;QAE/G,sEAAsE;QACtE,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;QAE5E,MAAM,mBAAmB,GAAwB;YAC7C,eAAe,EAAE,aAAa,CAAC,EAAE;YACjC,IAAI,EAAE,aAAa,CAAC,IAAI,IAAI,EAAE;YAC9B,aAAa,EAAE,aAAa,CAAC,aAAa;YAC1C,mBAAmB,EAAE,aAAa,CAAC,mBAAmB;YACtD,iBAAiB,EAAE,aAAa,CAAC,iBAAiB;YAClD,GAAG,EAAE,aAAa,CAAC,GAAG;SACzB,CAAC;QAEF,MAAM,KAAK,GAA0B,EAAE,CAAC;QACxC,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YACxB,IAAI,CAAC;gBACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,GAAG,EAAE,mBAAmB,EAAE,aAAa,CAAC,CAAC;gBAC3F,IAAI,IAAI;oBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,MAAM,GAAG,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBACvD,QAAQ,CAAC,gDAAgD,GAAG,CAAC,QAAQ,MAAM,GAAG,EAAE,CAAC,CAAC;YACtF,CAAC;QACL,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,iBAAiB,CAAC,aAAoC;QAC1D,MAAM,SAAS,GAAG,aAAa,CAAC,mBAAmB,CAAC;QACpD,IAAI,CAAC,SAAS,EAAE,CAAC;YACb,mDAAmD;YACnD,QAAQ,CAAC,gCAAgC,aAAa,CAAC,IAAI,qFAAqF,CAAC,CAAC;YAClJ,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,MAAM,GAAG,GAAG,SAAoC,CAAC;QACjD,MAAM,WAAW,GAAG,GAAG,CAAC,wBAAwB,CAAuB,CAAC;QACxE,IAAI,CAAC,WAAW,EAAE,CAAC;YACf,QAAQ,CAAC,gCAAgC,aAAa,CAAC,IAAI,mDAAmD,CAAC,CAAC;YAChH,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,OAAO;YACH,sBAAsB,EAAE,WAAW;YACnC,UAAU,EAAG,GAAG,CAAC,YAAY,CAAY,IAAI,SAAS;YACtD,iBAAiB,EAAG,GAAG,CAAC,mBAAmB,CAAc,IAAI,SAAS;SACzE,CAAC;IACN,CAAC;IAED;;OAEG;IACK,mBAAmB,CAAC,WAAmB;QAC3C,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,cAAc,CACxD,eAAe,EACf,WAAW,CACd,CAAC;QACF,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,QAAQ,CAAC,8DAA8D,WAAW,0EAA0E,CAAC,CAAC;YAC9J,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;YACvB,QAAQ,CAAC,wCAAwC,WAAW,qFAAqF,CAAC,CAAC;YACnJ,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,mBAAmB,CAC7B,MAAuB,EACvB,MAAc,EACd,WAAiB,EACjB,iBAA4B;QAE5B,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAChD,MAAM,MAAM,GAAG,iBAAiB,EAAE,MAAM;YACpC,CAAC,CAAC,IAAI,GAAG,CAAC,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC;YAC1D,CAAC,CAAC,IAAI,CAAC;QAEX,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;YAC/B,IAAI,GAAG,CAAC,WAAW;gBAAE,OAAO,KAAK,CAAC;YAClC,IAAI,GAAG,CAAC,YAAY,IAAI,WAAW;gBAAE,OAAO,KAAK,CAAC;YAClD,IAAI,MAAM,EAAE,CAAC;gBACT,MAAM,GAAG,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAC5C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;oBAAE,OAAO,KAAK,CAAC;YACvC,CAAC;YACD,OAAO,IAAI,CAAC;QAChB,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,iBAAiB,CAC3B,MAAuB,EACvB,GAA0B,EAC1B,mBAAwC,EACxC,aAA+C;QAE/C,4BAA4B;QAC5B,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;QAElE,qEAAqE;QACrE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;QAChE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpC,SAAS,CAAC,kDAAkD,GAAG,CAAC,QAAQ,aAAa,CAAC,CAAC;YACvF,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC;QAC7D,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QAE1C,yCAAyC;QACzC,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC3C,IAAI,QAAQ,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC7C,OAAO,IAAI,CAAC,CAAC,oBAAoB;QACrC,CAAC;QAED,MAAM,EAAE,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC1B,IAAI,WAAgC,CAAC;QAErC,IAAI,QAAQ,EAAE,CAAC;YACX,WAAW,GAAG,QAAQ,CAAC;QAC3B,CAAC;aAAM,CAAC;YACJ,WAAW,GAAG,MAAM,EAAE,CAAC,eAAe,CAAsB,mBAAmB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YACnG,WAAW,CAAC,SAAS,EAAE,CAAC;YACxB,WAAW,CAAC,eAAe,GAAG,mBAAmB,CAAC,eAAe,CAAC;YAClE,WAAW,CAAC,aAAa,GAAG,mBAAmB,CAAC,aAAa,CAAC;YAC9D,WAAW,CAAC,mBAAmB,GAAG,mBAAmB,CAAC,mBAAmB,CAAC;YAC1E,WAAW,CAAC,iBAAiB,GAAG,mBAAmB,CAAC,iBAAiB,CAAC;QAC1E,CAAC;QAED,WAAW,CAAC,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;QAC5B,WAAW,CAAC,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;QACrF,WAAW,CAAC,GAAG,GAAG,GAAG,CAAC,QAAQ,CAAC;QAC/B,WAAW,CAAC,IAAI,GAAG,IAAI,CAAC;QACxB,WAAW,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAEhC,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,CAAC;QACvC,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,mCAAmC,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAC;QACxE,CAAC;QAED,aAAa,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QACvC,OAAO,WAAW,CAAC;IACvB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,wBAAwB,CAAC,eAAuB;QAC1D,MAAM,EAAE,GAAG,IAAI,OAAO,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAsB;YACjD,UAAU,EAAE,mBAAmB;YAC/B,WAAW,EAAE,oBAAoB,eAAe,GAAG;YACnD,UAAU,EAAE,eAAe;SAC9B,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAErB,MAAM,GAAG,GAAG,IAAI,GAAG,EAA+B,CAAC;QACnD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACjB,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBAC9B,IAAI,EAAE,CAAC,GAAG,EAAE,CAAC;oBACT,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtC,CAAC;YACL,CAAC;QACL,CAAC;QACD,OAAO,GAAG,CAAC;IACf,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,qBAAqB,CAAC,MAAc,EAAE,QAAgB;QAChE,MAAM,GAAG,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAE5C,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,OAAO,EAAE,CAAC;YAC5F,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,CAAC;QAED,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,OAAO,EAAE,CAAC;YAC5G,OAAO,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACpC,CAAC;QAED,uFAAuF;QACvF,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC9C,CAAC;QAAC,MAAM,CAAC;YACL,SAAS,CAAC,iDAAiD,GAAG,UAAU,QAAQ,GAAG,CAAC,CAAC;YACrF,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,QAAgB;QACrC,MAAM,OAAO,GAAG,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAC1C,IAAI,OAAO,GAAG,CAAC;YAAE,OAAO,EAAE,CAAC;QAC3B,OAAO,QAAQ,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;IACrD,CAAC;CACJ,CAAA;AAzQY,mBAAmB;IAD/B,aAAa,CAAC,WAAW,EAAE,qBAAqB,CAAC;GACrC,mBAAmB,CAyQ/B"}

package/dist/Core/generic/AutotagBase.d.ts

@@ -1,7 +1,15 @@
 import { UserInfo } from '@memberjunction/core';
 import { MJContentSourceEntity, MJContentItemEntity } from '@memberjunction/core-entities';
+/** Progress callback for per-item updates during autotagging */
+export type AutotagProgressCallback = (processed: number, total: number, currentItem?: string) => void;
 export declare abstract class AutotagBase {
     abstract SetContentItemsToProcess(contentSources: MJContentSourceEntity[]): Promise<MJContentItemEntity[]>;
-    abstract Autotag(contextUser: UserInfo): Promise<void>;
+    /**
+     * Run autotagging for this source type.
+     * @param contextUser - The user context for server-side operations
+     * @param onProgress - Optional progress callback
+     * @param contentSourceIDs - Optional filter: only process these specific source IDs. If omitted, processes all sources for this type.
+     */
+    abstract Autotag(contextUser: UserInfo, onProgress?: AutotagProgressCallback, contentSourceIDs?: string[]): Promise<void>;
 }
 //# sourceMappingURL=AutotagBase.d.ts.map

package/dist/Core/generic/AutotagBase.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"AutotagBase.d.ts","sourceRoot":"","sources":["../../../src/Core/generic/AutotagBase.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AAE3F,8BAAsB,WAAW;aACb,wBAAwB,CAAC,cAAc,EAAE,qBAAqB,EAAE,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;aACjG,OAAO,CAAC,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;CAChE"}
+{"version":3,"file":"AutotagBase.d.ts","sourceRoot":"","sources":["../../../src/Core/generic/AutotagBase.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AAE3F,gEAAgE;AAChE,MAAM,MAAM,uBAAuB,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,KAAK,IAAI,CAAC;AAEvG,8BAAsB,WAAW;aACb,wBAAwB,CAAC,cAAc,EAAE,qBAAqB,EAAE,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IACjH;;;;;OAKG;aACa,OAAO,CAAC,WAAW,EAAE,QAAQ,EAAE,UAAU,CAAC,EAAE,uBAAuB,EAAE,gBAAgB,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CACnI"}

package/dist/Core/generic/AutotagBase.js.map

@@ -1 +1 @@
-{"version":3,"file":"AutotagBase.js","sourceRoot":"","sources":["../../../src/Core/generic/AutotagBase.ts"],"names":[],"mappings":"AAGA,MAAM,OAAgB,WAAW;CAGhC"}
+{"version":3,"file":"AutotagBase.js","sourceRoot":"","sources":["../../../src/Core/generic/AutotagBase.ts"],"names":[],"mappings":"AAMA,MAAM,OAAgB,WAAW;CAShC"}