@memberjunction/content-autotagging 3.4.0 → 4.0.0

package/dist/CloudStorage/generic/CloudStorageBase.d.ts

@@ -0,0 +1,28 @@
+import { AutotagBase } from "../../Core/index.js";
+import { AutotagBaseEngine } from "../../Engine/index.js";
+import { ContentSourceParams } from "../../Engine/index.js";
+import { UserInfo } from "@memberjunction/core";
+import { ContentSourceEntity, ContentItemEntity } from "@memberjunction/core-entities";
+export declare abstract class CloudStorageBase extends AutotagBase {
+    protected contextUser: UserInfo;
+    protected engine: AutotagBaseEngine;
+    protected contentSourceTypeID: string;
+    constructor();
+    /**
+     * Abstract method to be implemented in the subclass. This method authenticates the user to the cloud storage.
+    */
+    abstract Authenticate(): Promise<void>;
+    /**
+     * Abstract method to be implemented in the subclass. Given a list of content sources, this method should return a list
+     * of content source items that have been modified or added after the most recent process run for that content source.
+     * Each cloud storage provider has a different way of accessing files and checking for modified or added items.
+     * @param contentSources - An array of content sources to check for modified or added content source items
+     * @param lastRunDate - The date of the most recent process run for the content source
+     * @param contextUser - The user context for the autotagging process
+     * @returns - An array of content source items that have been modified or added after the most recent process run for that content source
+    */
+    abstract SetNewAndModifiedContentItems(contentSourceParams: ContentSourceParams, lastRunDate: Date, contextUser: UserInfo): Promise<ContentItemEntity[]>;
+    Autotag(contextUser: UserInfo): Promise<void>;
+    SetContentItemsToProcess(contentSources: ContentSourceEntity[]): Promise<ContentItemEntity[]>;
+}
+//# sourceMappingURL=CloudStorageBase.d.ts.map

package/dist/CloudStorage/generic/CloudStorageBase.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"CloudStorageBase.d.ts","sourceRoot":"","sources":["../../../src/CloudStorage/generic/CloudStorageBase.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AACjD,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAIvF,8BAAsB,gBAAiB,SAAQ,WAAW;IACtD,SAAS,CAAC,WAAW,EAAE,QAAQ,CAAC;IAChC,SAAS,CAAC,MAAM,EAAE,iBAAiB,CAAC;IACpC,SAAS,CAAC,mBAAmB,EAAE,MAAM,CAAA;;IAOrC;;MAEE;aACc,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAE7C;;;;;;;;MAQE;aACc,6BAA6B,CAAC,mBAAmB,EAAE,mBAAmB,EAAE,WAAW,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;IAElJ,OAAO,CAAC,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ7C,wBAAwB,CAAC,cAAc,EAAE,mBAAmB,EAAE,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;CAyB7G"}

package/dist/CloudStorage/generic/CloudStorageBase.js

@@ -0,0 +1,38 @@
+import { AutotagBase } from "../../Core/index.js";
+import { AutotagBaseEngine } from "../../Engine/index.js";
+import dotenv from 'dotenv';
+dotenv.config({ quiet: true });
+export class CloudStorageBase extends AutotagBase {
+    constructor() {
+        super();
+        this.engine = AutotagBaseEngine.Instance;
+    }
+    async Autotag(contextUser) {
+        this.contextUser = contextUser;
+        this.contentSourceTypeID = await this.engine.setSubclassContentSourceType('Cloud Storage', this.contextUser);
+        const contentSources = await this.engine.getAllContentSources(this.contextUser, this.contentSourceTypeID) || [];
+        const contentItemsToProcess = await this.SetContentItemsToProcess(contentSources);
+        await this.engine.ExtractTextAndProcessWithLLM(contentItemsToProcess, this.contextUser);
+    }
+    async SetContentItemsToProcess(contentSources) {
+        const contentItemsToProcess = [];
+        for (const contentSource of contentSources) {
+            await this.Authenticate();
+            const contentSourceParams = {
+                contentSourceID: contentSource.ID,
+                name: contentSource.Name,
+                ContentTypeID: contentSource.ContentTypeID,
+                ContentSourceTypeID: contentSource.ContentSourceTypeID,
+                ContentFileTypeID: contentSource.ContentFileTypeID,
+                URL: contentSource.URL
+            };
+            const lastRunDate = await this.engine.getContentSourceLastRunDate(contentSourceParams.contentSourceID, this.contextUser);
+            if (lastRunDate) {
+                const contentItems = await this.SetNewAndModifiedContentItems(contentSourceParams, lastRunDate, this.contextUser);
+                contentItemsToProcess.push(...contentItems);
+            }
+        }
+        return contentItemsToProcess;
+    }
+}
+//# sourceMappingURL=CloudStorageBase.js.map

package/dist/CloudStorage/generic/CloudStorageBase.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CloudStorageBase.js","sourceRoot":"","sources":["../../../src/CloudStorage/generic/CloudStorageBase.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAIjD,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;AAE9B,MAAM,OAAgB,gBAAiB,SAAQ,WAAW;IAKtD;QACI,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,iBAAiB,CAAC,QAAQ,CAAC;IAC7C,CAAC;IAkBM,KAAK,CAAC,OAAO,CAAC,WAAqB;QACtC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,mBAAmB,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,eAAe,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAC7G,MAAM,cAAc,GAA0B,MAAM,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC;QACvI,MAAM,qBAAqB,GAAwB,MAAM,IAAI,CAAC,wBAAwB,CAAC,cAAc,CAAC,CAAA;QACtG,MAAM,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,qBAAqB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAC5F,CAAC;IAEM,KAAK,CAAC,wBAAwB,CAAC,cAAqC;QACvE,MAAM,qBAAqB,GAAwB,EAAE,CAAA;QAErD,KAAK,MAAM,aAAa,IAAI,cAAc,EAAE,CAAC;YACzC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAE1B,MAAM,mBAAmB,GAAwB;gBAC7C,eAAe,EAAE,aAAa,CAAC,EAAE;gBACjC,IAAI,EAAE,aAAa,CAAC,IAAI;gBACxB,aAAa,EAAE,aAAa,CAAC,aAAa;gBAC1C,mBAAmB,EAAE,aAAa,CAAC,mBAAmB;gBACtD,iBAAiB,EAAE,aAAa,CAAC,iBAAiB;gBAClD,GAAG,EAAE,aAAa,CAAC,GAAG;aACzB,CAAA;YAED,MAAM,WAAW,GAAS,MAAM,IAAI,CAAC,MAAM,CAAC,2BAA2B,CAAC,mBAAmB,CAAC,eAAe,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YAE/H,IAAI,WAAW,EAAE,CAAC;gBACd,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,mBAAmB,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;gBAClH,qBAAqB,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;YAChD,CAAC;QACL,CAAC;QAED,OAAO,qBAAqB,CAAC;IACjC,CAAC;CACJ"}

package/dist/CloudStorage/index.d.ts

@@ -0,0 +1,3 @@
+export * from './generic/CloudStorageBase.js';
+export * from './providers/AutotagAzureBlob.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/CloudStorage/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/CloudStorage/index.ts"],"names":[],"mappings":"AAAA,cAAc,4BAA4B,CAAA;AAC1C,cAAc,8BAA8B,CAAA"}

package/dist/CloudStorage/index.js

@@ -0,0 +1,3 @@
+export * from './generic/CloudStorageBase.js';
+export * from './providers/AutotagAzureBlob.js';
+//# sourceMappingURL=index.js.map

package/dist/CloudStorage/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/CloudStorage/index.ts"],"names":[],"mappings":"AAAA,cAAc,4BAA4B,CAAA;AAC1C,cAAc,8BAA8B,CAAA"}

package/dist/CloudStorage/providers/AutotagAzureBlob.d.ts

@@ -0,0 +1,20 @@
+import { CloudStorageBase } from "../generic/CloudStorageBase.js";
+import { UserInfo } from "@memberjunction/core";
+import { ContentItemEntity } from "@memberjunction/core-entities";
+import { ContentSourceParams } from "../../Engine/index.js";
+export declare class AutotagAzureBlob extends CloudStorageBase {
+    private blobServiceClient;
+    private containerClient;
+    private connectionString;
+    private containerName;
+    constructor(connectionString: string, containerName: string);
+    /**
+     * Implemented abstract method from the CloudStorageBase class for cloud storage authentication. This method authenticates the user to the Azure Blob Storage.
+     * @returns void
+    */
+    Authenticate(): Promise<void>;
+    SetNewAndModifiedContentItems(contentSourceParams: ContentSourceParams, lastRunDate: Date, contextUser: UserInfo, prefix?: string): Promise<ContentItemEntity[]>;
+    extractText(file: string): Promise<string>;
+    streamToBuffer(readableStream: NodeJS.ReadableStream): Promise<Buffer>;
+}
+//# sourceMappingURL=AutotagAzureBlob.d.ts.map

package/dist/CloudStorage/providers/AutotagAzureBlob.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"AutotagAzureBlob.d.ts","sourceRoot":"","sources":["../../../src/CloudStorage/providers/AutotagAzureBlob.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAGhD,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAGlE,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAGnD,qBAAa,gBAAiB,SAAQ,gBAAgB;IAClD,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,aAAa,CAAS;gBAElB,gBAAgB,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM;IAO3D;;;MAGE;IACW,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAc7B,6BAA6B,CAAC,mBAAmB,EAAE,mBAAmB,EAAE,WAAW,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,SAAG,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;IAwC1J,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAQ1C,cAAc,CAAC,cAAc,EAAE,MAAM,CAAC,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC;CAYtF"}

package/dist/CloudStorage/providers/AutotagAzureBlob.js

@@ -0,0 +1,86 @@
+import { CloudStorageBase } from "../generic/CloudStorageBase.js";
+import { BlobServiceClient } from '@azure/storage-blob';
+import dotenv from 'dotenv';
+import { Metadata } from "@memberjunction/core";
+import path from "path";
+dotenv.config({ quiet: true });
+export class AutotagAzureBlob extends CloudStorageBase {
+    constructor(connectionString, containerName) {
+        super();
+        this.connectionString = connectionString;
+        this.containerName = containerName;
+    }
+    /**
+     * Implemented abstract method from the CloudStorageBase class for cloud storage authentication. This method authenticates the user to the Azure Blob Storage.
+     * @returns void
+    */
+    async Authenticate() {
+        try {
+            this.blobServiceClient = BlobServiceClient.fromConnectionString(this.connectionString);
+            for await (const container of this.blobServiceClient.listContainers()) {
+                console.log(`Container: ${container.name}`);
+            }
+            this.containerClient = this.blobServiceClient.getContainerClient(this.containerName);
+        }
+        catch (error) {
+            console.error(error);
+            throw new Error('Error authenticating to Azure Blob Storage');
+        }
+    }
+    async SetNewAndModifiedContentItems(contentSourceParams, lastRunDate, contextUser, prefix = '') {
+        const contentItemsToProcess = [];
+        for await (const blob of this.containerClient.listBlobsFlat()) {
+            const filePath = path.join(this.containerName, blob.name);
+            if (blob.properties.createdOn && blob.properties.createdOn > lastRunDate) {
+                // The file has been created, add a new record for this file
+                const md = new Metadata();
+                const contentItem = await md.GetEntityObject('Content Items', contextUser);
+                const text = await this.extractText(blob.name);
+                contentItem.ContentSourceID = contentSourceParams.contentSourceID;
+                contentItem.Name = blob.name;
+                contentItem.Description = await this.engine.getContentItemDescription(contentSourceParams, contextUser);
+                contentItem.URL = filePath;
+                contentItem.ContentTypeID = contentSourceParams.ContentTypeID;
+                contentItem.ContentSourceTypeID = contentSourceParams.ContentSourceTypeID;
+                contentItem.ContentFileTypeID = contentSourceParams.ContentFileTypeID;
+                contentItem.Checksum = await this.engine.getChecksumFromText(text);
+                contentItem.Text = text;
+                await contentItem.Save();
+                contentItemsToProcess.push(contentItem);
+            }
+            else if (blob.properties.lastModified && blob.properties.lastModified > lastRunDate) {
+                // The file has been modified, update the record for this file
+                const md = new Metadata();
+                const contentItem = await md.GetEntityObject('Content Items', contextUser);
+                const contentItemID = await this.engine.getContentItemIDFromURL(contentSourceParams, contextUser);
+                await contentItem.Load(contentItemID);
+                const text = await this.extractText(blob.name);
+                contentItem.Text = text;
+                contentItem.Checksum = await this.engine.getChecksumFromText(text);
+                contentItem.Save();
+                contentItemsToProcess.push(contentItem);
+            }
+        }
+        return contentItemsToProcess;
+    }
+    async extractText(file) {
+        const blockBlobClient = this.containerClient.getBlockBlobClient(file);
+        const downloadBlockBlobResponse = await blockBlobClient.download();
+        const document = await this.streamToBuffer(downloadBlockBlobResponse.readableStreamBody);
+        const text = await this.engine.parsePDF(document);
+        return text;
+    }
+    async streamToBuffer(readableStream) {
+        return new Promise((resolve, reject) => {
+            const chunks = [];
+            readableStream.on("data", (data) => {
+                chunks.push(data instanceof Buffer ? data : Buffer.from(data));
+            });
+            readableStream.on("end", () => {
+                resolve(Buffer.concat(chunks));
+            });
+            readableStream.on("error", reject);
+        });
+    }
+}
+//# sourceMappingURL=AutotagAzureBlob.js.map

package/dist/CloudStorage/providers/AutotagAzureBlob.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AutotagAzureBlob.js","sourceRoot":"","sources":["../../../src/CloudStorage/providers/AutotagAzureBlob.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAE/D,OAAO,EAAE,iBAAiB,EAAmB,MAAM,qBAAqB,CAAC;AACzE,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;AAE9B,MAAM,OAAO,gBAAiB,SAAQ,gBAAgB;IAMlD,YAAY,gBAAwB,EAAE,aAAqB;QACvD,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAA;QACxC,IAAI,CAAC,aAAa,GAAG,aAAa,CAAA;IACtC,CAAC;IAGD;;;MAGE;IACK,KAAK,CAAC,YAAY;QACrB,IAAI,CAAC;YACD,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC,oBAAoB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YACvF,IAAI,KAAK,EAAE,MAAM,SAAS,IAAI,IAAI,CAAC,iBAAiB,CAAC,cAAc,EAAE,EAAE,CAAC;gBACpE,OAAO,CAAC,GAAG,CAAC,cAAc,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;YAChD,CAAC;YAED,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACzF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;YACpB,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAA;QACjE,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,6BAA6B,CAAC,mBAAwC,EAAE,WAAiB,EAAE,WAAqB,EAAE,MAAM,GAAC,EAAE;QACpI,MAAM,qBAAqB,GAAwB,EAAE,CAAA;QAErD,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,CAAC,eAAe,CAAC,aAAa,EAAE,EAAE,CAAC;YAC5D,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;YACzD,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,GAAG,WAAW,EAAE,CAAC;gBACvE,4DAA4D;gBAC5D,MAAM,EAAE,GAAG,IAAI,QAAQ,EAAE,CAAA;gBACzB,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,eAAe,CAAoB,eAAe,EAAE,WAAW,CAAC,CAAA;gBAC7F,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;gBAC9C,WAAW,CAAC,eAAe,GAAG,mBAAmB,CAAC,eAAe,CAAA;gBACjE,WAAW,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAA;gBAC5B,WAAW,CAAC,WAAW,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,yBAAyB,CAAC,mBAAmB,EAAE,WAAW,CAAC,CAAA;gBACvG,WAAW,CAAC,GAAG,GAAG,QAAQ,CAAA;gBAC1B,WAAW,CAAC,aAAa,GAAG,mBAAmB,CAAC,aAAa,CAAA;gBAC7D,WAAW,CAAC,mBAAmB,GAAI,mBAAmB,CAAC,mBAAmB,CAAA;gBAC1E,WAAW,CAAC,iBAAiB,GAAG,mBAAmB,CAAC,iBAAiB,CAAA;gBACrE,WAAW,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAA;gBAClE,WAAW,CAAC,IAAI,GAAG,IAAI,CAAA;gBAEvB,MAAM,WAAW,CAAC,IAAI,EAAE,CAAA;gBACxB,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;YAC3C,CAAC;iBACI,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,GAAG,WAAW,EAAE,CAAC;gBAClF,8DAA8D;gBAC9D,MAAM,EAAE,GAAG,IAAI,QAAQ,EAAE,CAAA;gBACzB,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,eAAe,CAAoB,eAAe,EAAE,WAAW,CAAC,CAAA;gBAC7F,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,uBAAuB,CAAC,mBAAmB,EAAE,WAAW,CAAC,CAAA;gBACjG,MAAM,WAAW,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;gBACrC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;gBAC9C,WAAW,CAAC,IAAI,GAAG,IAAI,CAAA;gBACvB,WAAW,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAA;gBAClE,WAAW,CAAC,IAAI,EAAE,CAAA;gBAClB,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;YAC3C,CAAC;QACL,CAAC;QAED,OAAO,qBAAqB,CAAA;IAChC,CAAC;IAEM,KAAK,CAAC,WAAW,CAAC,IAAY;QACjC,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAA;QACrE,MAAM,yBAAyB,GAAG,MAAM,eAAe,CAAC,QAAQ,EAAE,CAAA;QAClE,MAAM,QAAQ,GAAW,MAAM,IAAI,CAAC,cAAc,CAAC,yBAAyB,CAAC,kBAAkB,CAAC,CAAA;QAChG,MAAM,IAAI,GAAW,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;QACzD,OAAO,IAAI,CAAA;IACf,CAAC;IAEM,KAAK,CAAC,cAAc,CAAC,cAAqC;QAC7D,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;gBAC/B,MAAM,CAAC,IAAI,CAAC,IAAI,YAAY,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YACnE,CAAC,CAAC,CAAC;YACH,cAAc,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBAC1B,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAiC,CAAC,CAAC,CAAC;YAC9D,CAAC,CAAC,CAAC;YACH,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;IACP,CAAC;CACJ"}

package/dist/Core/generic/AutotagBase.d.ts

@@ -0,0 +1,7 @@
+import { UserInfo } from '@memberjunction/core';
+import { ContentSourceEntity, ContentItemEntity } from '@memberjunction/core-entities';
+export declare abstract class AutotagBase {
+    abstract SetContentItemsToProcess(contentSources: ContentSourceEntity[]): Promise<ContentItemEntity[]>;
+    abstract Autotag(contextUser: UserInfo): Promise<void>;
+}
+//# sourceMappingURL=AutotagBase.d.ts.map

package/dist/Core/generic/AutotagBase.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"AutotagBase.d.ts","sourceRoot":"","sources":["../../../src/Core/generic/AutotagBase.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAEvF,8BAAsB,WAAW;aACb,wBAAwB,CAAC,cAAc,EAAE,mBAAmB,EAAE,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;aAC7F,OAAO,CAAC,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;CAChE"}

package/dist/Core/generic/AutotagBase.js

@@ -0,0 +1,3 @@
+export class AutotagBase {
+}
+//# sourceMappingURL=AutotagBase.js.map

package/dist/Core/generic/AutotagBase.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AutotagBase.js","sourceRoot":"","sources":["../../../src/Core/generic/AutotagBase.ts"],"names":[],"mappings":"AAGA,MAAM,OAAgB,WAAW;CAGhC"}

package/dist/Core/index.d.ts

@@ -0,0 +1,2 @@
+export * from './generic/AutotagBase.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/Core/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/Core/index.ts"],"names":[],"mappings":"AAAA,cAAc,uBAAuB,CAAC"}

package/dist/Core/index.js

@@ -0,0 +1,2 @@
+export * from './generic/AutotagBase.js';
+//# sourceMappingURL=index.js.map

package/dist/Core/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/Core/index.ts"],"names":[],"mappings":"AAAA,cAAc,uBAAuB,CAAC"}

package/dist/Engine/generic/AutotagBaseEngine.d.ts

@@ -0,0 +1,131 @@
+import { UserInfo } from '@memberjunction/core';
+import { ContentSourceEntity, ContentItemEntity } from '@memberjunction/core-entities';
+import { ContentSourceParams, ContentSourceTypeParams } from './content.types.js';
+import { ProcessRunParams, JsonObject, ContentItemProcessParams } from './process.types.js';
+import { BaseLLM } from '@memberjunction/ai';
+import { AIEngine } from '@memberjunction/aiengine';
+export declare class AutotagBaseEngine extends AIEngine {
+    constructor();
+    static get Instance(): AutotagBaseEngine;
+    /**
+     * Given a list of content items, extract the text from each content item with the LLM and send off the required parameters to the LLM for tagging.
+     * @param contentItems
+     * @returns
+     */
+    ExtractTextAndProcessWithLLM(contentItems: ContentItemEntity[], contextUser: UserInfo): Promise<void>;
+    /**
+     * Given processing parameters that include the text from our content item, process the text with the LLM and extract the
+     * information related to that content type.
+     * @param params
+     * @returns
+     */
+    ProcessContentItemText(params: ContentItemProcessParams, contextUser: UserInfo): Promise<void>;
+    promptAndRetrieveResultsFromLLM(params: ContentItemProcessParams, contextUser: UserInfo): Promise<JsonObject>;
+    processChunkWithLLM(llm: BaseLLM, systemPrompt: string, userPrompt: string, LLMResults: JsonObject, modelAPIName: string): Promise<JsonObject>;
+    getLLMPrompts(params: ContentItemProcessParams, chunk: string, LLMResults: JsonObject, contextUser: UserInfo): Promise<{
+        systemPrompt: string;
+        userPrompt: string;
+    }>;
+    saveLLMResults(LLMResults: JsonObject, contextUser: UserInfo): Promise<void>;
+    deleteInvalidContentItem(contentItemID: string, contextUser: UserInfo): Promise<void>;
+    chunkExtractedText(text: string, tokenLimit: number): string[];
+    /**
+     * Given the processing results from the LLM and the Content Element Item that was saved to the database, this function saves the tags as Content Element Tags in the database.
+     * @param md: The metadata object to save the tags
+     * @param contentElementItem: The content element item that was saved to the database
+     * @param results: The results of the processing from the LLM
+     * @param contextUser: The user context to save the tags
+     * @returns
+     */
+    saveContentItemTags(contentItemID: string, LLMResults: JsonObject, contextUser: UserInfo): Promise<void>;
+    saveResultsToContentItemAttribute(LLMResults: JsonObject, contextUser: UserInfo): Promise<void>;
+    /***
+    * Retrieves all of the content sources of a given content source type data from the database.
+    * @param contextUser: The user context to retrieve the content source data
+    * @returns A list of content sources
+    */
+    getAllContentSources(contextUser: UserInfo, contentSourceTypeID: string): Promise<ContentSourceEntity[]>;
+    setSubclassContentSourceType(subclass: string, contextUser: UserInfo): Promise<string>;
+    getContentSourceParams(contentSource: ContentSourceEntity, contextUser: UserInfo): Promise<any>;
+    getDefaultContentSourceTypeParams(contentSourceTypeParamID: string, contextUser: UserInfo): Promise<ContentSourceTypeParams>;
+    castValueAsCorrectType(value: string, type: string): any;
+    stringToBoolean(string: string): boolean;
+    parseStringArray(value: string): string[];
+    /**
+     * Given a run date, this function converts the run date to the user's timezone and formats it as a date object.
+     * @param lastRunDate: The retrieved last run date from the database
+     * @returns The last run date converted to the user's timezone
+     */
+    convertLastRunDateToTimezone(lastRunDate: Date): Promise<Date>;
+    /**
+     * Retrieves the last run date of the provided content source from the database. If no previous runs exist, the epoch date is returned.
+     * @param contentSourceID: The ID of the content source to retrieve the last run date
+     * @param contextUser: The user context to retrieve the last run date
+     * @returns
+     */
+    getContentSourceLastRunDate(contentSourceID: string, contextUser: UserInfo): Promise<Date>;
+    getContentItemParams(contentTypeID: string, contextUser: UserInfo): Promise<{
+        modelID: string;
+        minTags: number;
+        maxTags: number;
+    }>;
+    /**
+     * Given a content source type ID, this function retrieves the content source type name from the database.
+     * @param contentSourceTypeID
+     * @param contextUser
+     * @returns
+     */
+    getContentSourceTypeName(contentSourceTypeID: string, contextUser: UserInfo): Promise<string>;
+    /**
+     * Given a content type ID, this function retrieves the content type name from the database.
+     * @param contentTypeID
+     * @param contextUser
+     * @returns
+     */
+    getContentTypeName(contentTypeID: string, contextUser: UserInfo): Promise<string>;
+    /**
+     * Given a content file type ID, this function retrieves the content file type name from the database.
+     * @param contentFileTypeID
+     * @param contextUser
+     * @returns
+     */
+    getContentFileTypeName(contentFileTypeID: string, contextUser: UserInfo): Promise<string>;
+    getAdditionalContentTypePrompt(contentTypeID: string, contextUser: UserInfo): Promise<string>;
+    /**
+    * Given the content source parameters, this function creates a description of the content source item.
+    * @param contentSourceParams: The parameters of the content source item
+    * @returns The description of the content source item
+    */
+    getContentItemDescription(contentSourceParams: ContentSourceParams, contextUser: any): Promise<string>;
+    getChecksumFromURL(url: string): Promise<string>;
+    getChecksumFromText(text: string): Promise<string>;
+    getContentItemIDFromURL(contentSourceParams: ContentSourceParams, contextUser: UserInfo): Promise<string>;
+    /**
+     * Given the results of the processing from the LLM, this function saves the details of the process run in the database.
+     * @param processRunParams: The parameters holding the details of the process run
+     * @param contextUser: The user context to save the process run
+     * @returns
+     */
+    saveProcessRun(processRunParams: ProcessRunParams, contextUser: UserInfo): Promise<void>;
+    /**
+    * Given a buffer of data, this function extracts text from a PDF file
+    * @param dataBuffer: The buffer of data to extract text from
+    * @returns The extracted text from the PDF file
+    */
+    parsePDF(dataBuffer: Buffer): Promise<string>;
+    /**
+    * Given a buffer of data, this function extracts text from a DOCX file
+    * @param dataBuffer: The buffer of data to extract text from
+    * @returns The extracted text from the DOCX file
+    */
+    parseDOCX(dataBuffer: Buffer): Promise<string>;
+    parseHTML(data: string): Promise<string>;
+    /**
+    * Given a file path, as along as its one of the supported file types, this function choses the correct parser
+    * and returns the extracted text.
+    * @param filePath - The path to the file to extract text from
+    * @returns - The extracted text from the file
+    */
+    parseFileFromPath(filePath: string): Promise<string>;
+}
+//# sourceMappingURL=AutotagBaseEngine.d.ts.map

package/dist/Engine/generic/AutotagBaseEngine.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"AutotagBaseEngine.d.ts","sourceRoot":"","sources":["../../../src/Engine/generic/AutotagBaseEngine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqB,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AAElE,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAoJ,MAAM,+BAA+B,CAAA;AACxO,OAAO,EAAE,mBAAmB,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAA;AAI9E,OAAO,EAAE,gBAAgB,EAAE,UAAU,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAA;AAKxF,OAAO,EAAE,OAAO,EAAe,MAAM,oBAAoB,CAAA;AACzD,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAA;AAGnD,qBACa,iBAAkB,SAAQ,QAAQ;;IAM3C,WAAkB,QAAQ,IAAI,iBAAiB,CAE9C;IAED;;;;OAIG;IACU,4BAA4B,CAAC,YAAY,EAAE,iBAAiB,EAAE,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IA0ClH;;;;;OAKG;IACU,sBAAsB,CAAC,MAAM,EAAE,wBAAwB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAK9F,+BAA+B,CAAC,MAAM,EAAE,wBAAwB,EAAE,WAAW,EAAE,QAAQ;IAqBvF,mBAAmB,CAAC,GAAG,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAC,UAAU,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;IA+B7I,aAAa,CAAC,MAAM,EAAE,wBAAwB,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;IAiCpK,cAAc,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ;IAY5D,wBAAwB,CAAC,aAAa,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ;IAO3E,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE;IAwBrE;;;;;;;OAOG;IACU,mBAAmB,CAAC,aAAa,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ;IAaxF,iCAAiC,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ;IAiC5F;;;;MAIE;IACW,oBAAoB,CAAC,WAAW,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAwBxG,4BAA4B,CAAC,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAgBtF,sBAAsB,CAAC,aAAa,EAAE,mBAAmB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC;IA6B/F,iCAAiC,CAAC,wBAAwB,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,uBAAuB,CAAC;IAmBlI,sBAAsB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,GAAG;IAiBxD,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO;IAIxC,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE;IAKhD;;;;OAIG;IACU,4BAA4B,CAAC,WAAW,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAM3E;;;;;OAKG;IACU,2BAA2B,CAAC,eAAe,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IA6B1F,oBAAoB,CAAC,aAAa,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;KAAE,CAAC;IAqBhJ;;;;;OAKG;IACU,wBAAwB,CAAC,mBAAmB,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAqB1G;;;;;OAKG;IACU,kBAAkB,CAAC,aAAa,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAqB9F;;;;;OAKG;IACU,sBAAsB,CAAC,iBAAiB,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAqBzF,8BAA8B,CAAC,aAAa,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAwB1G;;;;MAIE;IACW,yBAAyB,CAAC,mBAAmB,EAAE,mBAAmB,EAAE,WAAW,KAAA,GAAG,OAAO,CAAC,MAAM,CAAC;IAMjG,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAOhD,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAKlD,uBAAuB,CAAC,mBAAmB,EAAE,mBAAmB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAuBtH;;;;;OAKG;IACU,cAAc,CAAC,gBAAgB,EAAE,gBAAgB,EAAE,WAAW,EAAE,QAAQ;IAYrF;;;;MAIE;IACW,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAK1D;;;;MAIE;IACW,SAAS,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAK9C,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAmBrD;;;;;MAKE;IACW,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;CAYpE"}