@memberjunction/content-autotagging 2.5.0

package/dist/src/CloudStorage/generic/CloudStorageBase.d.ts

@@ -0,0 +1,16 @@
+import { AutotagBase } from "../../Core";
+import { AutotagBaseEngine } from "../../Engine";
+import { ContentSourceParams } from "../../Engine";
+import { UserInfo } from "@memberjunction/core";
+import { ContentSourceEntity, ContentItemEntity } from "@memberjunction/core-entities";
+export declare abstract class CloudStorageBase extends AutotagBase {
+    protected contextUser: UserInfo;
+    protected engine: AutotagBaseEngine;
+    protected contentSourceTypeID: string;
+    constructor();
+    abstract Authenticate(): Promise<void>;
+    abstract SetNewAndModifiedContentItems(contentSourceParams: ContentSourceParams, lastRunDate: Date, contextUser: UserInfo): Promise<ContentItemEntity[]>;
+    Autotag(contextUser: UserInfo): Promise<void>;
+    SetContentItemsToProcess(contentSources: ContentSourceEntity[]): Promise<ContentItemEntity[]>;
+}
+//# sourceMappingURL=CloudStorageBase.d.ts.map

package/dist/src/CloudStorage/generic/CloudStorageBase.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"CloudStorageBase.d.ts","sourceRoot":"","sources":["../../../../src/CloudStorage/generic/CloudStorageBase.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AACjD,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAIvF,8BAAsB,gBAAiB,SAAQ,WAAW;IACtD,SAAS,CAAC,WAAW,EAAE,QAAQ,CAAC;IAChC,SAAS,CAAC,MAAM,EAAE,iBAAiB,CAAC;IACpC,SAAS,CAAC,mBAAmB,EAAE,MAAM,CAAA;;aAUrB,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;aAW7B,6BAA6B,CAAC,mBAAmB,EAAE,mBAAmB,EAAE,WAAW,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;IAElJ,OAAO,CAAC,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ7C,wBAAwB,CAAC,cAAc,EAAE,mBAAmB,EAAE,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;CAyB7G"}

package/dist/src/CloudStorage/generic/CloudStorageBase.js

@@ -0,0 +1,48 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CloudStorageBase = void 0;
+const Core_1 = require("../../Core");
+const Engine_1 = require("../../Engine");
+const dotenv_1 = __importDefault(require("dotenv"));
+dotenv_1.default.config();
+class CloudStorageBase extends Core_1.AutotagBase {
+    contextUser;
+    engine;
+    contentSourceTypeID;
+    constructor() {
+        super();
+        this.engine = Engine_1.AutotagBaseEngine.Instance;
+    }
+    async Autotag(contextUser) {
+        this.contextUser = contextUser;
+        this.contentSourceTypeID = await this.engine.setSubclassContentSourceType('Cloud Storage', this.contextUser);
+        const contentSources = await this.engine.getAllContentSources(this.contextUser, this.contentSourceTypeID) || [];
+        const contentItemsToProcess = await this.SetContentItemsToProcess(contentSources);
+        await this.engine.ExtractTextAndProcessWithLLM(contentItemsToProcess, this.contextUser);
+    }
+    async SetContentItemsToProcess(contentSources) {
+        const contentItemsToProcess = [];
+        for (const contentSource of contentSources) {
+            await this.Authenticate();
+            const contentSourceParams = {
+                contentSourceID: contentSource.ID,
+                name: contentSource.Name,
+                ContentTypeID: contentSource.ContentTypeID,
+                ContentSourceTypeID: contentSource.ContentSourceTypeID,
+                ContentFileTypeID: contentSource.ContentFileTypeID,
+                URL: contentSource.URL
+            };
+            const lastRunDate = await this.engine.getContentSourceLastRunDate(contentSourceParams.contentSourceID, this.contextUser);
+            if (lastRunDate) {
+                const contentItems = await this.SetNewAndModifiedContentItems(contentSourceParams, lastRunDate, this.contextUser);
+                contentItemsToProcess.push(...contentItems);
+            }
+        }
+        return contentItemsToProcess;
+    }
+}
+exports.CloudStorageBase = CloudStorageBase;
+//# sourceMappingURL=CloudStorageBase.js.map

package/dist/src/CloudStorage/generic/CloudStorageBase.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CloudStorageBase.js","sourceRoot":"","sources":["../../../../src/CloudStorage/generic/CloudStorageBase.ts"],"names":[],"mappings":";;;;;;AAAA,qCAAyC;AACzC,yCAAiD;AAIjD,oDAA4B;AAC5B,gBAAM,CAAC,MAAM,EAAE,CAAA;AAEf,MAAsB,gBAAiB,SAAQ,kBAAW;IAC5C,WAAW,CAAW;IACtB,MAAM,CAAoB;IAC1B,mBAAmB,CAAQ;IAErC;QACI,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,0BAAiB,CAAC,QAAQ,CAAC;IAC7C,CAAC;IAkBM,KAAK,CAAC,OAAO,CAAC,WAAqB;QACtC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,mBAAmB,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,eAAe,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAC7G,MAAM,cAAc,GAA0B,MAAM,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC;QACvI,MAAM,qBAAqB,GAAwB,MAAM,IAAI,CAAC,wBAAwB,CAAC,cAAc,CAAC,CAAA;QACtG,MAAM,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,qBAAqB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAC5F,CAAC;IAEM,KAAK,CAAC,wBAAwB,CAAC,cAAqC;QACvE,MAAM,qBAAqB,GAAwB,EAAE,CAAA;QAErD,KAAK,MAAM,aAAa,IAAI,cAAc,EAAE,CAAC;YACzC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAE1B,MAAM,mBAAmB,GAAwB;gBAC7C,eAAe,EAAE,aAAa,CAAC,EAAE;gBACjC,IAAI,EAAE,aAAa,CAAC,IAAI;gBACxB,aAAa,EAAE,aAAa,CAAC,aAAa;gBAC1C,mBAAmB,EAAE,aAAa,CAAC,mBAAmB;gBACtD,iBAAiB,EAAE,aAAa,CAAC,iBAAiB;gBAClD,GAAG,EAAE,aAAa,CAAC,GAAG;aACzB,CAAA;YAED,MAAM,WAAW,GAAS,MAAM,IAAI,CAAC,MAAM,CAAC,2BAA2B,CAAC,mBAAmB,CAAC,eAAe,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YAE/H,IAAI,WAAW,EAAE,CAAC;gBACd,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,mBAAmB,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;gBAClH,qBAAqB,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;YAChD,CAAC;QACL,CAAC;QAED,OAAO,qBAAqB,CAAC;IACjC,CAAC;CACJ;AA3DD,4CA2DC"}

package/dist/src/CloudStorage/index.d.ts

@@ -0,0 +1,3 @@
+export * from './generic/CloudStorageBase';
+export * from './providers/AutotagAzureBlob';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/CloudStorage/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/CloudStorage/index.ts"],"names":[],"mappings":"AAAA,cAAc,4BAA4B,CAAA;AAC1C,cAAc,8BAA8B,CAAA"}

package/dist/src/CloudStorage/index.js

@@ -0,0 +1,19 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./generic/CloudStorageBase"), exports);
+__exportStar(require("./providers/AutotagAzureBlob"), exports);
+//# sourceMappingURL=index.js.map

package/dist/src/CloudStorage/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/CloudStorage/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,6DAA0C;AAC1C,+DAA4C"}

package/dist/src/CloudStorage/providers/AutotagAzureBlob.d.ts

@@ -0,0 +1,18 @@
+/// <reference types="node" />
+/// <reference types="node" />
+import { CloudStorageBase } from "../generic/CloudStorageBase";
+import { UserInfo } from "@memberjunction/core";
+import { ContentItemEntity } from "@memberjunction/core-entities";
+import { ContentSourceParams } from "../../Engine";
+export declare class AutotagAzureBlob extends CloudStorageBase {
+    private blobServiceClient;
+    private containerClient;
+    private connectionString;
+    private containerName;
+    constructor(connectionString: string, containerName: string);
+    Authenticate(): Promise<void>;
+    SetNewAndModifiedContentItems(contentSourceParams: ContentSourceParams, lastRunDate: Date, contextUser: UserInfo, prefix?: string): Promise<ContentItemEntity[]>;
+    extractText(file: string): Promise<string>;
+    streamToBuffer(readableStream: NodeJS.ReadableStream): Promise<Buffer>;
+}
+//# sourceMappingURL=AutotagAzureBlob.d.ts.map

package/dist/src/CloudStorage/providers/AutotagAzureBlob.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"AutotagAzureBlob.d.ts","sourceRoot":"","sources":["../../../../src/CloudStorage/providers/AutotagAzureBlob.ts"],"names":[],"mappings":";;AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAGhD,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAGlE,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAGnD,qBAAa,gBAAiB,SAAQ,gBAAgB;IAClD,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,aAAa,CAAS;gBAElB,gBAAgB,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM;IAW9C,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAc7B,6BAA6B,CAAC,mBAAmB,EAAE,mBAAmB,EAAE,WAAW,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,SAAG,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;IAwC1J,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAQ1C,cAAc,CAAC,cAAc,EAAE,MAAM,CAAC,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC;CAYtF"}

package/dist/src/CloudStorage/providers/AutotagAzureBlob.js

@@ -0,0 +1,91 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AutotagAzureBlob = void 0;
+const CloudStorageBase_1 = require("../generic/CloudStorageBase");
+const storage_blob_1 = require("@azure/storage-blob");
+const dotenv_1 = __importDefault(require("dotenv"));
+const core_1 = require("@memberjunction/core");
+const path_1 = __importDefault(require("path"));
+dotenv_1.default.config();
+class AutotagAzureBlob extends CloudStorageBase_1.CloudStorageBase {
+    blobServiceClient;
+    containerClient;
+    connectionString;
+    containerName;
+    constructor(connectionString, containerName) {
+        super();
+        this.connectionString = connectionString;
+        this.containerName = containerName;
+    }
+    async Authenticate() {
+        try {
+            this.blobServiceClient = storage_blob_1.BlobServiceClient.fromConnectionString(this.connectionString);
+            for await (const container of this.blobServiceClient.listContainers()) {
+                console.log(`Container: ${container.name}`);
+            }
+            this.containerClient = this.blobServiceClient.getContainerClient(this.containerName);
+        }
+        catch (error) {
+            console.error(error);
+            throw new Error('Error authenticating to Azure Blob Storage');
+        }
+    }
+    async SetNewAndModifiedContentItems(contentSourceParams, lastRunDate, contextUser, prefix = '') {
+        const contentItemsToProcess = [];
+        for await (const blob of this.containerClient.listBlobsFlat()) {
+            const filePath = path_1.default.join(this.containerName, blob.name);
+            if (blob.properties.createdOn && blob.properties.createdOn > lastRunDate) {
+                const md = new core_1.Metadata();
+                const contentItem = await md.GetEntityObject('Content Items', contextUser);
+                const text = await this.extractText(blob.name);
+                contentItem.ContentSourceID = contentSourceParams.contentSourceID;
+                contentItem.Name = blob.name;
+                contentItem.Description = await this.engine.getContentItemDescription(contentSourceParams, contextUser);
+                contentItem.URL = filePath;
+                contentItem.ContentTypeID = contentSourceParams.ContentTypeID;
+                contentItem.ContentSourceTypeID = contentSourceParams.ContentSourceTypeID;
+                contentItem.ContentFileTypeID = contentSourceParams.ContentFileTypeID;
+                contentItem.Checksum = await this.engine.getChecksumFromText(text);
+                contentItem.Text = text;
+                await contentItem.Save();
+                contentItemsToProcess.push(contentItem);
+            }
+            else if (blob.properties.lastModified && blob.properties.lastModified > lastRunDate) {
+                const md = new core_1.Metadata();
+                const contentItem = await md.GetEntityObject('Content Items', contextUser);
+                const contentItemID = await this.engine.getContentItemIDFromURL(contentSourceParams, contextUser);
+                await contentItem.Load(contentItemID);
+                const text = await this.extractText(blob.name);
+                contentItem.Text = text;
+                contentItem.Checksum = await this.engine.getChecksumFromText(text);
+                contentItem.Save();
+                contentItemsToProcess.push(contentItem);
+            }
+        }
+        return contentItemsToProcess;
+    }
+    async extractText(file) {
+        const blockBlobClient = this.containerClient.getBlockBlobClient(file);
+        const downloadBlockBlobResponse = await blockBlobClient.download();
+        const document = await this.streamToBuffer(downloadBlockBlobResponse.readableStreamBody);
+        const text = await this.engine.parsePDF(document);
+        return text;
+    }
+    async streamToBuffer(readableStream) {
+        return new Promise((resolve, reject) => {
+            const chunks = [];
+            readableStream.on("data", (data) => {
+                chunks.push(data instanceof Buffer ? data : Buffer.from(data));
+            });
+            readableStream.on("end", () => {
+                resolve(Buffer.concat(chunks));
+            });
+            readableStream.on("error", reject);
+        });
+    }
+}
+exports.AutotagAzureBlob = AutotagAzureBlob;
+//# sourceMappingURL=AutotagAzureBlob.js.map

package/dist/src/CloudStorage/providers/AutotagAzureBlob.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AutotagAzureBlob.js","sourceRoot":"","sources":["../../../../src/CloudStorage/providers/AutotagAzureBlob.ts"],"names":[],"mappings":";;;;;;AAAA,kEAA+D;AAE/D,sDAAyE;AACzE,oDAA4B;AAE5B,+CAAgD;AAChD,gDAAwB;AAExB,gBAAM,CAAC,MAAM,EAAE,CAAA;AAEf,MAAa,gBAAiB,SAAQ,mCAAgB;IAC1C,iBAAiB,CAAoB;IACrC,eAAe,CAAkB;IACjC,gBAAgB,CAAS;IACzB,aAAa,CAAS;IAE9B,YAAY,gBAAwB,EAAE,aAAqB;QACvD,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAA;QACxC,IAAI,CAAC,aAAa,GAAG,aAAa,CAAA;IACtC,CAAC;IAOM,KAAK,CAAC,YAAY;QACrB,IAAI,CAAC;YACD,IAAI,CAAC,iBAAiB,GAAG,gCAAiB,CAAC,oBAAoB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YACvF,IAAI,KAAK,EAAE,MAAM,SAAS,IAAI,IAAI,CAAC,iBAAiB,CAAC,cAAc,EAAE,EAAE,CAAC;gBACpE,OAAO,CAAC,GAAG,CAAC,cAAc,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;YAChD,CAAC;YAED,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACzF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;YACpB,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAA;QACjE,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,6BAA6B,CAAC,mBAAwC,EAAE,WAAiB,EAAE,WAAqB,EAAE,MAAM,GAAC,EAAE;QACpI,MAAM,qBAAqB,GAAwB,EAAE,CAAA;QAErD,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,CAAC,eAAe,CAAC,aAAa,EAAE,EAAE,CAAC;YAC5D,MAAM,QAAQ,GAAG,cAAI,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;YACzD,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,GAAG,WAAW,EAAE,CAAC;gBAEvE,MAAM,EAAE,GAAG,IAAI,eAAQ,EAAE,CAAA;gBACzB,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,eAAe,CAAoB,eAAe,EAAE,WAAW,CAAC,CAAA;gBAC7F,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;gBAC9C,WAAW,CAAC,eAAe,GAAG,mBAAmB,CAAC,eAAe,CAAA;gBACjE,WAAW,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAA;gBAC5B,WAAW,CAAC,WAAW,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,yBAAyB,CAAC,mBAAmB,EAAE,WAAW,CAAC,CAAA;gBACvG,WAAW,CAAC,GAAG,GAAG,QAAQ,CAAA;gBAC1B,WAAW,CAAC,aAAa,GAAG,mBAAmB,CAAC,aAAa,CAAA;gBAC7D,WAAW,CAAC,mBAAmB,GAAI,mBAAmB,CAAC,mBAAmB,CAAA;gBAC1E,WAAW,CAAC,iBAAiB,GAAG,mBAAmB,CAAC,iBAAiB,CAAA;gBACrE,WAAW,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAA;gBAClE,WAAW,CAAC,IAAI,GAAG,IAAI,CAAA;gBAEvB,MAAM,WAAW,CAAC,IAAI,EAAE,CAAA;gBACxB,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;YAC3C,CAAC;iBACI,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,GAAG,WAAW,EAAE,CAAC;gBAElF,MAAM,EAAE,GAAG,IAAI,eAAQ,EAAE,CAAA;gBACzB,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,eAAe,CAAoB,eAAe,EAAE,WAAW,CAAC,CAAA;gBAC7F,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,uBAAuB,CAAC,mBAAmB,EAAE,WAAW,CAAC,CAAA;gBACjG,MAAM,WAAW,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;gBACrC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;gBAC9C,WAAW,CAAC,IAAI,GAAG,IAAI,CAAA;gBACvB,WAAW,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAA;gBAClE,WAAW,CAAC,IAAI,EAAE,CAAA;gBAClB,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;YAC3C,CAAC;QACL,CAAC;QAED,OAAO,qBAAqB,CAAA;IAChC,CAAC;IAEM,KAAK,CAAC,WAAW,CAAC,IAAY;QACjC,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAA;QACrE,MAAM,yBAAyB,GAAG,MAAM,eAAe,CAAC,QAAQ,EAAE,CAAA;QAClE,MAAM,QAAQ,GAAW,MAAM,IAAI,CAAC,cAAc,CAAC,yBAAyB,CAAC,kBAAkB,CAAC,CAAA;QAChG,MAAM,IAAI,GAAW,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;QACzD,OAAO,IAAI,CAAA;IACf,CAAC;IAEM,KAAK,CAAC,cAAc,CAAC,cAAqC;QAC7D,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;gBAC/B,MAAM,CAAC,IAAI,CAAC,IAAI,YAAY,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YACnE,CAAC,CAAC,CAAC;YACH,cAAc,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBAC1B,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;YACnC,CAAC,CAAC,CAAC;YACH,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;IACP,CAAC;CACJ;AA3FD,4CA2FC"}

package/dist/src/Core/generic/AutotagBase.d.ts

@@ -0,0 +1,7 @@
+import { UserInfo } from '@memberjunction/core';
+import { ContentSourceEntity, ContentItemEntity } from '@memberjunction/core-entities';
+export declare abstract class AutotagBase {
+    abstract SetContentItemsToProcess(contentSources: ContentSourceEntity[]): Promise<ContentItemEntity[]>;
+    abstract Autotag(contextUser: UserInfo): Promise<void>;
+}
+//# sourceMappingURL=AutotagBase.d.ts.map

package/dist/src/Core/generic/AutotagBase.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"AutotagBase.d.ts","sourceRoot":"","sources":["../../../../src/Core/generic/AutotagBase.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAEvF,8BAAsB,WAAW;aACb,wBAAwB,CAAC,cAAc,EAAE,mBAAmB,EAAE,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;aAC7F,OAAO,CAAC,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;CAChE"}

package/dist/src/Core/generic/AutotagBase.js

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AutotagBase = void 0;
+class AutotagBase {
+}
+exports.AutotagBase = AutotagBase;
+//# sourceMappingURL=AutotagBase.js.map

package/dist/src/Core/generic/AutotagBase.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AutotagBase.js","sourceRoot":"","sources":["../../../../src/Core/generic/AutotagBase.ts"],"names":[],"mappings":";;;AAGA,MAAsB,WAAW;CAGhC;AAHD,kCAGC"}

package/dist/src/Core/index.d.ts

@@ -0,0 +1,2 @@
+export * from './generic/AutotagBase';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/Core/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/Core/index.ts"],"names":[],"mappings":"AAAA,cAAc,uBAAuB,CAAC"}

package/dist/src/Core/index.js

@@ -0,0 +1,18 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./generic/AutotagBase"), exports);
+//# sourceMappingURL=index.js.map

package/dist/src/Core/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/Core/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,wDAAsC"}

package/dist/src/Engine/generic/AutotagBaseEngine.d.ts

@@ -0,0 +1,51 @@
+/// <reference types="node" />
+import { UserInfo } from '@memberjunction/core';
+import { ContentSourceEntity, ContentItemEntity } from '@memberjunction/core-entities';
+import { ContentSourceParams, ContentSourceTypeParams } from './content.types';
+import { ProcessRunParams, JsonObject, ContentItemProcessParams } from './process.types';
+import { BaseLLM } from '@memberjunction/ai';
+import { AIEngine } from '@memberjunction/aiengine';
+export declare class AutotagBaseEngine extends AIEngine {
+    constructor();
+    static get Instance(): AutotagBaseEngine;
+    ExtractTextAndProcessWithLLM(contentItems: ContentItemEntity[], contextUser: UserInfo): Promise<void>;
+    ProcessContentItemText(params: ContentItemProcessParams, contextUser: UserInfo): Promise<void>;
+    promptAndRetrieveResultsFromLLM(params: ContentItemProcessParams, contextUser: UserInfo): Promise<JsonObject>;
+    processChunkWithLLM(llm: BaseLLM, systemPrompt: string, userPrompt: string, LLMResults: JsonObject, modelAPIName: string): Promise<JsonObject>;
+    getLLMPrompts(params: ContentItemProcessParams, chunk: string, LLMResults: JsonObject, contextUser: UserInfo): Promise<{
+        systemPrompt: string;
+        userPrompt: string;
+    }>;
+    saveLLMResults(LLMResults: JsonObject, contextUser: UserInfo): Promise<void>;
+    deleteInvalidContentItem(contentItemID: string, contextUser: UserInfo): Promise<void>;
+    chunkExtractedText(text: string, tokenLimit: number): string[];
+    saveContentItemTags(contentItemID: string, LLMResults: JsonObject, contextUser: UserInfo): Promise<void>;
+    saveResultsToContentItemAttribute(LLMResults: JsonObject, contextUser: UserInfo): Promise<void>;
+    getAllContentSources(contextUser: UserInfo, contentSourceTypeID: string): Promise<ContentSourceEntity[]>;
+    setSubclassContentSourceType(subclass: string, contextUser: UserInfo): Promise<string>;
+    getContentSourceParams(contentSource: ContentSourceEntity, contextUser: UserInfo): Promise<any>;
+    getDefaultContentSourceTypeParams(contentSourceTypeParamID: string, contextUser: UserInfo): Promise<ContentSourceTypeParams>;
+    castValueAsCorrectType(value: string, type: string): any;
+    stringToBoolean(string: string): boolean;
+    convertLastRunDateToTimezone(lastRunDate: Date): Promise<Date>;
+    getContentSourceLastRunDate(contentSourceID: string, contextUser: UserInfo): Promise<Date>;
+    getContentItemParams(contentTypeID: string, contextUser: UserInfo): Promise<{
+        modelID: string;
+        minTags: number;
+        maxTags: number;
+    }>;
+    getContentSourceTypeName(contentSourceTypeID: string, contextUser: UserInfo): Promise<string>;
+    getContentTypeName(contentTypeID: string, contextUser: UserInfo): Promise<string>;
+    getContentFileTypeName(contentFileTypeID: string, contextUser: UserInfo): Promise<string>;
+    getAdditionalContentTypePrompt(contentTypeID: string, contextUser: UserInfo): Promise<string>;
+    getContentItemDescription(contentSourceParams: ContentSourceParams, contextUser: any): Promise<string>;
+    getChecksumFromURL(url: string): Promise<string>;
+    getChecksumFromText(text: string): Promise<string>;
+    getContentItemIDFromURL(contentSourceParams: ContentSourceParams, contextUser: UserInfo): Promise<string>;
+    saveProcessRun(processRunParams: ProcessRunParams, contextUser: UserInfo): Promise<void>;
+    parsePDF(dataBuffer: Buffer): Promise<string>;
+    parseDOCX(dataBuffer: Buffer): Promise<string>;
+    parseHTML(data: string): Promise<string>;
+    parseFileFromPath(filePath: string): Promise<string>;
+}
+//# sourceMappingURL=AutotagBaseEngine.d.ts.map

package/dist/src/Engine/generic/AutotagBaseEngine.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"AutotagBaseEngine.d.ts","sourceRoot":"","sources":["../../../../src/Engine/generic/AutotagBaseEngine.ts"],"names":[],"mappings":";AAAA,OAAO,EAAqB,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AAElE,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAoJ,MAAM,+BAA+B,CAAA;AACxO,OAAO,EAAE,mBAAmB,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAA;AAI9E,OAAO,EAAE,gBAAgB,EAAE,UAAU,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAA;AAKxF,OAAO,EAAE,OAAO,EAAe,MAAM,oBAAoB,CAAA;AACzD,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAA;AAGnD,qBACa,iBAAkB,SAAQ,QAAQ;;IAM3C,WAAkB,QAAQ,IAAI,iBAAiB,CAE9C;IAOY,4BAA4B,CAAC,YAAY,EAAE,iBAAiB,EAAE,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAgDrG,sBAAsB,CAAC,MAAM,EAAE,wBAAwB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAK9F,+BAA+B,CAAC,MAAM,EAAE,wBAAwB,EAAE,WAAW,EAAE,QAAQ;IAoBvF,mBAAmB,CAAC,GAAG,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAC,UAAU,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;IA+B7I,aAAa,CAAC,MAAM,EAAE,wBAAwB,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;IAiCpK,cAAc,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ;IAY5D,wBAAwB,CAAC,aAAa,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ;IAO3E,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE;IAgCxD,mBAAmB,CAAC,aAAa,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ;IAaxF,iCAAiC,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ;IAsC/E,oBAAoB,CAAC,WAAW,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAwBxG,4BAA4B,CAAC,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAgBtF,sBAAsB,CAAC,aAAa,EAAE,mBAAmB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC;IA6B/F,iCAAiC,CAAC,wBAAwB,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,uBAAuB,CAAC;IAmBlI,sBAAsB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,GAAG;IAexD,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO;IASlC,4BAA4B,CAAC,WAAW,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAY9D,2BAA2B,CAAC,eAAe,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IA6B1F,oBAAoB,CAAC,aAAa,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;KAAE,CAAC;IA2BnI,wBAAwB,CAAC,mBAAmB,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IA2B7F,kBAAkB,CAAC,aAAa,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IA2BjF,sBAAsB,CAAC,iBAAiB,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAqBzF,8BAA8B,CAAC,aAAa,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IA6B7F,yBAAyB,CAAC,mBAAmB,EAAE,mBAAmB,EAAE,WAAW,KAAA,GAAG,OAAO,CAAC,MAAM,CAAC;IAMjG,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAOhD,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAKlD,uBAAuB,CAAC,mBAAmB,EAAE,mBAAmB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IA6BzG,cAAc,CAAC,gBAAgB,EAAE,gBAAgB,EAAE,WAAW,EAAE,QAAQ;IAiBxE,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAU7C,SAAS,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAK9C,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAyBxC,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;CAYpE"}