@memberjunction/content-autotagging 2.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/dist/src/CloudStorage/generic/CloudStorageBase.d.ts +16 -0
  2. package/dist/src/CloudStorage/generic/CloudStorageBase.d.ts.map +1 -0
  3. package/dist/src/CloudStorage/generic/CloudStorageBase.js +48 -0
  4. package/dist/src/CloudStorage/generic/CloudStorageBase.js.map +1 -0
  5. package/dist/src/CloudStorage/index.d.ts +3 -0
  6. package/dist/src/CloudStorage/index.d.ts.map +1 -0
  7. package/dist/src/CloudStorage/index.js +19 -0
  8. package/dist/src/CloudStorage/index.js.map +1 -0
  9. package/dist/src/CloudStorage/providers/AutotagAzureBlob.d.ts +18 -0
  10. package/dist/src/CloudStorage/providers/AutotagAzureBlob.d.ts.map +1 -0
  11. package/dist/src/CloudStorage/providers/AutotagAzureBlob.js +91 -0
  12. package/dist/src/CloudStorage/providers/AutotagAzureBlob.js.map +1 -0
  13. package/dist/src/Core/generic/AutotagBase.d.ts +7 -0
  14. package/dist/src/Core/generic/AutotagBase.d.ts.map +1 -0
  15. package/dist/src/Core/generic/AutotagBase.js +7 -0
  16. package/dist/src/Core/generic/AutotagBase.js.map +1 -0
  17. package/dist/src/Core/index.d.ts +2 -0
  18. package/dist/src/Core/index.d.ts.map +1 -0
  19. package/dist/src/Core/index.js +18 -0
  20. package/dist/src/Core/index.js.map +1 -0
  21. package/dist/src/Engine/generic/AutotagBaseEngine.d.ts +51 -0
  22. package/dist/src/Engine/generic/AutotagBaseEngine.d.ts.map +1 -0
  23. package/dist/src/Engine/generic/AutotagBaseEngine.js +553 -0
  24. package/dist/src/Engine/generic/AutotagBaseEngine.js.map +1 -0
  25. package/dist/src/Engine/generic/content.types.d.ts +32 -0
  26. package/dist/src/Engine/generic/content.types.d.ts.map +1 -0
  27. package/dist/src/Engine/generic/content.types.js +38 -0
  28. package/dist/src/Engine/generic/content.types.js.map +1 -0
  29. package/dist/src/Engine/generic/process.types.d.ts +30 -0
  30. package/dist/src/Engine/generic/process.types.d.ts.map +1 -0
  31. package/dist/src/Engine/generic/process.types.js +33 -0
  32. package/dist/src/Engine/generic/process.types.js.map +1 -0
  33. package/dist/src/Engine/index.d.ts +4 -0
  34. package/dist/src/Engine/index.d.ts.map +1 -0
  35. package/dist/src/Engine/index.js +20 -0
  36. package/dist/src/Engine/index.js.map +1 -0
  37. package/dist/src/LocalFileSystem/generic/AutotagLocalFileSystem.d.ts +20 -0
  38. package/dist/src/LocalFileSystem/generic/AutotagLocalFileSystem.d.ts.map +1 -0
  39. package/dist/src/LocalFileSystem/generic/AutotagLocalFileSystem.js +155 -0
  40. package/dist/src/LocalFileSystem/generic/AutotagLocalFileSystem.js.map +1 -0
  41. package/dist/src/LocalFileSystem/index.d.ts +2 -0
  42. package/dist/src/LocalFileSystem/index.d.ts.map +1 -0
  43. package/dist/src/LocalFileSystem/index.js +18 -0
  44. package/dist/src/LocalFileSystem/index.js.map +1 -0
  45. package/dist/src/RSSFeed/generic/AutotagRSSFeed.d.ts +19 -0
  46. package/dist/src/RSSFeed/generic/AutotagRSSFeed.d.ts.map +1 -0
  47. package/dist/src/RSSFeed/generic/AutotagRSSFeed.js +168 -0
  48. package/dist/src/RSSFeed/generic/AutotagRSSFeed.js.map +1 -0
  49. package/dist/src/RSSFeed/generic/RSS.types.d.ts +13 -0
  50. package/dist/src/RSSFeed/generic/RSS.types.d.ts.map +1 -0
  51. package/dist/src/RSSFeed/generic/RSS.types.js +17 -0
  52. package/dist/src/RSSFeed/generic/RSS.types.js.map +1 -0
  53. package/dist/src/RSSFeed/index.d.ts +3 -0
  54. package/dist/src/RSSFeed/index.d.ts.map +1 -0
  55. package/dist/src/RSSFeed/index.js +19 -0
  56. package/dist/src/RSSFeed/index.js.map +1 -0
  57. package/dist/src/Websites/generic/AutotagWebsite.d.ts +33 -0
  58. package/dist/src/Websites/generic/AutotagWebsite.d.ts.map +1 -0
  59. package/dist/src/Websites/generic/AutotagWebsite.js +324 -0
  60. package/dist/src/Websites/generic/AutotagWebsite.js.map +1 -0
  61. package/dist/src/Websites/index.d.ts +2 -0
  62. package/dist/src/Websites/index.d.ts.map +1 -0
  63. package/dist/src/Websites/index.js +18 -0
  64. package/dist/src/Websites/index.js.map +1 -0
  65. package/dist/src/index.d.ts +7 -0
  66. package/dist/src/index.d.ts.map +1 -0
  67. package/dist/src/index.js +23 -0
  68. package/dist/src/index.js.map +1 -0
  69. package/dist/tsconfig.tsbuildinfo +1 -0
  70. package/package.json +45 -0
@@ -0,0 +1,30 @@
1
+ export declare class ProcessRunParams {
2
+ sourceID: string;
3
+ startTime: Date;
4
+ endTime: Date;
5
+ numItemsProcessed: number;
6
+ }
7
+ export declare class ContentItemProcessParams {
8
+ text: string;
9
+ modelID: string;
10
+ minTags: number;
11
+ maxTags: number;
12
+ contentItemID: string;
13
+ contentTypeID: string;
14
+ contentFileTypeID: string;
15
+ contentSourceTypeID: string;
16
+ }
17
+ export declare class ContentItemProcessResults {
18
+ title: string;
19
+ author: string[];
20
+ publicationDate: Date;
21
+ keywords: string[];
22
+ content_text: string;
23
+ processStartTime: Date;
24
+ processEndTime: Date;
25
+ contentItemID: string;
26
+ }
27
+ export interface JsonObject {
28
+ [key: string]: any;
29
+ }
30
+ //# sourceMappingURL=process.types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"process.types.d.ts","sourceRoot":"","sources":["../../../../src/Engine/generic/process.types.ts"],"names":[],"mappings":"AAAA,qBAAa,gBAAgB;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,IAAI,CAAC;IAChB,OAAO,EAAE,IAAI,CAAC;IACd,iBAAiB,EAAE,MAAM,CAAC;CAC7B;AAED,qBAAa,wBAAwB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;CAC/B;AAED,qBAAa,yBAAyB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,eAAe,EAAE,IAAI,CAAC;IACtB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,IAAI,CAAC;IACvB,cAAc,EAAE,IAAI,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,UAAU;IACvB,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACtB"}
@@ -0,0 +1,33 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.ContentItemProcessResults = exports.ContentItemProcessParams = exports.ProcessRunParams = void 0;
4
+ class ProcessRunParams {
5
+ sourceID;
6
+ startTime;
7
+ endTime;
8
+ numItemsProcessed;
9
+ }
10
+ exports.ProcessRunParams = ProcessRunParams;
11
+ class ContentItemProcessParams {
12
+ text;
13
+ modelID;
14
+ minTags;
15
+ maxTags;
16
+ contentItemID;
17
+ contentTypeID;
18
+ contentFileTypeID;
19
+ contentSourceTypeID;
20
+ }
21
+ exports.ContentItemProcessParams = ContentItemProcessParams;
22
+ class ContentItemProcessResults {
23
+ title;
24
+ author;
25
+ publicationDate;
26
+ keywords;
27
+ content_text;
28
+ processStartTime;
29
+ processEndTime;
30
+ contentItemID;
31
+ }
32
+ exports.ContentItemProcessResults = ContentItemProcessResults;
33
+ //# sourceMappingURL=process.types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"process.types.js","sourceRoot":"","sources":["../../../../src/Engine/generic/process.types.ts"],"names":[],"mappings":";;;AAAA,MAAa,gBAAgB;IACzB,QAAQ,CAAS;IACjB,SAAS,CAAO;IAChB,OAAO,CAAO;IACd,iBAAiB,CAAS;CAC7B;AALD,4CAKC;AAED,MAAa,wBAAwB;IACjC,IAAI,CAAS;IACb,OAAO,CAAS;IAChB,OAAO,CAAS;IAChB,OAAO,CAAS;IAChB,aAAa,CAAS;IACtB,aAAa,CAAS;IACtB,iBAAiB,CAAS;IAC1B,mBAAmB,CAAS;CAC/B;AATD,4DASC;AAED,MAAa,yBAAyB;IAClC,KAAK,CAAS;IACd,MAAM,CAAW;IACjB,eAAe,CAAO;IACtB,QAAQ,CAAW;IACnB,YAAY,CAAS;IACrB,gBAAgB,CAAO;IACvB,cAAc,CAAO;IACrB,aAAa,CAAS;CACzB;AATD,8DASC"}
@@ -0,0 +1,4 @@
1
+ export * from './generic/AutotagBaseEngine';
2
+ export * from './generic/content.types';
3
+ export * from './generic/process.types';
4
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/Engine/index.ts"],"names":[],"mappings":"AAAA,cAAc,6BAA6B,CAAA;AAC3C,cAAc,yBAAyB,CAAC;AACxC,cAAc,yBAAyB,CAAC"}
@@ -0,0 +1,20 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
+ for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
+ };
16
+ Object.defineProperty(exports, "__esModule", { value: true });
17
+ __exportStar(require("./generic/AutotagBaseEngine"), exports);
18
+ __exportStar(require("./generic/content.types"), exports);
19
+ __exportStar(require("./generic/process.types"), exports);
20
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/Engine/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,8DAA2C;AAC3C,0DAAwC;AACxC,0DAAwC"}
@@ -0,0 +1,20 @@
1
+ import { AutotagBase } from "../../Core";
2
+ import { ContentSourceParams } from "../../Engine";
3
+ import { UserInfo } from "@memberjunction/core";
4
+ import { ContentSourceEntity, ContentItemEntity } from "@memberjunction/core-entities";
5
+ import { OpenAI } from "openai";
6
+ export declare class AutotagLocalFileSystem extends AutotagBase {
7
+ private contextUser;
8
+ private engine;
9
+ protected contentSourceTypeID: string;
10
+ static _openAI: OpenAI;
11
+ constructor();
12
+ getContextUser(): UserInfo | null;
13
+ Autotag(contextUser: UserInfo): Promise<void>;
14
+ SetContentItemsToProcess(contentSources: ContentSourceEntity[]): Promise<ContentItemEntity[]>;
15
+ setContentSourceParams(contentSource: ContentSourceEntity): Promise<ContentSourceParams>;
16
+ SetNewAndModifiedContentItems(contentSourceParams: ContentSourceParams, lastRunDate: Date, contextUser: UserInfo): Promise<ContentItemEntity[]>;
17
+ setAddedContentItem(filePath: string, contentSourceParams: ContentSourceParams): Promise<ContentItemEntity>;
18
+ setModifiedContentItem(filePath: string, contentSourceParams: ContentSourceParams): Promise<ContentItemEntity>;
19
+ }
20
+ //# sourceMappingURL=AutotagLocalFileSystem.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"AutotagLocalFileSystem.d.ts","sourceRoot":"","sources":["../../../../src/LocalFileSystem/generic/AutotagLocalFileSystem.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAqB,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACtE,OAAO,EAAE,QAAQ,EAAqB,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AACvF,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAKhC,qBACa,sBAAuB,SAAQ,WAAW;IACnD,OAAO,CAAC,WAAW,CAAW;IAC9B,OAAO,CAAC,MAAM,CAAoB;IAClC,SAAS,CAAC,mBAAmB,EAAE,MAAM,CAAA;IACrC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC;;IAOhB,cAAc,IAAI,QAAQ,GAAG,IAAI;IAS3B,OAAO,CAAC,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAc7C,wBAAwB,CAAC,cAAc,EAAE,mBAAmB,EAAE,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;IA+B7F,sBAAsB,CAAC,aAAa,EAAE,mBAAmB;IAgCzD,6BAA6B,CAAC,mBAAmB,EAAE,mBAAmB,EAAE,WAAW,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;IA+B/I,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,mBAAmB,EAAE,mBAAmB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAuB3G,sBAAsB,CAAC,QAAQ,EAAE,MAAM,EAAE,mBAAmB,EAAE,mBAAmB,GAAG,OAAO,CAAC,iBAAiB,CAAC;CAgB9H"}
@@ -0,0 +1,155 @@
1
+ "use strict";
2
+ var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
3
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
4
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
5
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
6
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
7
+ };
8
+ var __metadata = (this && this.__metadata) || function (k, v) {
9
+ if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
10
+ };
11
+ var __importDefault = (this && this.__importDefault) || function (mod) {
12
+ return (mod && mod.__esModule) ? mod : { "default": mod };
13
+ };
14
+ Object.defineProperty(exports, "__esModule", { value: true });
15
+ exports.AutotagLocalFileSystem = void 0;
16
+ const global_1 = require("@memberjunction/global");
17
+ const fs_1 = __importDefault(require("fs"));
18
+ const Core_1 = require("../../Core");
19
+ const Engine_1 = require("../../Engine");
20
+ const core_1 = require("@memberjunction/core");
21
+ const path_1 = __importDefault(require("path"));
22
+ const dotenv_1 = __importDefault(require("dotenv"));
23
+ dotenv_1.default.config();
24
+ let AutotagLocalFileSystem = class AutotagLocalFileSystem extends Core_1.AutotagBase {
25
+ contextUser;
26
+ engine;
27
+ contentSourceTypeID;
28
+ static _openAI;
29
+ constructor() {
30
+ super();
31
+ this.engine = Engine_1.AutotagBaseEngine.Instance;
32
+ }
33
+ getContextUser() {
34
+ return this.contextUser;
35
+ }
36
+ async Autotag(contextUser) {
37
+ this.contextUser = contextUser;
38
+ this.contentSourceTypeID = await this.engine.setSubclassContentSourceType('Local File System', this.contextUser);
39
+ const contentSources = await this.engine.getAllContentSources(this.contextUser, this.contentSourceTypeID) || [];
40
+ const contentItemsToProcess = await this.SetContentItemsToProcess(contentSources);
41
+ await this.engine.ExtractTextAndProcessWithLLM(contentItemsToProcess, this.contextUser);
42
+ }
43
+ async SetContentItemsToProcess(contentSources) {
44
+ const contentItemsToProcess = [];
45
+ for (const contentSource of contentSources) {
46
+ if (fs_1.default.existsSync(contentSource.URL)) {
47
+ const contentSourceParams = await this.setContentSourceParams(contentSource);
48
+ const lastRunDate = await this.engine.getContentSourceLastRunDate(contentSourceParams.contentSourceID, this.contextUser);
49
+ if (lastRunDate) {
50
+ const contentItems = await this.SetNewAndModifiedContentItems(contentSourceParams, lastRunDate, this.contextUser);
51
+ if (contentItems && contentItems.length > 0) {
52
+ contentItemsToProcess.push(...contentItems);
53
+ }
54
+ else {
55
+ console.log(`No content items found to process for content source: ${contentSource.Get('Name')}`);
56
+ }
57
+ }
58
+ else {
59
+ throw new Error('Invalid last run date');
60
+ }
61
+ }
62
+ else {
63
+ console.log(`Invalid Content Source ${contentSource.Name}`);
64
+ }
65
+ }
66
+ return contentItemsToProcess;
67
+ }
68
+ async setContentSourceParams(contentSource) {
69
+ const contentSourceParamsMap = await this.engine.getContentSourceParams(contentSource, this.contextUser);
70
+ if (contentSourceParamsMap) {
71
+ contentSourceParamsMap.forEach((value, key) => {
72
+ if (key in this) {
73
+ this[key] = value;
74
+ }
75
+ });
76
+ }
77
+ const contentSourceParams = {
78
+ contentSourceID: contentSource.ID,
79
+ name: contentSource.Name,
80
+ ContentTypeID: contentSource.ContentTypeID,
81
+ ContentSourceTypeID: contentSource.ContentSourceTypeID,
82
+ ContentFileTypeID: contentSource.ContentFileTypeID,
83
+ URL: contentSource.URL
84
+ };
85
+ return contentSourceParams;
86
+ }
87
+ async SetNewAndModifiedContentItems(contentSourceParams, lastRunDate, contextUser) {
88
+ const contentItems = [];
89
+ let contentSourcePath = contentSourceParams.URL;
90
+ const filesAndDirs = fs_1.default.readdirSync(contentSourcePath);
91
+ for (const file of filesAndDirs) {
92
+ const filePath = path_1.default.join(contentSourcePath, file);
93
+ const stats = fs_1.default.statSync(filePath);
94
+ if (stats.isDirectory()) {
95
+ contentSourceParams.URL = filePath;
96
+ await this.SetNewAndModifiedContentItems(contentSourceParams, lastRunDate, contextUser);
97
+ }
98
+ else if (stats.isFile()) {
99
+ const modifiedDate = new Date(stats.mtime.toUTCString());
100
+ const changedDate = new Date(stats.ctime.toUTCString());
101
+ if (changedDate > lastRunDate) {
102
+ const contentItem = await this.setAddedContentItem(filePath, contentSourceParams);
103
+ contentItems.push(contentItem);
104
+ }
105
+ else if (modifiedDate > lastRunDate) {
106
+ const contentItem = await this.setModifiedContentItem(filePath, contentSourceParams);
107
+ contentItems.push(contentItem);
108
+ }
109
+ }
110
+ }
111
+ return contentItems;
112
+ }
113
+ async setAddedContentItem(filePath, contentSourceParams) {
114
+ const md = new core_1.Metadata();
115
+ const text = await this.engine.parseFileFromPath(filePath);
116
+ const contentItem = await md.GetEntityObject('Content Items', this.contextUser);
117
+ contentItem.NewRecord();
118
+ contentItem.ContentSourceID = contentSourceParams.contentSourceID;
119
+ contentItem.Name = contentSourceParams.name;
120
+ contentItem.Description = await this.engine.getContentItemDescription(contentSourceParams, this.contextUser);
121
+ contentItem.ContentTypeID = contentSourceParams.ContentTypeID;
122
+ contentItem.ContentFileTypeID = contentSourceParams.ContentFileTypeID;
123
+ contentItem.ContentSourceTypeID = contentSourceParams.ContentSourceTypeID;
124
+ contentItem.Checksum = await this.engine.getChecksumFromText(text);
125
+ contentItem.URL = contentSourceParams.URL;
126
+ contentItem.Text = text;
127
+ if (await contentItem.Save()) {
128
+ return contentItem;
129
+ }
130
+ else {
131
+ throw new Error('Failed to save content item');
132
+ }
133
+ }
134
+ async setModifiedContentItem(filePath, contentSourceParams) {
135
+ const md = new core_1.Metadata();
136
+ const contentItem = await md.GetEntityObject('Content Items', this.contextUser);
137
+ const contentItemID = await this.engine.getContentItemIDFromURL(contentSourceParams, this.contextUser);
138
+ await contentItem.Load(contentItemID);
139
+ const text = await this.engine.parseFileFromPath(filePath);
140
+ contentItem.Text = text;
141
+ contentItem.Checksum = await this.engine.getChecksumFromText(text);
142
+ if (await contentItem.Save()) {
143
+ return contentItem;
144
+ }
145
+ else {
146
+ throw new Error('Failed to save content item');
147
+ }
148
+ }
149
+ };
150
+ exports.AutotagLocalFileSystem = AutotagLocalFileSystem;
151
+ exports.AutotagLocalFileSystem = AutotagLocalFileSystem = __decorate([
152
+ (0, global_1.RegisterClass)(Core_1.AutotagBase, 'AutotagLocalFileSystem'),
153
+ __metadata("design:paramtypes", [])
154
+ ], AutotagLocalFileSystem);
155
+ //# sourceMappingURL=AutotagLocalFileSystem.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"AutotagLocalFileSystem.js","sourceRoot":"","sources":["../../../../src/LocalFileSystem/generic/AutotagLocalFileSystem.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,mDAAuD;AACvD,4CAAoB;AACpB,qCAAyC;AACzC,yCAAsE;AACtE,+CAAmE;AAGnE,gDAAwB;AACxB,oDAA4B;AAC5B,gBAAM,CAAC,MAAM,EAAE,CAAA;AAGR,IAAM,sBAAsB,GAA5B,MAAM,sBAAuB,SAAQ,kBAAW;IAC3C,WAAW,CAAW;IACtB,MAAM,CAAoB;IACxB,mBAAmB,CAAQ;IACrC,MAAM,CAAC,OAAO,CAAS;IAEvB;QACI,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,0BAAiB,CAAC,QAAQ,CAAC;IAC7C,CAAC;IAEM,cAAc;QACjB,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAOM,KAAK,CAAC,OAAO,CAAC,WAAqB;QACtC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,mBAAmB,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,mBAAmB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACjH,MAAM,cAAc,GAA0B,MAAM,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC;QACvI,MAAM,qBAAqB,GAAwB,MAAM,IAAI,CAAC,wBAAwB,CAAC,cAAc,CAAC,CAAA;QACtG,MAAM,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,qBAAqB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAC5F,CAAC;IAQM,KAAK,CAAC,wBAAwB,CAAC,cAAqC;QACvE,MAAM,qBAAqB,GAAwB,EAAE,CAAA;QAErD,KAAK,MAAM,aAAa,IAAI,cAAc,EAAE,CAAC;YAEzC,IAAI,YAAE,CAAC,UAAU,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC;gBACnC,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,aAAa,CAAC,CAAC;gBAC7E,MAAM,WAAW,GAAS,MAAM,IAAI,CAAC,MAAM,CAAC,2BAA2B,CAAC,mBAAmB,CAAC,eAAe,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;gBAG9H,IAAI,WAAW,EAAE,CAAC;oBAClB,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,mBAAmB,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;oBAC9G,IAAI,YAAY,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC1C,qBAAqB,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;oBAChD,CAAC;yBACI,CAAC;wBAEF,OAAO,CAAC,GAAG,CAAC,yDAAyD,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;oBACtG,CAAC;gBACL,CAAC;qBACI,CAAC;oBACN,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;gBACzC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACJ,OAAO,CAAC,GAAG,CAAC,0BAA0B,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC;YAChE,CAAC;QACL,CAAC;QAED,OAAO,qBAAqB,CAAC;IACjC,CAAC;IAEM,KAAK,CAAC,sBAAsB,CAAC,aAAkC;QAElE,MAAM,sBAAsB,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,aAAa,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACzG,IAAI,sBAAsB,EAAE,CAAC;YAEzB,sBAAsB,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;gBAC1C,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;oBACb,IAAY,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;gBAC/B,CAAC;YACL,CAAC,CAAC,CAAA;QACN,CAAC;QAED,MAAM,mBAAmB,GAAwB;YAC7C,eAAe,EAAE,aAAa,CAAC,EAAE;YACjC,IAAI,EAAE,aAAa,CAAC,IAAI;YACxB,aAAa,EAAE,aAAa,CAAC,aAAa;YAC1C,mBAAmB,EAAE,aAAa,CAAC,mBAAmB;YACtD,iBAAiB,EAAE,aAAa,CAAC,iBAAiB;YAClD,GAAG,EAAE,aAAa,CAAC,GAAG;SACzB,CAAA;QAED,OAAO,mBAAmB,CAAC;IAC/B,CAAC;IAUM,KAAK,CAAC,6BAA6B,CAAC,mBAAwC,EAAE,WAAiB,EAAE,WAAqB;QACzH,MAAM,YAAY,GAAwB,EAAE,CAAA;QAC5C,IAAI,iBAAiB,GAAG,mBAAmB,CAAC,GAAG,CAAA;QAC/C,MAAM,YAAY,GAAG,YAAE,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAA;QAEtD,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;YAC9B,MAAM,QAAQ,GAAG,cAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAAA;YACnD,MAAM,KAAK,GAAG,YAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;YACnC,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACtB,mBAAmB,CAAC,GAAG,GAAG,QAAQ,CAAA;gBAClC,MAAM,IAAI,CAAC,6BAA6B,CAAC,mBAAmB,EAAE,WAAW,EAAE,WAAW,CAAC,CAAA;YAC3F,CAAC;iBAEI,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;gBACtB,MAAM,YAAY,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAA;gBACxD,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAA;gBACvD,IAAI,WAAW,GAAG,WAAW,EAAE,CAAC;oBAE5B,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EAAE,mBAAmB,CAAC,CAAC;oBAClF,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBACnC,CAAC;qBACI,IAAI,YAAY,GAAG,WAAW,EAAE,CAAC;oBAElC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,QAAQ,EAAE,mBAAmB,CAAC,CAAC;oBACrF,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBACnC,CAAC;YACL,CAAC;QACL,CAAC;QACD,OAAO,YAAY,CAAC;IACxB,CAAC;IAEM,KAAK,CAAC,mBAAmB,CAAC,QAAgB,EAAE,mBAAwC;QACvF,MAAM,EAAE,GAAG,IAAI,eAAQ,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAC3D,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,eAAe,CAAM,eAAe,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACrF,WAAW,CAAC,SAAS,EAAE,CAAC;QACxB,WAAW,CAAC,eAAe,GAAG,mBAAmB,CAAC,eAAe,CAAA;QACjE,WAAW,CAAC,IAAI,GAAG,mBAAmB,CAAC,IAAI,CAAA;QAC3C,WAAW,CAAC,WAAW,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,yBAAyB,CAAC,mBAAmB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;QAC5G,WAAW,CAAC,aAAa,GAAG,mBAAmB,CAAC,aAAa,CAAA;QAC7D,WAAW,CAAC,iBAAiB,GAAG,mBAAmB,CAAC,iBAAiB,CAAA;QACrE,WAAW,CAAC,mBAAmB,GAAG,mBAAmB,CAAC,mBAAmB,CAAA;QACzE,WAAW,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAA;QAClE,WAAW,CAAC,GAAG,GAAG,mBAAmB,CAAC,GAAG,CAAA;QACzC,WAAW,CAAC,IAAI,GAAG,IAAI,CAAA;QAEvB,IAAG,MAAM,WAAW,CAAC,IAAI,EAAE,EAAC,CAAC;YACzB,OAAO,WAAW,CAAC;QACvB,CAAC;aACI,CAAC;YACF,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACnD,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,sBAAsB,CAAC,QAAgB,EAAE,mBAAwC;QAC1F,MAAM,EAAE,GAAG,IAAI,eAAQ,EAAE,CAAC;QAC1B,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,eAAe,CAAM,eAAe,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACrF,MAAM,aAAa,GAAW,MAAM,IAAI,CAAC,MAAM,CAAC,uBAAuB,CAAC,mBAAmB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAC/G,MAAM,WAAW,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACtC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAC3D,WAAW,CAAC,IAAI,GAAG,IAAI,CAAA;QACvB,WAAW,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAA;QAElE,IAAG,MAAM,WAAW,CAAC,IAAI,EAAE,EAAC,CAAC;YACzB,OAAO,WAAW,CAAC;QACvB,CAAC;aACI,CAAC;YACF,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACnD,CAAC;IACL,CAAC;CACJ,CAAA;AAvKY,wDAAsB;iCAAtB,sBAAsB;IADlC,IAAA,sBAAa,EAAC,kBAAW,EAAE,wBAAwB,CAAC;;GACxC,sBAAsB,CAuKlC"}
@@ -0,0 +1,2 @@
1
+ export * from './generic/AutotagLocalFileSystem';
2
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/LocalFileSystem/index.ts"],"names":[],"mappings":"AAAA,cAAc,kCAAkC,CAAA"}
@@ -0,0 +1,18 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
+ for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
+ };
16
+ Object.defineProperty(exports, "__esModule", { value: true });
17
+ __exportStar(require("./generic/AutotagLocalFileSystem"), exports);
18
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/LocalFileSystem/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,mEAAgD"}
@@ -0,0 +1,19 @@
1
+ import { UserInfo } from '@memberjunction/core';
2
+ import { AutotagBase } from "../../Core";
3
+ import { ContentSourceParams } from "../../Engine";
4
+ import { ContentSourceEntity, ContentItemEntity } from '@memberjunction/core-entities';
5
+ import { RSSItem } from './RSS.types';
6
+ export declare class AutotagRSSFeed extends AutotagBase {
7
+ private contextUser;
8
+ private engine;
9
+ protected contentSourceTypeID: string;
10
+ constructor();
11
+ protected getContextUser(): UserInfo;
12
+ Autotag(contextUser: UserInfo): Promise<void>;
13
+ SetContentItemsToProcess(contentSources: ContentSourceEntity[]): Promise<ContentItemEntity[]>;
14
+ SetNewAndModifiedContentItems(allRSSItems: RSSItem[], contentSourceParams: ContentSourceParams): Promise<ContentItemEntity[]>;
15
+ parseRSSFeed(url: string): Promise<RSSItem[]>;
16
+ protected urlIsValid(url: string): Promise<boolean>;
17
+ getChecksumFromRSSItem(RSSContentItem: RSSItem, contextUser: UserInfo): Promise<string>;
18
+ }
19
+ //# sourceMappingURL=AutotagRSSFeed.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"AutotagRSSFeed.d.ts","sourceRoot":"","sources":["../../../../src/RSSFeed/generic/AutotagRSSFeed.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAqB,MAAM,sBAAsB,CAAC;AAEnE,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAqB,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACtE,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AACvF,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAOtC,qBACa,cAAe,SAAQ,WAAW;IAC3C,OAAO,CAAC,WAAW,CAAW;IAC9B,OAAO,CAAC,MAAM,CAAoB;IAClC,SAAS,CAAC,mBAAmB,EAAE,MAAM,CAAA;;IAOrC,SAAS,CAAC,cAAc,IAAI,QAAQ;IASvB,OAAO,CAAC,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAc7C,wBAAwB,CAAC,cAAc,EAAE,mBAAmB,EAAE,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;IAuC7F,6BAA6B,CAAC,WAAW,EAAE,OAAO,EAAE,EAAE,mBAAmB,EAAE,mBAAmB,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;IAkD7H,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;cAqC1C,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAW5C,sBAAsB,CAAC,cAAc,EAAE,OAAO,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;CAIvG"}
@@ -0,0 +1,168 @@
1
+ "use strict";
2
+ var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
3
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
4
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
5
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
6
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
7
+ };
8
+ var __metadata = (this && this.__metadata) || function (k, v) {
9
+ if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
10
+ };
11
+ var __importDefault = (this && this.__importDefault) || function (mod) {
12
+ return (mod && mod.__esModule) ? mod : { "default": mod };
13
+ };
14
+ Object.defineProperty(exports, "__esModule", { value: true });
15
+ exports.AutotagRSSFeed = void 0;
16
+ const core_1 = require("@memberjunction/core");
17
+ const global_1 = require("@memberjunction/global");
18
+ const Core_1 = require("../../Core");
19
+ const Engine_1 = require("../../Engine");
20
+ const RSS_types_1 = require("./RSS.types");
21
+ const axios_1 = __importDefault(require("axios"));
22
+ const crypto_1 = __importDefault(require("crypto"));
23
+ const rss_parser_1 = __importDefault(require("rss-parser"));
24
+ const dotenv_1 = __importDefault(require("dotenv"));
25
+ dotenv_1.default.config();
26
+ let AutotagRSSFeed = class AutotagRSSFeed extends Core_1.AutotagBase {
27
+ contextUser;
28
+ engine;
29
+ contentSourceTypeID;
30
+ constructor() {
31
+ super();
32
+ this.engine = Engine_1.AutotagBaseEngine.Instance;
33
+ }
34
+ getContextUser() {
35
+ return this.contextUser;
36
+ }
37
+ async Autotag(contextUser) {
38
+ this.contextUser = contextUser;
39
+ this.contentSourceTypeID = await this.engine.setSubclassContentSourceType('RSS Feed', this.contextUser);
40
+ const contentSources = await this.engine.getAllContentSources(this.contextUser, this.contentSourceTypeID);
41
+ const contentItemsToProcess = await this.SetContentItemsToProcess(contentSources);
42
+ await this.engine.ExtractTextAndProcessWithLLM(contentItemsToProcess, this.contextUser);
43
+ }
44
+ async SetContentItemsToProcess(contentSources) {
45
+ const contentItemsToProcess = [];
46
+ for (const contentSource of contentSources) {
47
+ const contentSourceParamsMap = await this.engine.getContentSourceParams(contentSource, this.contextUser);
48
+ if (contentSourceParamsMap) {
49
+ contentSourceParamsMap.forEach((value, key) => {
50
+ if (key in this) {
51
+ this[key] = value;
52
+ }
53
+ });
54
+ }
55
+ const contentSourceParams = {
56
+ contentSourceID: contentSource.ID,
57
+ name: contentSource.Name,
58
+ ContentTypeID: contentSource.ContentTypeID,
59
+ ContentFileTypeID: contentSource.ContentFileTypeID,
60
+ ContentSourceTypeID: contentSource.ContentSourceTypeID,
61
+ URL: contentSource.URL
62
+ };
63
+ const allRSSItems = await this.parseRSSFeed(contentSourceParams.URL);
64
+ const contentItems = await this.SetNewAndModifiedContentItems(allRSSItems, contentSourceParams);
65
+ if (contentItems && contentItems.length > 0) {
66
+ contentItemsToProcess.push(...contentItems);
67
+ }
68
+ else {
69
+ console.log(`No content items found to process for content source: ${contentSource.Get('Name')}`);
70
+ }
71
+ }
72
+ return contentItemsToProcess;
73
+ }
74
+ async SetNewAndModifiedContentItems(allRSSItems, contentSourceParams) {
75
+ const contentItemsToProcess = [];
76
+ for (const RSSContentItem of allRSSItems) {
77
+ const rv = new core_1.RunView();
78
+ const results = await rv.RunView({
79
+ EntityName: 'Content Items',
80
+ ExtraFilter: `ContentSourceID = '${contentSourceParams.contentSourceID}' AND (URL = '${RSSContentItem.link}' OR Description = '${RSSContentItem.description}')`,
81
+ ResultType: 'entity_object',
82
+ }, this.contextUser);
83
+ if (results.Success && results.Results.length) {
84
+ const contentItemResult = results.Results[0];
85
+ const lastStoredHash = contentItemResult.Checksum;
86
+ const newHash = await this.getChecksumFromRSSItem(RSSContentItem, this.contextUser);
87
+ if (lastStoredHash !== newHash) {
88
+ const md = new core_1.Metadata();
89
+ const contentItem = await md.GetEntityObject('Content Items', this.contextUser);
90
+ contentItem.Load(contentItemResult.ID);
91
+ contentItem.Checksum = newHash;
92
+ contentItem.Text = JSON.stringify(RSSContentItem);
93
+ await contentItem.Save();
94
+ contentItemsToProcess.push(contentItem);
95
+ }
96
+ }
97
+ else {
98
+ const md = new core_1.Metadata();
99
+ const contentItem = await md.GetEntityObject('Content Items', this.contextUser);
100
+ contentItem.ContentSourceID = contentSourceParams.contentSourceID;
101
+ contentItem.Name = contentSourceParams.name;
102
+ contentItem.Description = RSSContentItem.description || await this.engine.getContentItemDescription(contentSourceParams, this.contextUser);
103
+ contentItem.ContentTypeID = contentSourceParams.ContentTypeID;
104
+ contentItem.ContentFileTypeID = contentSourceParams.ContentFileTypeID;
105
+ contentItem.ContentSourceTypeID = contentSourceParams.ContentSourceTypeID;
106
+ contentItem.Checksum = await this.getChecksumFromRSSItem(RSSContentItem, this.contextUser);
107
+ contentItem.URL = RSSContentItem.link || contentSourceParams.URL;
108
+ contentItem.Text = JSON.stringify(RSSContentItem);
109
+ await contentItem.Save();
110
+ contentItemsToProcess.push(contentItem);
111
+ }
112
+ }
113
+ return contentItemsToProcess;
114
+ }
115
+ async parseRSSFeed(url) {
116
+ try {
117
+ if (await this.urlIsValid(url)) {
118
+ const RSSItems = [];
119
+ const parser = new rss_parser_1.default();
120
+ const feed = await parser.parseURL(url);
121
+ const items = feed.items;
122
+ items.forEach(async (item) => {
123
+ const rssItem = new RSS_types_1.RSSItem();
124
+ rssItem.title = item.title ?? '';
125
+ rssItem.link = item.link ?? '';
126
+ rssItem.description = item.description ?? '';
127
+ rssItem.pubDate = item.pubDate ?? '';
128
+ rssItem.guid = item.guid ?? '';
129
+ rssItem.category = item.category ?? '';
130
+ const content = item['content:encoded'] ?? item['content'] ?? '';
131
+ rssItem.content = await this.engine.parseHTML(content);
132
+ rssItem.author = item.author ?? '';
133
+ rssItem.comments = item.comments ?? '';
134
+ rssItem.source = item.source ?? '';
135
+ RSSItems.push(rssItem);
136
+ });
137
+ return RSSItems;
138
+ }
139
+ else {
140
+ throw new Error(`Invalid URL: ${url}`);
141
+ }
142
+ }
143
+ catch (error) {
144
+ console.error('Error fetching RSS feed:', error);
145
+ return [];
146
+ }
147
+ }
148
+ async urlIsValid(url) {
149
+ try {
150
+ const response = await axios_1.default.head(url);
151
+ return response.status === 200;
152
+ }
153
+ catch (e) {
154
+ console.error(`Invalid URL: ${url}`);
155
+ return false;
156
+ }
157
+ }
158
+ async getChecksumFromRSSItem(RSSContentItem, contextUser) {
159
+ const hash = crypto_1.default.createHash('sha256').update(JSON.stringify(RSSContentItem)).digest('hex');
160
+ return hash;
161
+ }
162
+ };
163
+ exports.AutotagRSSFeed = AutotagRSSFeed;
164
+ exports.AutotagRSSFeed = AutotagRSSFeed = __decorate([
165
+ (0, global_1.RegisterClass)(Core_1.AutotagBase, 'AutotagRSSFeed'),
166
+ __metadata("design:paramtypes", [])
167
+ ], AutotagRSSFeed);
168
+ //# sourceMappingURL=AutotagRSSFeed.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"AutotagRSSFeed.js","sourceRoot":"","sources":["../../../../src/RSSFeed/generic/AutotagRSSFeed.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,+CAAmE;AACnE,mDAAuD;AACvD,qCAAyC;AACzC,yCAAsE;AAEtE,2CAAsC;AACtC,kDAAyB;AACzB,oDAA2B;AAC3B,4DAA+B;AAC/B,oDAA4B;AAC5B,gBAAM,CAAC,MAAM,EAAE,CAAA;AAGR,IAAM,cAAc,GAApB,MAAM,cAAe,SAAQ,kBAAW;IACnC,WAAW,CAAW;IACtB,MAAM,CAAoB;IACxB,mBAAmB,CAAQ;IAErC;QACI,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,0BAAiB,CAAC,QAAQ,CAAC;IAC7C,CAAC;IAES,cAAc;QACpB,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAOM,KAAK,CAAC,OAAO,CAAC,WAAqB;QACtC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,mBAAmB,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,UAAU,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACxG,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC1G,MAAM,qBAAqB,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,cAAc,CAAC,CAAC;QAClF,MAAM,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,qBAAqB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAC5F,CAAC;IAQM,KAAK,CAAC,wBAAwB,CAAC,cAAqC;QACvE,MAAM,qBAAqB,GAAwB,EAAE,CAAA;QACrD,KAAK,MAAM,aAAa,IAAI,cAAc,EAAE,CAAC;YAGzC,MAAM,sBAAsB,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,aAAa,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YACzG,IAAI,sBAAsB,EAAE,CAAC;gBAEzB,sBAAsB,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;oBAC1C,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;wBACb,IAAY,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;oBAC/B,CAAC;gBACL,CAAC,CAAC,CAAA;YACN,CAAC;YAED,MAAM,mBAAmB,GAAwB;gBAC7C,eAAe,EAAE,aAAa,CAAC,EAAE;gBACjC,IAAI,EAAE,aAAa,CAAC,IAAI;gBACxB,aAAa,EAAE,aAAa,CAAC,aAAa;gBAC1C,iBAAiB,EAAE,aAAa,CAAC,iBAAiB;gBAClD,mBAAmB,EAAE,aAAa,CAAC,mBAAmB;gBACtD,GAAG,EAAE,aAAa,CAAC,GAAG;aACzB,CAAA;YAED,MAAM,WAAW,GAAc,MAAM,IAAI,CAAC,YAAY,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;YAEhF,MAAM,YAAY,GAAwB,MAAM,IAAI,CAAC,6BAA6B,CAAC,WAAW,EAAE,mBAAmB,CAAC,CAAA;YAEpH,IAAI,YAAY,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1C,qBAAqB,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;YAChD,CAAC;iBACI,CAAC;gBAEF,OAAO,CAAC,GAAG,CAAC,yDAAyD,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACtG,CAAC;QACL,CAAC;QACD,OAAO,qBAAqB,CAAA;IAChC,CAAC;IAEM,KAAK,CAAC,6BAA6B,CAAC,WAAsB,EAAE,mBAAwC;QACvG,MAAM,qBAAqB,GAAwB,EAAE,CAAC;QACtD,KAAK,MAAM,cAAc,IAAI,WAAW,EAAE,CAAC;YACvC,MAAM,EAAE,GAAG,IAAI,cAAO,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC;gBAC7B,UAAU,EAAE,eAAe;gBAC3B,WAAW,EAAE,sBAAsB,mBAAmB,CAAC,eAAe,iBAAiB,cAAc,CAAC,IAAI,uBAAuB,cAAc,CAAC,WAAW,IAAI;gBAC/J,UAAU,EAAE,eAAe;aAC9B,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;YAEpB,IAAI,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC5C,MAAM,iBAAiB,GAAuB,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBAEjE,MAAM,cAAc,GAAW,iBAAiB,CAAC,QAAQ,CAAA;gBACzD,MAAM,OAAO,GAAW,MAAM,IAAI,CAAC,sBAAsB,CAAC,cAAc,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;gBAE3F,IAAI,cAAc,KAAK,OAAO,EAAE,CAAC;oBAE7B,MAAM,EAAE,GAAG,IAAI,eAAQ,EAAE,CAAC;oBAC1B,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,eAAe,CAAoB,eAAe,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;oBACnG,WAAW,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;oBACvC,WAAW,CAAC,QAAQ,GAAG,OAAO,CAAA;oBAC9B,WAAW,CAAC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAA;oBAEjD,MAAM,WAAW,CAAC,IAAI,EAAE,CAAC;oBACzB,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBAC5C,CAAC;YACL,CAAC;iBACI,CAAC;gBAEF,MAAM,EAAE,GAAG,IAAI,eAAQ,EAAE,CAAC;gBAC1B,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,eAAe,CAAoB,eAAe,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;gBACnG,WAAW,CAAC,eAAe,GAAG,mBAAmB,CAAC,eAAe,CAAA;gBACjE,WAAW,CAAC,IAAI,GAAG,mBAAmB,CAAC,IAAI,CAAA;gBAC3C,WAAW,CAAC,WAAW,GAAG,cAAc,CAAC,WAAW,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,yBAAyB,CAAC,mBAAmB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;gBAC1I,WAAW,CAAC,aAAa,GAAG,mBAAmB,CAAC,aAAa,CAAA;gBAC7D,WAAW,CAAC,iBAAiB,GAAG,mBAAmB,CAAC,iBAAiB,CAAA;gBACrE,WAAW,CAAC,mBAAmB,GAAG,mBAAmB,CAAC,mBAAmB,CAAA;gBACzE,WAAW,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,cAAc,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;gBAC1F,WAAW,CAAC,GAAG,GAAG,cAAc,CAAC,IAAI,IAAI,mBAAmB,CAAC,GAAG,CAAA;gBAChE,WAAW,CAAC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAA;gBAEjD,MAAM,WAAW,CAAC,IAAI,EAAE,CAAC;gBACzB,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAE5C,CAAC;QACL,CAAC;QACD,OAAO,qBAAqB,CAAA;IAChC,CAAC;IAEM,KAAK,CAAC,YAAY,CAAC,GAAW;QACjC,IAAI,CAAC;YACD,IAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,QAAQ,GAAc,EAAE,CAAA;gBAC9B,MAAM,MAAM,GAAG,IAAI,oBAAM,EAAE,CAAC;gBAC5B,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;gBACxC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;gBAGzB,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,IAAS,EAAE,EAAE;oBAC9B,MAAM,OAAO,GAAG,IAAI,mBAAO,EAAE,CAAC;oBAC9B,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;oBACjC,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;oBAC/B,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;oBAC7C,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;oBACrC,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;oBAC/B,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;oBACvC,MAAM,OAAO,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;oBACjE,OAAO,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;oBACvD,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;oBACnC,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;oBACvC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;oBACnC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC3B,CAAC,CAAC,CAAC;gBAEH,OAAO,QAAQ,CAAA;YACnB,CAAC;iBACI,CAAC;gBACF,MAAM,IAAI,KAAK,CAAC,gBAAgB,GAAG,EAAE,CAAC,CAAC;YAC3C,CAAC;QACL,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,0BAA0B,EAAE,KAAK,CAAC,CAAC;YACjD,OAAO,EAAE,CAAC;QACZ,CAAC;IACP,CAAC;IAES,KAAK,CAAC,UAAU,CAAC,GAAW;QAClC,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,eAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvC,OAAO,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC;QACnC,CAAC;QACD,OAAO,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,KAAK,CAAC,gBAAgB,GAAG,EAAE,CAAC,CAAC;YACrC,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,sBAAsB,CAAC,cAAuB,EAAE,WAAqB;QAC9E,MAAM,IAAI,GAAG,gBAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAC7F,OAAO,IAAI,CAAA;IACf,CAAC;CACJ,CAAA;AA9KY,wCAAc;yBAAd,cAAc;IAD1B,IAAA,sBAAa,EAAC,kBAAW,EAAE,gBAAgB,CAAC;;GAChC,cAAc,CA8K1B"}
@@ -0,0 +1,13 @@
1
+ export declare class RSSItem {
2
+ title?: string;
3
+ link?: string;
4
+ description?: string;
5
+ pubDate?: string;
6
+ guid?: string;
7
+ category?: string;
8
+ content?: string;
9
+ author?: string;
10
+ comments?: string;
11
+ source?: string;
12
+ }
13
+ //# sourceMappingURL=RSS.types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"RSS.types.d.ts","sourceRoot":"","sources":["../../../../src/RSSFeed/generic/RSS.types.ts"],"names":[],"mappings":"AAAA,qBAAa,OAAO;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB"}
@@ -0,0 +1,17 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.RSSItem = void 0;
4
+ class RSSItem {
5
+ title;
6
+ link;
7
+ description;
8
+ pubDate;
9
+ guid;
10
+ category;
11
+ content;
12
+ author;
13
+ comments;
14
+ source;
15
+ }
16
+ exports.RSSItem = RSSItem;
17
+ //# sourceMappingURL=RSS.types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"RSS.types.js","sourceRoot":"","sources":["../../../../src/RSSFeed/generic/RSS.types.ts"],"names":[],"mappings":";;;AAAA,MAAa,OAAO;IAChB,KAAK,CAAU;IACf,IAAI,CAAU;IACd,WAAW,CAAU;IACrB,OAAO,CAAU;IACjB,IAAI,CAAU;IACd,QAAQ,CAAU;IAClB,OAAO,CAAU;IACjB,MAAM,CAAU;IAChB,QAAQ,CAAU;IAClB,MAAM,CAAU;CACnB;AAXD,0BAWC"}
@@ -0,0 +1,3 @@
1
+ export * from './generic/RSS.types';
2
+ export * from './generic/AutotagRSSFeed';
3
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/RSSFeed/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAA;AACnC,cAAc,0BAA0B,CAAA"}
@@ -0,0 +1,19 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
+ for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
+ };
16
+ Object.defineProperty(exports, "__esModule", { value: true });
17
+ __exportStar(require("./generic/RSS.types"), exports);
18
+ __exportStar(require("./generic/AutotagRSSFeed"), exports);
19
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/RSSFeed/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,sDAAmC;AACnC,2DAAwC"}