@memberjunction/content-autotagging 2.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/dist/src/CloudStorage/generic/CloudStorageBase.d.ts +16 -0
  2. package/dist/src/CloudStorage/generic/CloudStorageBase.d.ts.map +1 -0
  3. package/dist/src/CloudStorage/generic/CloudStorageBase.js +48 -0
  4. package/dist/src/CloudStorage/generic/CloudStorageBase.js.map +1 -0
  5. package/dist/src/CloudStorage/index.d.ts +3 -0
  6. package/dist/src/CloudStorage/index.d.ts.map +1 -0
  7. package/dist/src/CloudStorage/index.js +19 -0
  8. package/dist/src/CloudStorage/index.js.map +1 -0
  9. package/dist/src/CloudStorage/providers/AutotagAzureBlob.d.ts +18 -0
  10. package/dist/src/CloudStorage/providers/AutotagAzureBlob.d.ts.map +1 -0
  11. package/dist/src/CloudStorage/providers/AutotagAzureBlob.js +91 -0
  12. package/dist/src/CloudStorage/providers/AutotagAzureBlob.js.map +1 -0
  13. package/dist/src/Core/generic/AutotagBase.d.ts +7 -0
  14. package/dist/src/Core/generic/AutotagBase.d.ts.map +1 -0
  15. package/dist/src/Core/generic/AutotagBase.js +7 -0
  16. package/dist/src/Core/generic/AutotagBase.js.map +1 -0
  17. package/dist/src/Core/index.d.ts +2 -0
  18. package/dist/src/Core/index.d.ts.map +1 -0
  19. package/dist/src/Core/index.js +18 -0
  20. package/dist/src/Core/index.js.map +1 -0
  21. package/dist/src/Engine/generic/AutotagBaseEngine.d.ts +51 -0
  22. package/dist/src/Engine/generic/AutotagBaseEngine.d.ts.map +1 -0
  23. package/dist/src/Engine/generic/AutotagBaseEngine.js +553 -0
  24. package/dist/src/Engine/generic/AutotagBaseEngine.js.map +1 -0
  25. package/dist/src/Engine/generic/content.types.d.ts +32 -0
  26. package/dist/src/Engine/generic/content.types.d.ts.map +1 -0
  27. package/dist/src/Engine/generic/content.types.js +38 -0
  28. package/dist/src/Engine/generic/content.types.js.map +1 -0
  29. package/dist/src/Engine/generic/process.types.d.ts +30 -0
  30. package/dist/src/Engine/generic/process.types.d.ts.map +1 -0
  31. package/dist/src/Engine/generic/process.types.js +33 -0
  32. package/dist/src/Engine/generic/process.types.js.map +1 -0
  33. package/dist/src/Engine/index.d.ts +4 -0
  34. package/dist/src/Engine/index.d.ts.map +1 -0
  35. package/dist/src/Engine/index.js +20 -0
  36. package/dist/src/Engine/index.js.map +1 -0
  37. package/dist/src/LocalFileSystem/generic/AutotagLocalFileSystem.d.ts +20 -0
  38. package/dist/src/LocalFileSystem/generic/AutotagLocalFileSystem.d.ts.map +1 -0
  39. package/dist/src/LocalFileSystem/generic/AutotagLocalFileSystem.js +155 -0
  40. package/dist/src/LocalFileSystem/generic/AutotagLocalFileSystem.js.map +1 -0
  41. package/dist/src/LocalFileSystem/index.d.ts +2 -0
  42. package/dist/src/LocalFileSystem/index.d.ts.map +1 -0
  43. package/dist/src/LocalFileSystem/index.js +18 -0
  44. package/dist/src/LocalFileSystem/index.js.map +1 -0
  45. package/dist/src/RSSFeed/generic/AutotagRSSFeed.d.ts +19 -0
  46. package/dist/src/RSSFeed/generic/AutotagRSSFeed.d.ts.map +1 -0
  47. package/dist/src/RSSFeed/generic/AutotagRSSFeed.js +168 -0
  48. package/dist/src/RSSFeed/generic/AutotagRSSFeed.js.map +1 -0
  49. package/dist/src/RSSFeed/generic/RSS.types.d.ts +13 -0
  50. package/dist/src/RSSFeed/generic/RSS.types.d.ts.map +1 -0
  51. package/dist/src/RSSFeed/generic/RSS.types.js +17 -0
  52. package/dist/src/RSSFeed/generic/RSS.types.js.map +1 -0
  53. package/dist/src/RSSFeed/index.d.ts +3 -0
  54. package/dist/src/RSSFeed/index.d.ts.map +1 -0
  55. package/dist/src/RSSFeed/index.js +19 -0
  56. package/dist/src/RSSFeed/index.js.map +1 -0
  57. package/dist/src/Websites/generic/AutotagWebsite.d.ts +33 -0
  58. package/dist/src/Websites/generic/AutotagWebsite.d.ts.map +1 -0
  59. package/dist/src/Websites/generic/AutotagWebsite.js +324 -0
  60. package/dist/src/Websites/generic/AutotagWebsite.js.map +1 -0
  61. package/dist/src/Websites/index.d.ts +2 -0
  62. package/dist/src/Websites/index.d.ts.map +1 -0
  63. package/dist/src/Websites/index.js +18 -0
  64. package/dist/src/Websites/index.js.map +1 -0
  65. package/dist/src/index.d.ts +7 -0
  66. package/dist/src/index.d.ts.map +1 -0
  67. package/dist/src/index.js +23 -0
  68. package/dist/src/index.js.map +1 -0
  69. package/dist/tsconfig.tsbuildinfo +1 -0
  70. package/package.json +45 -0
@@ -0,0 +1,33 @@
1
+ import { AutotagBase } from '../../Core';
2
+ import { ContentSourceParams } from '../../Engine';
3
+ import { UserInfo } from '@memberjunction/core';
4
+ import { ContentSourceEntity, ContentItemEntity } from '@memberjunction/core-entities';
5
+ import * as cheerio from 'cheerio';
6
+ export declare class AutotagWebsite extends AutotagBase {
7
+ private contextUser;
8
+ private engine;
9
+ protected contentSourceTypeID: string;
10
+ protected CrawlOtherSitesInTopLevelDomain: boolean;
11
+ protected CrawlSitesInLowerLevelDomain: boolean;
12
+ protected MaxDepth: number;
13
+ protected RootURL: string;
14
+ protected URLPattern: string;
15
+ protected visitedURLs: Set<string>;
16
+ constructor();
17
+ protected getContextUser(): UserInfo;
18
+ Autotag(contextUser: UserInfo): Promise<void>;
19
+ SetContentItemsToProcess(contentSources: ContentSourceEntity[]): Promise<ContentItemEntity[]>;
20
+ protected SetNewAndModifiedContentItems(contentItemLinks: string[], contentSourceParams: ContentSourceParams, contextUser: UserInfo): Promise<ContentItemEntity[]>;
21
+ fetchPageContent(url: string): Promise<string>;
22
+ getTextWithLineBreaks(element: any, $: cheerio.CheerioAPI): string;
23
+ parseWebPage(url: string): Promise<string>;
24
+ protected getAllLinksFromContentSource(url: string, rootURL: string, regex: RegExp): Promise<string[]>;
25
+ protected getTopLevelLinks(url: string, rootURL: string): Promise<void>;
26
+ protected isHighestDomain(url: string): boolean;
27
+ protected getBasePath(url: string): string;
28
+ protected getPathName(url: string): string;
29
+ protected urlIsValid(url: string): Promise<boolean>;
30
+ protected getLowerLevelLinks(url: string, rootURL: string, crawlDepth: number, scrapedURLs: Set<string>, regex: RegExp): Promise<Set<string>>;
31
+ protected delay(ms: number): Promise<unknown>;
32
+ }
33
+ //# sourceMappingURL=AutotagWebsite.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"AutotagWebsite.d.ts","sourceRoot":"","sources":["../../../../src/Websites/generic/AutotagWebsite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAqB,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAEtE,OAAO,EAAE,QAAQ,EAAqB,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AACvF,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAMnC,qBACa,cAAe,SAAQ,WAAW;IAC3C,OAAO,CAAC,WAAW,CAAW;IAC9B,OAAO,CAAC,MAAM,CAAoB;IAClC,SAAS,CAAC,mBAAmB,EAAE,MAAM,CAAA;IACrC,SAAS,CAAC,+BAA+B,EAAE,OAAO,CAAC;IACnD,SAAS,CAAC,4BAA4B,EAAE,OAAO,CAAC;IAChD,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC;IAC7B,SAAS,CAAC,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;;IAQnC,SAAS,CAAC,cAAc,IAAI,QAAQ;IASvB,OAAO,CAAC,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAe7C,wBAAwB,CAAC,cAAc,EAAE,mBAAmB,EAAE,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;cA6D1F,6BAA6B,CAAC,gBAAgB,EAAE,MAAM,EAAE,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;IAsE3J,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAKpD,qBAAqB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,CAAC,UAAU,GAAG,MAAM;IAqB5D,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;cAoBvC,4BAA4B,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;cAoB5F,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAwC7E,SAAS,CAAC,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAW/C,SAAS,CAAC,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAW1C,SAAS,CAAC,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;cAa1B,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;cAmBzC,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;cA+CnI,KAAK,CAAC,EAAE,EAAE,MAAM;CAGnC"}
@@ -0,0 +1,324 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
19
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
20
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
21
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
22
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
23
+ };
24
+ var __importStar = (this && this.__importStar) || function (mod) {
25
+ if (mod && mod.__esModule) return mod;
26
+ var result = {};
27
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
28
+ __setModuleDefault(result, mod);
29
+ return result;
30
+ };
31
+ var __metadata = (this && this.__metadata) || function (k, v) {
32
+ if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
33
+ };
34
+ var __importDefault = (this && this.__importDefault) || function (mod) {
35
+ return (mod && mod.__esModule) ? mod : { "default": mod };
36
+ };
37
+ Object.defineProperty(exports, "__esModule", { value: true });
38
+ exports.AutotagWebsite = void 0;
39
+ const Core_1 = require("../../Core");
40
+ const Engine_1 = require("../../Engine");
41
+ const global_1 = require("@memberjunction/global");
42
+ const core_1 = require("@memberjunction/core");
43
+ const cheerio = __importStar(require("cheerio"));
44
+ const axios_1 = __importDefault(require("axios"));
45
+ const url_1 = require("url");
46
+ const dotenv_1 = __importDefault(require("dotenv"));
47
+ dotenv_1.default.config();
48
+ let AutotagWebsite = class AutotagWebsite extends Core_1.AutotagBase {
49
+ contextUser;
50
+ engine;
51
+ contentSourceTypeID;
52
+ CrawlOtherSitesInTopLevelDomain;
53
+ CrawlSitesInLowerLevelDomain;
54
+ MaxDepth;
55
+ RootURL;
56
+ URLPattern;
57
+ visitedURLs;
58
+ constructor() {
59
+ super();
60
+ this.engine = Engine_1.AutotagBaseEngine.Instance;
61
+ this.visitedURLs = new Set();
62
+ }
63
+ getContextUser() {
64
+ return this.contextUser;
65
+ }
66
+ async Autotag(contextUser) {
67
+ this.contextUser = contextUser;
68
+ this.contentSourceTypeID = await this.engine.setSubclassContentSourceType('Website', this.contextUser);
69
+ const contentSources = await this.engine.getAllContentSources(this.contextUser, this.contentSourceTypeID);
70
+ const contentItemsToProcess = await this.SetContentItemsToProcess(contentSources);
71
+ await this.engine.ExtractTextAndProcessWithLLM(contentItemsToProcess, this.contextUser);
72
+ }
73
+ async SetContentItemsToProcess(contentSources) {
74
+ const contentItemsToProcess = [];
75
+ for (const contentSource of contentSources) {
76
+ const contentSourceParamsMap = await this.engine.getContentSourceParams(contentSource, this.contextUser);
77
+ if (contentSourceParamsMap) {
78
+ contentSourceParamsMap.forEach((value, key) => {
79
+ if (key in this) {
80
+ this[key] = value;
81
+ }
82
+ });
83
+ }
84
+ const contentSourceParams = {
85
+ contentSourceID: contentSource.ID,
86
+ name: contentSource.Name,
87
+ ContentTypeID: contentSource.ContentTypeID,
88
+ ContentFileTypeID: contentSource.ContentFileTypeID,
89
+ ContentSourceTypeID: contentSource.ContentSourceTypeID,
90
+ URL: contentSource.URL
91
+ };
92
+ try {
93
+ const startURL = contentSourceParams.URL;
94
+ const rootURL = this.RootURL ? this.RootURL : this.getBasePath(startURL);
95
+ const regex = this.URLPattern && new RegExp(this.URLPattern) || new RegExp('.*');
96
+ const allContentItemLinks = await this.getAllLinksFromContentSource(startURL, rootURL, regex);
97
+ const contentItems = await this.SetNewAndModifiedContentItems(allContentItemLinks, contentSourceParams, this.contextUser);
98
+ if (contentItems && contentItems.length > 0) {
99
+ contentItemsToProcess.push(...contentItems);
100
+ }
101
+ else {
102
+ console.log(`No content items found to process for content source: ${contentSource.Get('Name')}`);
103
+ }
104
+ }
105
+ catch (e) {
106
+ console.error(`Failed to process content source: ${contentSource.Get('Name')}`);
107
+ }
108
+ }
109
+ return contentItemsToProcess;
110
+ }
111
+ async SetNewAndModifiedContentItems(contentItemLinks, contentSourceParams, contextUser) {
112
+ const addedContentItems = [];
113
+ for (const contentItemLink of contentItemLinks) {
114
+ try {
115
+ const newHash = await this.engine.getChecksumFromURL(contentItemLink);
116
+ const rv = new core_1.RunView();
117
+ const results = await rv.RunViews([
118
+ {
119
+ EntityName: 'Content Items',
120
+ ExtraFilter: `Checksum = '${newHash}'`,
121
+ ResultType: 'entity_object'
122
+ },
123
+ {
124
+ EntityName: 'Content Items',
125
+ ExtraFilter: `ContentSourceID = '${contentSourceParams.contentSourceID}' AND URL = '${contentItemLink}'`,
126
+ ResultType: 'entity_object'
127
+ }
128
+ ], this.contextUser);
129
+ const contentItemResultsWithChecksum = results[0];
130
+ const contentItemResultsWithURL = results[1];
131
+ if (contentItemResultsWithChecksum.Success && contentItemResultsWithChecksum.Results.length) {
132
+ continue;
133
+ }
134
+ else if (contentItemResultsWithURL.Success && contentItemResultsWithURL.Results.length) {
135
+ const contentItemResult = contentItemResultsWithURL.Results[0];
136
+ const lastStoredHash = contentItemResult.Checksum;
137
+ if (lastStoredHash !== newHash) {
138
+ const md = new core_1.Metadata();
139
+ const contentItem = await md.GetEntityObject('Content Items', this.contextUser);
140
+ contentItem.Load(contentItemResult.ID);
141
+ contentItem.Checksum = newHash;
142
+ contentItem.Text = await this.parseWebPage(contentItemLink);
143
+ await contentItem.Save();
144
+ addedContentItems.push(contentItem);
145
+ }
146
+ }
147
+ else {
148
+ const md = new core_1.Metadata();
149
+ const contentItem = await md.GetEntityObject('Content Items', this.contextUser);
150
+ contentItem.ContentSourceID = contentSourceParams.contentSourceID;
151
+ contentItem.Name = this.getPathName(contentItemLink);
152
+ contentItem.Description = await this.engine.getContentItemDescription(contentSourceParams, this.contextUser);
153
+ contentItem.ContentTypeID = contentSourceParams.ContentTypeID;
154
+ contentItem.ContentFileTypeID = contentSourceParams.ContentFileTypeID;
155
+ contentItem.ContentSourceTypeID = contentSourceParams.ContentSourceTypeID;
156
+ contentItem.Checksum = await this.engine.getChecksumFromURL(contentItemLink);
157
+ contentItem.URL = contentItemLink;
158
+ contentItem.Text = await this.parseWebPage(contentItemLink);
159
+ await contentItem.Save();
160
+ addedContentItems.push(contentItem);
161
+ }
162
+ }
163
+ catch (e) {
164
+ console.log(e);
165
+ }
166
+ }
167
+ return addedContentItems;
168
+ }
169
+ async fetchPageContent(url) {
170
+ const { data } = await axios_1.default.get(url);
171
+ return data;
172
+ }
173
+ getTextWithLineBreaks(element, $) {
174
+ let text = '';
175
+ const children = $(element).contents();
176
+ for (let i = 0; i < children.length; i++) {
177
+ const el = children[i];
178
+ if (el.type === 'text') {
179
+ text += $(el).text().trim() + ' ';
180
+ }
181
+ else if (el.type === 'tag') {
182
+ text += '\n' + this.getTextWithLineBreaks(el, $) + '\n';
183
+ }
184
+ }
185
+ return text;
186
+ }
187
+ async parseWebPage(url) {
188
+ try {
189
+ const pageContent = await this.fetchPageContent(url);
190
+ const $ = cheerio.load(pageContent);
191
+ const text = this.getTextWithLineBreaks($('body')[0], $);
192
+ return text;
193
+ }
194
+ catch (error) {
195
+ console.error(`Error processing ${url}:`, error);
196
+ return '';
197
+ }
198
+ }
199
+ async getAllLinksFromContentSource(url, rootURL, regex) {
200
+ try {
201
+ await this.getLowerLevelLinks(url, rootURL, this.MaxDepth, new Set(), regex);
202
+ await this.getTopLevelLinks(url, this.getBasePath(url));
203
+ return Array.from(this.visitedURLs);
204
+ }
205
+ catch (e) {
206
+ console.error(`Failed to get links from ${url}`);
207
+ return [];
208
+ }
209
+ }
210
+ async getTopLevelLinks(url, rootURL) {
211
+ if (!this.CrawlOtherSitesInTopLevelDomain) {
212
+ this.visitedURLs.add(url);
213
+ return;
214
+ }
215
+ if (this.visitedURLs.has(url) || !await this.urlIsValid(url) || this.isHighestDomain(url)) {
216
+ return;
217
+ }
218
+ this.visitedURLs.add(url);
219
+ try {
220
+ const { data } = await axios_1.default.get(url);
221
+ const $ = cheerio.load(data);
222
+ $('a').each((_, element) => {
223
+ const link = $(element).attr('href');
224
+ if (link) {
225
+ const newURL = new url_1.URL(link, url).href;
226
+ if (newURL.startsWith(rootURL) && !this.visitedURLs.has(newURL)) {
227
+ this.visitedURLs.add(newURL);
228
+ }
229
+ }
230
+ });
231
+ await this.delay(1000);
232
+ }
233
+ catch (e) {
234
+ console.error(`Failed to get links from ${url}`);
235
+ return;
236
+ }
237
+ }
238
+ isHighestDomain(url) {
239
+ try {
240
+ const parsedURL = new url_1.URL(url);
241
+ return parsedURL.pathname === '/' || parsedURL.pathname === '';
242
+ }
243
+ catch (e) {
244
+ console.error(`Invalid URL for same level parsing: ${url}`);
245
+ throw e;
246
+ }
247
+ }
248
+ getBasePath(url) {
249
+ const parsedURL = new url_1.URL(url);
250
+ const pathSegments = parsedURL.pathname.split('/').filter(segment => segment);
251
+ if (pathSegments.length > 0) {
252
+ pathSegments.pop();
253
+ }
254
+ const basePath = parsedURL.origin + '/' + pathSegments.join('/');
255
+ return basePath;
256
+ }
257
+ getPathName(url) {
258
+ try {
259
+ const parsedURL = new url_1.URL(url);
260
+ const pathSegments = parsedURL.pathname.split('/').filter(segment => segment);
261
+ const path = parsedURL.origin + '/' + pathSegments.join('/');
262
+ return path;
263
+ }
264
+ catch (e) {
265
+ console.error(`Invalid URL for same level parsing: ${url}`);
266
+ throw e;
267
+ }
268
+ }
269
+ async urlIsValid(url) {
270
+ try {
271
+ const response = await axios_1.default.head(url);
272
+ return response.status === 200;
273
+ }
274
+ catch (e) {
275
+ console.error(`Invalid URL: ${url}`);
276
+ return false;
277
+ }
278
+ }
279
+ async getLowerLevelLinks(url, rootURL, crawlDepth, scrapedURLs, regex) {
280
+ try {
281
+ console.log(`Scraping ${url}`);
282
+ if (scrapedURLs.has(url) || await this.urlIsValid(url) === false || crawlDepth < 0 || !this.CrawlSitesInLowerLevelDomain) {
283
+ return new Set();
284
+ }
285
+ let combinedLinks = new Set();
286
+ const extractedLinks = new Set();
287
+ const { data } = await axios_1.default.get(url);
288
+ const $ = cheerio.load(data);
289
+ $('a').each((_, element) => {
290
+ const link = $(element).attr('href');
291
+ if (link) {
292
+ const newURL = new url_1.URL(link, url).href;
293
+ if (newURL.startsWith(rootURL) && newURL !== url && !this.visitedURLs.has(newURL) && regex.test(newURL)) {
294
+ extractedLinks.add(newURL);
295
+ this.visitedURLs.add(newURL);
296
+ }
297
+ }
298
+ });
299
+ await this.delay(1000);
300
+ scrapedURLs.add(url);
301
+ if (crawlDepth === 0) {
302
+ return extractedLinks;
303
+ }
304
+ for (const subLink of extractedLinks) {
305
+ const lowerLevelLinks = await this.getLowerLevelLinks(subLink, rootURL, crawlDepth - 1, scrapedURLs, regex);
306
+ combinedLinks = new Set([...extractedLinks, ...lowerLevelLinks]);
307
+ }
308
+ return combinedLinks;
309
+ }
310
+ catch (e) {
311
+ console.error(`Failed to get links from ${url}`);
312
+ return new Set();
313
+ }
314
+ }
315
+ async delay(ms) {
316
+ return new Promise(resolve => setTimeout(resolve, ms));
317
+ }
318
+ };
319
+ exports.AutotagWebsite = AutotagWebsite;
320
+ exports.AutotagWebsite = AutotagWebsite = __decorate([
321
+ (0, global_1.RegisterClass)(Core_1.AutotagBase, 'AutotagWebsite'),
322
+ __metadata("design:paramtypes", [])
323
+ ], AutotagWebsite);
324
+ //# sourceMappingURL=AutotagWebsite.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"AutotagWebsite.js","sourceRoot":"","sources":["../../../../src/Websites/generic/AutotagWebsite.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,qCAAyC;AACzC,yCAAsE;AACtE,mDAAuD;AACvD,+CAAmE;AAEnE,iDAAmC;AACnC,kDAA0B;AAC1B,6BAA0B;AAC1B,oDAA4B;AAC5B,gBAAM,CAAC,MAAM,EAAE,CAAA;AAGR,IAAM,cAAc,GAApB,MAAM,cAAe,SAAQ,kBAAW;IACnC,WAAW,CAAW;IACtB,MAAM,CAAoB;IACxB,mBAAmB,CAAQ;IAC3B,+BAA+B,CAAU;IACzC,4BAA4B,CAAU;IACtC,QAAQ,CAAS;IACjB,OAAO,CAAS;IAChB,UAAU,CAAS;IACnB,WAAW,CAAc;IAEnC;QACI,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,0BAAiB,CAAC,QAAQ,CAAC;QACzC,IAAI,CAAC,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACzC,CAAC;IAES,cAAc;QACpB,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAOM,KAAK,CAAC,OAAO,CAAC,WAAqB;QACtC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,mBAAmB,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACvG,MAAM,cAAc,GAA0B,MAAM,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;QACjI,MAAM,qBAAqB,GAAwB,MAAM,IAAI,CAAC,wBAAwB,CAAC,cAAc,CAAC,CAAC;QACvG,MAAM,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,qBAAqB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAC5F,CAAC;IASM,KAAK,CAAC,wBAAwB,CAAC,cAAqC;QACvE,MAAM,qBAAqB,GAAwB,EAAE,CAAA;QAGrD,KAAK,MAAM,aAAa,IAAI,cAAc,EAAE,CAAC;YACzC,MAAM,sBAAsB,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,aAAa,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YACzG,IAAI,sBAAsB,EAAE,CAAC;gBAEzB,sBAAsB,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;oBAC1C,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;wBACb,IAAY,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;oBAC/B,CAAC;gBACL,CAAC,CAAC,CAAA;YACN,CAAC;YAED,MAAM,mBAAmB,GAAwB;gBAC7C,eAAe,EAAE,aAAa,CAAC,EAAE;gBACjC,IAAI,EAAE,aAAa,CAAC,IAAI;gBACxB,aAAa,EAAE,aAAa,CAAC,aAAa;gBAC1C,iBAAiB,EAAE,aAAa,CAAC,iBAAiB;gBAClD,mBAAmB,EAAE,aAAa,CAAC,mBAAmB;gBACtD,GAAG,EAAE,aAAa,CAAC,GAAG;aACzB,CAAA;YAED,IAAI,CAAC;gBAGD,MAAM,QAAQ,GAAW,mBAAmB,CAAC,GAAG,CAAC;gBAGjD,MAAM,OAAO,GAAW,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBAGjF,MAAM,KAAK,GAAW,IAAI,CAAC,UAAU,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC;gBAEzF,MAAM,mBAAmB,GAAa,MAAM,IAAI,CAAC,4BAA4B,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;gBACxG,MAAM,YAAY,GAAwB,MAAM,IAAI,CAAC,6BAA6B,CAAC,mBAAmB,EAAE,mBAAmB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;gBAC/I,IAAI,YAAY,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC1C,qBAAqB,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;gBAChD,CAAC;qBACI,CAAC;oBAEF,OAAO,CAAC,GAAG,CAAC,yDAAyD,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBACtG,CAAC;YACL,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,OAAO,CAAC,KAAK,CAAC,qCAAqC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACpF,CAAC;QACL,CAAC;QAED,OAAO,qBAAqB,CAAC;IACjC,CAAC;IAWS,KAAK,CAAC,6BAA6B,CAAC,gBAA0B,EAAE,mBAAwC,EAAE,WAAqB;QAErI,MAAM,iBAAiB,GAAwB,EAAE,CAAC;QAClD,KAAK,MAAM,eAAe,IAAI,gBAAgB,EAAE,CAAC;YAC7C,IAAI,CAAC;gBACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;gBAEtE,MAAM,EAAE,GAAG,IAAI,cAAO,EAAE,CAAC;gBACzB,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAoB;oBACjD;wBACI,UAAU,EAAE,eAAe;wBAC3B,WAAW,EAAE,eAAe,OAAO,GAAG;wBACtC,UAAU,EAAE,eAAe;qBAC9B;oBACD;wBACI,UAAU,EAAE,eAAe;wBAC3B,WAAW,EAAE,sBAAsB,mBAAmB,CAAC,eAAe,gBAAgB,eAAe,GAAG;wBACxG,UAAU,EAAE,eAAe;qBAC9B;iBACJ,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;gBAEpB,MAAM,8BAA8B,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;gBACjD,MAAM,yBAAyB,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;gBAE5C,IAAI,8BAA8B,CAAC,OAAO,IAAI,8BAA8B,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;oBAE1F,SAAS;gBACb,CAAC;qBAEI,IAAI,yBAAyB,CAAC,OAAO,IAAI,yBAAyB,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;oBAErF,MAAM,iBAAiB,GAAsB,yBAAyB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;oBAClF,MAAM,cAAc,GAAW,iBAAiB,CAAC,QAAQ,CAAA;oBAEzD,IAAI,cAAc,KAAK,OAAO,EAAE,CAAC;wBAE7B,MAAM,EAAE,GAAG,IAAI,eAAQ,EAAE,CAAC;wBAC1B,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,eAAe,CAAoB,eAAe,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;wBACnG,WAAW,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;wBACvC,WAAW,CAAC,QAAQ,GAAG,OAAO,CAAA;wBAC9B,WAAW,CAAC,IAAI,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,eAAe,CAAC,CAAA;wBAE3D,MAAM,WAAW,CAAC,IAAI,EAAE,CAAC;wBACzB,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;oBACxC,CAAC;gBACL,CAAC;qBACI,CAAC;oBAEF,MAAM,EAAE,GAAG,IAAI,eAAQ,EAAE,CAAC;oBAC1B,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,eAAe,CAAoB,eAAe,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;oBACnG,WAAW,CAAC,eAAe,GAAG,mBAAmB,CAAC,eAAe,CAAA;oBACjE,WAAW,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,CAAA;oBACpD,WAAW,CAAC,WAAW,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,yBAAyB,CAAC,mBAAmB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;oBAC5G,WAAW,CAAC,aAAa,GAAG,mBAAmB,CAAC,aAAa,CAAA;oBAC7D,WAAW,CAAC,iBAAiB,GAAG,mBAAmB,CAAC,iBAAiB,CAAA;oBACrE,WAAW,CAAC,mBAAmB,GAAG,mBAAmB,CAAC,mBAAmB,CAAA;oBACzE,WAAW,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAA;oBAC5E,WAAW,CAAC,GAAG,GAAG,eAAe,CAAA;oBACjC,WAAW,CAAC,IAAI,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,eAAe,CAAC,CAAA;oBAE3D,MAAM,WAAW,CAAC,IAAI,EAAE,CAAC;oBACzB,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBACxC,CAAC;YACD,CAAC;YAAA,OAAO,CAAC,EAAE,CAAC;gBACR,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YAClB,CAAC;QACT,CAAC;QACD,OAAO,iBAAiB,CAAC;IAC7B,CAAC;IAEM,KAAK,CAAC,gBAAgB,CAAC,GAAW;QACrC,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,eAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACtC,OAAO,IAAI,CAAC;IAChB,CAAC;IAEM,qBAAqB,CAAC,OAAY,EAAE,CAAqB;QAC5D,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;QAEvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;YACvB,IAAI,EAAE,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACrB,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC;YACtC,CAAC;iBAAM,IAAI,EAAE,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;gBAC3B,IAAI,IAAI,IAAI,GAAG,IAAI,CAAC,qBAAqB,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC;YAC5D,CAAC;QACL,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAOM,KAAK,CAAC,YAAY,CAAC,GAAW;QACjC,IAAI,CAAC;YACD,MAAM,WAAW,GAAW,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACpC,MAAM,IAAI,GAAW,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACjE,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,oBAAoB,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;YACjD,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IASS,KAAK,CAAC,4BAA4B,CAAC,GAAW,EAAE,OAAe,EAAE,KAAa;QAEpF,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,GAAG,EAAU,EAAE,KAAK,CAAC,CAAC;YACrF,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC;YAExD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACxC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,OAAO,CAAC,KAAK,CAAC,4BAA4B,GAAG,EAAE,CAAC,CAAC;YACjD,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IASS,KAAK,CAAC,gBAAgB,CAAC,GAAW,EAAE,OAAe;QACzD,IAAI,CAAC,IAAI,CAAC,+BAA+B,EAAE,CAAC;YACxC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC1B,OAAM;QACV,CAAC;QAGD,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC;YACxF,OAAM;QACV,CAAC;QAED,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAE1B,IAAI,CAAC;YACD,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,eAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAG7B,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;gBACvB,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrC,IAAI,IAAI,EAAE,CAAC;oBACP,MAAM,MAAM,GAAG,IAAI,SAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC;oBACvC,IAAI,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;wBAC9D,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBACjC,CAAC;gBACL,CAAC;YACL,CAAC,CAAC,CAAC;YACH,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC3B,CAAC;QACD,OAAO,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,KAAK,CAAC,4BAA4B,GAAG,EAAE,CAAC,CAAC;YACjD,OAAM;QACV,CAAC;IACL,CAAC;IAOS,eAAe,CAAC,GAAW;QACjC,IAAI,CAAC;YACD,MAAM,SAAS,GAAQ,IAAI,SAAG,CAAC,GAAG,CAAC,CAAC;YACpC,OAAO,SAAS,CAAC,QAAQ,KAAK,GAAG,IAAI,SAAS,CAAC,QAAQ,KAAK,EAAE,CAAC;QACnE,CAAC;QACD,OAAO,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,KAAK,CAAC,uCAAuC,GAAG,EAAE,CAAC,CAAC;YAC5D,MAAM,CAAC,CAAC;QACZ,CAAC;IACL,CAAC;IAES,WAAW,CAAC,GAAW;QAC7B,MAAM,SAAS,GAAQ,IAAI,SAAG,CAAC,GAAG,CAAC,CAAC;QACpC,MAAM,YAAY,GAAa,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;QACxF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,YAAY,CAAC,GAAG,EAAE,CAAC;QACvB,CAAC;QACD,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,GAAG,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjE,OAAO,QAAQ,CAAC;IACpB,CAAC;IAGS,WAAW,CAAC,GAAW;QAC7B,IAAI,CAAC;YACD,MAAM,SAAS,GAAQ,IAAI,SAAG,CAAC,GAAG,CAAC,CAAC;YACpC,MAAM,YAAY,GAAa,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;YACxF,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,GAAG,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC7D,OAAO,IAAI,CAAA;QACf,CAAC;QACD,OAAO,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,KAAK,CAAC,uCAAuC,GAAG,EAAE,CAAC,CAAC;YAC5D,MAAM,CAAC,CAAC;QACZ,CAAC;IACL,CAAC;IAES,KAAK,CAAC,UAAU,CAAC,GAAW;QAClC,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,eAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvC,OAAO,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC;QACnC,CAAC;QACD,OAAO,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,KAAK,CAAC,gBAAgB,GAAG,EAAE,CAAC,CAAC;YACrC,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;IAUS,KAAK,CAAC,kBAAkB,CAAC,GAAW,EAAE,OAAe,EAAE,UAAkB,EAAE,WAAwB,EAAE,KAAa;QAExH,IAAI,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,YAAY,GAAG,EAAE,CAAC,CAAC;YAE/B,IAAI,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,KAAK,IAAI,UAAU,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,4BAA4B,EAAE,CAAC;gBACvH,OAAO,IAAI,GAAG,EAAU,CAAC;YAC7B,CAAC;YAED,IAAI,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;YACtC,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;YAEzC,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,eAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAG7B,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;gBACvB,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrC,IAAI,IAAI,EAAE,CAAC;oBACP,MAAM,MAAM,GAAG,IAAI,SAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC;oBACvC,IAAI,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;wBACtG,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;wBAC3B,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBACjC,CAAC;gBACL,CAAC;YACL,CAAC,CAAC,CAAC;YACH,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACvB,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAGrB,IAAI,UAAU,KAAK,CAAC,EAAE,CAAC;gBACnB,OAAO,cAAc,CAAC;YAC1B,CAAC;YAED,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;gBAEnC,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,GAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;gBAC1G,aAAa,GAAG,IAAI,GAAG,CAAS,CAAC,GAAG,cAAc,EAAE,GAAG,eAAe,CAAC,CAAC,CAAC;YAC7E,CAAC;YACD,OAAO,aAAa,CAAC;QACzB,CAAC;QACD,OAAO,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,KAAK,CAAC,4BAA4B,GAAG,EAAE,CAAC,CAAC;YACjD,OAAO,IAAI,GAAG,EAAU,CAAC;QAC7B,CAAC;IACL,CAAC;IAES,KAAK,CAAC,KAAK,CAAC,EAAU;QAC5B,OAAO,IAAI,OAAO,CAAE,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAE,CAAC;IAC7D,CAAC;CACJ,CAAA;AA9XY,wCAAc;yBAAd,cAAc;IAD1B,IAAA,sBAAa,EAAC,kBAAW,EAAE,gBAAgB,CAAC;;GAChC,cAAc,CA8X1B"}
@@ -0,0 +1,2 @@
1
+ export * from './generic/AutotagWebsite';
2
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/Websites/index.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
@@ -0,0 +1,18 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
+ for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
+ };
16
+ Object.defineProperty(exports, "__esModule", { value: true });
17
+ __exportStar(require("./generic/AutotagWebsite"), exports);
18
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/Websites/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,2DAAwC"}
@@ -0,0 +1,7 @@
1
+ export * from './Core';
2
+ export * from './Engine';
3
+ export * from './LocalFileSystem';
4
+ export * from './RSSFeed';
5
+ export * from './Websites';
6
+ export * from './CloudStorage';
7
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,QAAQ,CAAC;AACvB,cAAc,UAAU,CAAC;AACzB,cAAc,mBAAmB,CAAC;AAClC,cAAc,WAAW,CAAC;AAC1B,cAAc,YAAY,CAAC;AAC3B,cAAc,gBAAgB,CAAA"}
@@ -0,0 +1,23 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
+ for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
+ };
16
+ Object.defineProperty(exports, "__esModule", { value: true });
17
+ __exportStar(require("./Core"), exports);
18
+ __exportStar(require("./Engine"), exports);
19
+ __exportStar(require("./LocalFileSystem"), exports);
20
+ __exportStar(require("./RSSFeed"), exports);
21
+ __exportStar(require("./Websites"), exports);
22
+ __exportStar(require("./CloudStorage"), exports);
23
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,yCAAuB;AACvB,2CAAyB;AACzB,oDAAkC;AAClC,4CAA0B;AAC1B,6CAA2B;AAC3B,iDAA8B"}