@memberjunction/content-autotagging 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/CloudStorage/generic/CloudStorageBase.d.ts +16 -0
- package/dist/src/CloudStorage/generic/CloudStorageBase.d.ts.map +1 -0
- package/dist/src/CloudStorage/generic/CloudStorageBase.js +48 -0
- package/dist/src/CloudStorage/generic/CloudStorageBase.js.map +1 -0
- package/dist/src/CloudStorage/index.d.ts +3 -0
- package/dist/src/CloudStorage/index.d.ts.map +1 -0
- package/dist/src/CloudStorage/index.js +19 -0
- package/dist/src/CloudStorage/index.js.map +1 -0
- package/dist/src/CloudStorage/providers/AutotagAzureBlob.d.ts +18 -0
- package/dist/src/CloudStorage/providers/AutotagAzureBlob.d.ts.map +1 -0
- package/dist/src/CloudStorage/providers/AutotagAzureBlob.js +91 -0
- package/dist/src/CloudStorage/providers/AutotagAzureBlob.js.map +1 -0
- package/dist/src/Core/generic/AutotagBase.d.ts +7 -0
- package/dist/src/Core/generic/AutotagBase.d.ts.map +1 -0
- package/dist/src/Core/generic/AutotagBase.js +7 -0
- package/dist/src/Core/generic/AutotagBase.js.map +1 -0
- package/dist/src/Core/index.d.ts +2 -0
- package/dist/src/Core/index.d.ts.map +1 -0
- package/dist/src/Core/index.js +18 -0
- package/dist/src/Core/index.js.map +1 -0
- package/dist/src/Engine/generic/AutotagBaseEngine.d.ts +51 -0
- package/dist/src/Engine/generic/AutotagBaseEngine.d.ts.map +1 -0
- package/dist/src/Engine/generic/AutotagBaseEngine.js +553 -0
- package/dist/src/Engine/generic/AutotagBaseEngine.js.map +1 -0
- package/dist/src/Engine/generic/content.types.d.ts +32 -0
- package/dist/src/Engine/generic/content.types.d.ts.map +1 -0
- package/dist/src/Engine/generic/content.types.js +38 -0
- package/dist/src/Engine/generic/content.types.js.map +1 -0
- package/dist/src/Engine/generic/process.types.d.ts +30 -0
- package/dist/src/Engine/generic/process.types.d.ts.map +1 -0
- package/dist/src/Engine/generic/process.types.js +33 -0
- package/dist/src/Engine/generic/process.types.js.map +1 -0
- package/dist/src/Engine/index.d.ts +4 -0
- package/dist/src/Engine/index.d.ts.map +1 -0
- package/dist/src/Engine/index.js +20 -0
- package/dist/src/Engine/index.js.map +1 -0
- package/dist/src/LocalFileSystem/generic/AutotagLocalFileSystem.d.ts +20 -0
- package/dist/src/LocalFileSystem/generic/AutotagLocalFileSystem.d.ts.map +1 -0
- package/dist/src/LocalFileSystem/generic/AutotagLocalFileSystem.js +155 -0
- package/dist/src/LocalFileSystem/generic/AutotagLocalFileSystem.js.map +1 -0
- package/dist/src/LocalFileSystem/index.d.ts +2 -0
- package/dist/src/LocalFileSystem/index.d.ts.map +1 -0
- package/dist/src/LocalFileSystem/index.js +18 -0
- package/dist/src/LocalFileSystem/index.js.map +1 -0
- package/dist/src/RSSFeed/generic/AutotagRSSFeed.d.ts +19 -0
- package/dist/src/RSSFeed/generic/AutotagRSSFeed.d.ts.map +1 -0
- package/dist/src/RSSFeed/generic/AutotagRSSFeed.js +168 -0
- package/dist/src/RSSFeed/generic/AutotagRSSFeed.js.map +1 -0
- package/dist/src/RSSFeed/generic/RSS.types.d.ts +13 -0
- package/dist/src/RSSFeed/generic/RSS.types.d.ts.map +1 -0
- package/dist/src/RSSFeed/generic/RSS.types.js +17 -0
- package/dist/src/RSSFeed/generic/RSS.types.js.map +1 -0
- package/dist/src/RSSFeed/index.d.ts +3 -0
- package/dist/src/RSSFeed/index.d.ts.map +1 -0
- package/dist/src/RSSFeed/index.js +19 -0
- package/dist/src/RSSFeed/index.js.map +1 -0
- package/dist/src/Websites/generic/AutotagWebsite.d.ts +33 -0
- package/dist/src/Websites/generic/AutotagWebsite.d.ts.map +1 -0
- package/dist/src/Websites/generic/AutotagWebsite.js +324 -0
- package/dist/src/Websites/generic/AutotagWebsite.js.map +1 -0
- package/dist/src/Websites/index.d.ts +2 -0
- package/dist/src/Websites/index.d.ts.map +1 -0
- package/dist/src/Websites/index.js +18 -0
- package/dist/src/Websites/index.js.map +1 -0
- package/dist/src/index.d.ts +7 -0
- package/dist/src/index.d.ts.map +1 -0
- package/dist/src/index.js +23 -0
- package/dist/src/index.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -0
- package/package.json +45 -0
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { AutotagBase } from '../../Core';
|
|
2
|
+
import { ContentSourceParams } from '../../Engine';
|
|
3
|
+
import { UserInfo } from '@memberjunction/core';
|
|
4
|
+
import { ContentSourceEntity, ContentItemEntity } from '@memberjunction/core-entities';
|
|
5
|
+
import * as cheerio from 'cheerio';
|
|
6
|
+
export declare class AutotagWebsite extends AutotagBase {
|
|
7
|
+
private contextUser;
|
|
8
|
+
private engine;
|
|
9
|
+
protected contentSourceTypeID: string;
|
|
10
|
+
protected CrawlOtherSitesInTopLevelDomain: boolean;
|
|
11
|
+
protected CrawlSitesInLowerLevelDomain: boolean;
|
|
12
|
+
protected MaxDepth: number;
|
|
13
|
+
protected RootURL: string;
|
|
14
|
+
protected URLPattern: string;
|
|
15
|
+
protected visitedURLs: Set<string>;
|
|
16
|
+
constructor();
|
|
17
|
+
protected getContextUser(): UserInfo;
|
|
18
|
+
Autotag(contextUser: UserInfo): Promise<void>;
|
|
19
|
+
SetContentItemsToProcess(contentSources: ContentSourceEntity[]): Promise<ContentItemEntity[]>;
|
|
20
|
+
protected SetNewAndModifiedContentItems(contentItemLinks: string[], contentSourceParams: ContentSourceParams, contextUser: UserInfo): Promise<ContentItemEntity[]>;
|
|
21
|
+
fetchPageContent(url: string): Promise<string>;
|
|
22
|
+
getTextWithLineBreaks(element: any, $: cheerio.CheerioAPI): string;
|
|
23
|
+
parseWebPage(url: string): Promise<string>;
|
|
24
|
+
protected getAllLinksFromContentSource(url: string, rootURL: string, regex: RegExp): Promise<string[]>;
|
|
25
|
+
protected getTopLevelLinks(url: string, rootURL: string): Promise<void>;
|
|
26
|
+
protected isHighestDomain(url: string): boolean;
|
|
27
|
+
protected getBasePath(url: string): string;
|
|
28
|
+
protected getPathName(url: string): string;
|
|
29
|
+
protected urlIsValid(url: string): Promise<boolean>;
|
|
30
|
+
protected getLowerLevelLinks(url: string, rootURL: string, crawlDepth: number, scrapedURLs: Set<string>, regex: RegExp): Promise<Set<string>>;
|
|
31
|
+
protected delay(ms: number): Promise<unknown>;
|
|
32
|
+
}
|
|
33
|
+
//# sourceMappingURL=AutotagWebsite.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AutotagWebsite.d.ts","sourceRoot":"","sources":["../../../../src/Websites/generic/AutotagWebsite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAqB,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAEtE,OAAO,EAAE,QAAQ,EAAqB,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AACvF,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAMnC,qBACa,cAAe,SAAQ,WAAW;IAC3C,OAAO,CAAC,WAAW,CAAW;IAC9B,OAAO,CAAC,MAAM,CAAoB;IAClC,SAAS,CAAC,mBAAmB,EAAE,MAAM,CAAA;IACrC,SAAS,CAAC,+BAA+B,EAAE,OAAO,CAAC;IACnD,SAAS,CAAC,4BAA4B,EAAE,OAAO,CAAC;IAChD,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC;IAC7B,SAAS,CAAC,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;;IAQnC,SAAS,CAAC,cAAc,IAAI,QAAQ;IASvB,OAAO,CAAC,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAe7C,wBAAwB,CAAC,cAAc,EAAE,mBAAmB,EAAE,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;cA6D1F,6BAA6B,CAAC,gBAAgB,EAAE,MAAM,EAAE,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;IAsE3J,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAKpD,qBAAqB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,CAAC,UAAU,GAAG,MAAM;IAqB5D,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;cAoBvC,4BAA4B,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;cAoB5F,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAwC7E,SAAS,CAAC,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAW/C,SAAS,CAAC,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAW1C,SAAS,CAAC,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;cAa1B,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;cAmBzC,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;cA+CnI,KAAK,CAAC,EAAE,EAAE,MAAM;CAGnC"}
|
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
19
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
20
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
21
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
22
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
23
|
+
};
|
|
24
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
25
|
+
if (mod && mod.__esModule) return mod;
|
|
26
|
+
var result = {};
|
|
27
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
28
|
+
__setModuleDefault(result, mod);
|
|
29
|
+
return result;
|
|
30
|
+
};
|
|
31
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
32
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
33
|
+
};
|
|
34
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
35
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
36
|
+
};
|
|
37
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
38
|
+
exports.AutotagWebsite = void 0;
|
|
39
|
+
const Core_1 = require("../../Core");
|
|
40
|
+
const Engine_1 = require("../../Engine");
|
|
41
|
+
const global_1 = require("@memberjunction/global");
|
|
42
|
+
const core_1 = require("@memberjunction/core");
|
|
43
|
+
const cheerio = __importStar(require("cheerio"));
|
|
44
|
+
const axios_1 = __importDefault(require("axios"));
|
|
45
|
+
const url_1 = require("url");
|
|
46
|
+
const dotenv_1 = __importDefault(require("dotenv"));
|
|
47
|
+
dotenv_1.default.config();
|
|
48
|
+
let AutotagWebsite = class AutotagWebsite extends Core_1.AutotagBase {
|
|
49
|
+
contextUser;
|
|
50
|
+
engine;
|
|
51
|
+
contentSourceTypeID;
|
|
52
|
+
CrawlOtherSitesInTopLevelDomain;
|
|
53
|
+
CrawlSitesInLowerLevelDomain;
|
|
54
|
+
MaxDepth;
|
|
55
|
+
RootURL;
|
|
56
|
+
URLPattern;
|
|
57
|
+
visitedURLs;
|
|
58
|
+
constructor() {
|
|
59
|
+
super();
|
|
60
|
+
this.engine = Engine_1.AutotagBaseEngine.Instance;
|
|
61
|
+
this.visitedURLs = new Set();
|
|
62
|
+
}
|
|
63
|
+
getContextUser() {
|
|
64
|
+
return this.contextUser;
|
|
65
|
+
}
|
|
66
|
+
async Autotag(contextUser) {
|
|
67
|
+
this.contextUser = contextUser;
|
|
68
|
+
this.contentSourceTypeID = await this.engine.setSubclassContentSourceType('Website', this.contextUser);
|
|
69
|
+
const contentSources = await this.engine.getAllContentSources(this.contextUser, this.contentSourceTypeID);
|
|
70
|
+
const contentItemsToProcess = await this.SetContentItemsToProcess(contentSources);
|
|
71
|
+
await this.engine.ExtractTextAndProcessWithLLM(contentItemsToProcess, this.contextUser);
|
|
72
|
+
}
|
|
73
|
+
async SetContentItemsToProcess(contentSources) {
|
|
74
|
+
const contentItemsToProcess = [];
|
|
75
|
+
for (const contentSource of contentSources) {
|
|
76
|
+
const contentSourceParamsMap = await this.engine.getContentSourceParams(contentSource, this.contextUser);
|
|
77
|
+
if (contentSourceParamsMap) {
|
|
78
|
+
contentSourceParamsMap.forEach((value, key) => {
|
|
79
|
+
if (key in this) {
|
|
80
|
+
this[key] = value;
|
|
81
|
+
}
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
const contentSourceParams = {
|
|
85
|
+
contentSourceID: contentSource.ID,
|
|
86
|
+
name: contentSource.Name,
|
|
87
|
+
ContentTypeID: contentSource.ContentTypeID,
|
|
88
|
+
ContentFileTypeID: contentSource.ContentFileTypeID,
|
|
89
|
+
ContentSourceTypeID: contentSource.ContentSourceTypeID,
|
|
90
|
+
URL: contentSource.URL
|
|
91
|
+
};
|
|
92
|
+
try {
|
|
93
|
+
const startURL = contentSourceParams.URL;
|
|
94
|
+
const rootURL = this.RootURL ? this.RootURL : this.getBasePath(startURL);
|
|
95
|
+
const regex = this.URLPattern && new RegExp(this.URLPattern) || new RegExp('.*');
|
|
96
|
+
const allContentItemLinks = await this.getAllLinksFromContentSource(startURL, rootURL, regex);
|
|
97
|
+
const contentItems = await this.SetNewAndModifiedContentItems(allContentItemLinks, contentSourceParams, this.contextUser);
|
|
98
|
+
if (contentItems && contentItems.length > 0) {
|
|
99
|
+
contentItemsToProcess.push(...contentItems);
|
|
100
|
+
}
|
|
101
|
+
else {
|
|
102
|
+
console.log(`No content items found to process for content source: ${contentSource.Get('Name')}`);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
catch (e) {
|
|
106
|
+
console.error(`Failed to process content source: ${contentSource.Get('Name')}`);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
return contentItemsToProcess;
|
|
110
|
+
}
|
|
111
|
+
async SetNewAndModifiedContentItems(contentItemLinks, contentSourceParams, contextUser) {
|
|
112
|
+
const addedContentItems = [];
|
|
113
|
+
for (const contentItemLink of contentItemLinks) {
|
|
114
|
+
try {
|
|
115
|
+
const newHash = await this.engine.getChecksumFromURL(contentItemLink);
|
|
116
|
+
const rv = new core_1.RunView();
|
|
117
|
+
const results = await rv.RunViews([
|
|
118
|
+
{
|
|
119
|
+
EntityName: 'Content Items',
|
|
120
|
+
ExtraFilter: `Checksum = '${newHash}'`,
|
|
121
|
+
ResultType: 'entity_object'
|
|
122
|
+
},
|
|
123
|
+
{
|
|
124
|
+
EntityName: 'Content Items',
|
|
125
|
+
ExtraFilter: `ContentSourceID = '${contentSourceParams.contentSourceID}' AND URL = '${contentItemLink}'`,
|
|
126
|
+
ResultType: 'entity_object'
|
|
127
|
+
}
|
|
128
|
+
], this.contextUser);
|
|
129
|
+
const contentItemResultsWithChecksum = results[0];
|
|
130
|
+
const contentItemResultsWithURL = results[1];
|
|
131
|
+
if (contentItemResultsWithChecksum.Success && contentItemResultsWithChecksum.Results.length) {
|
|
132
|
+
continue;
|
|
133
|
+
}
|
|
134
|
+
else if (contentItemResultsWithURL.Success && contentItemResultsWithURL.Results.length) {
|
|
135
|
+
const contentItemResult = contentItemResultsWithURL.Results[0];
|
|
136
|
+
const lastStoredHash = contentItemResult.Checksum;
|
|
137
|
+
if (lastStoredHash !== newHash) {
|
|
138
|
+
const md = new core_1.Metadata();
|
|
139
|
+
const contentItem = await md.GetEntityObject('Content Items', this.contextUser);
|
|
140
|
+
contentItem.Load(contentItemResult.ID);
|
|
141
|
+
contentItem.Checksum = newHash;
|
|
142
|
+
contentItem.Text = await this.parseWebPage(contentItemLink);
|
|
143
|
+
await contentItem.Save();
|
|
144
|
+
addedContentItems.push(contentItem);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
else {
|
|
148
|
+
const md = new core_1.Metadata();
|
|
149
|
+
const contentItem = await md.GetEntityObject('Content Items', this.contextUser);
|
|
150
|
+
contentItem.ContentSourceID = contentSourceParams.contentSourceID;
|
|
151
|
+
contentItem.Name = this.getPathName(contentItemLink);
|
|
152
|
+
contentItem.Description = await this.engine.getContentItemDescription(contentSourceParams, this.contextUser);
|
|
153
|
+
contentItem.ContentTypeID = contentSourceParams.ContentTypeID;
|
|
154
|
+
contentItem.ContentFileTypeID = contentSourceParams.ContentFileTypeID;
|
|
155
|
+
contentItem.ContentSourceTypeID = contentSourceParams.ContentSourceTypeID;
|
|
156
|
+
contentItem.Checksum = await this.engine.getChecksumFromURL(contentItemLink);
|
|
157
|
+
contentItem.URL = contentItemLink;
|
|
158
|
+
contentItem.Text = await this.parseWebPage(contentItemLink);
|
|
159
|
+
await contentItem.Save();
|
|
160
|
+
addedContentItems.push(contentItem);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
catch (e) {
|
|
164
|
+
console.log(e);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
return addedContentItems;
|
|
168
|
+
}
|
|
169
|
+
async fetchPageContent(url) {
|
|
170
|
+
const { data } = await axios_1.default.get(url);
|
|
171
|
+
return data;
|
|
172
|
+
}
|
|
173
|
+
getTextWithLineBreaks(element, $) {
|
|
174
|
+
let text = '';
|
|
175
|
+
const children = $(element).contents();
|
|
176
|
+
for (let i = 0; i < children.length; i++) {
|
|
177
|
+
const el = children[i];
|
|
178
|
+
if (el.type === 'text') {
|
|
179
|
+
text += $(el).text().trim() + ' ';
|
|
180
|
+
}
|
|
181
|
+
else if (el.type === 'tag') {
|
|
182
|
+
text += '\n' + this.getTextWithLineBreaks(el, $) + '\n';
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
return text;
|
|
186
|
+
}
|
|
187
|
+
async parseWebPage(url) {
|
|
188
|
+
try {
|
|
189
|
+
const pageContent = await this.fetchPageContent(url);
|
|
190
|
+
const $ = cheerio.load(pageContent);
|
|
191
|
+
const text = this.getTextWithLineBreaks($('body')[0], $);
|
|
192
|
+
return text;
|
|
193
|
+
}
|
|
194
|
+
catch (error) {
|
|
195
|
+
console.error(`Error processing ${url}:`, error);
|
|
196
|
+
return '';
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
async getAllLinksFromContentSource(url, rootURL, regex) {
|
|
200
|
+
try {
|
|
201
|
+
await this.getLowerLevelLinks(url, rootURL, this.MaxDepth, new Set(), regex);
|
|
202
|
+
await this.getTopLevelLinks(url, this.getBasePath(url));
|
|
203
|
+
return Array.from(this.visitedURLs);
|
|
204
|
+
}
|
|
205
|
+
catch (e) {
|
|
206
|
+
console.error(`Failed to get links from ${url}`);
|
|
207
|
+
return [];
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
async getTopLevelLinks(url, rootURL) {
|
|
211
|
+
if (!this.CrawlOtherSitesInTopLevelDomain) {
|
|
212
|
+
this.visitedURLs.add(url);
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
if (this.visitedURLs.has(url) || !await this.urlIsValid(url) || this.isHighestDomain(url)) {
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
this.visitedURLs.add(url);
|
|
219
|
+
try {
|
|
220
|
+
const { data } = await axios_1.default.get(url);
|
|
221
|
+
const $ = cheerio.load(data);
|
|
222
|
+
$('a').each((_, element) => {
|
|
223
|
+
const link = $(element).attr('href');
|
|
224
|
+
if (link) {
|
|
225
|
+
const newURL = new url_1.URL(link, url).href;
|
|
226
|
+
if (newURL.startsWith(rootURL) && !this.visitedURLs.has(newURL)) {
|
|
227
|
+
this.visitedURLs.add(newURL);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
});
|
|
231
|
+
await this.delay(1000);
|
|
232
|
+
}
|
|
233
|
+
catch (e) {
|
|
234
|
+
console.error(`Failed to get links from ${url}`);
|
|
235
|
+
return;
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
isHighestDomain(url) {
|
|
239
|
+
try {
|
|
240
|
+
const parsedURL = new url_1.URL(url);
|
|
241
|
+
return parsedURL.pathname === '/' || parsedURL.pathname === '';
|
|
242
|
+
}
|
|
243
|
+
catch (e) {
|
|
244
|
+
console.error(`Invalid URL for same level parsing: ${url}`);
|
|
245
|
+
throw e;
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
getBasePath(url) {
|
|
249
|
+
const parsedURL = new url_1.URL(url);
|
|
250
|
+
const pathSegments = parsedURL.pathname.split('/').filter(segment => segment);
|
|
251
|
+
if (pathSegments.length > 0) {
|
|
252
|
+
pathSegments.pop();
|
|
253
|
+
}
|
|
254
|
+
const basePath = parsedURL.origin + '/' + pathSegments.join('/');
|
|
255
|
+
return basePath;
|
|
256
|
+
}
|
|
257
|
+
getPathName(url) {
|
|
258
|
+
try {
|
|
259
|
+
const parsedURL = new url_1.URL(url);
|
|
260
|
+
const pathSegments = parsedURL.pathname.split('/').filter(segment => segment);
|
|
261
|
+
const path = parsedURL.origin + '/' + pathSegments.join('/');
|
|
262
|
+
return path;
|
|
263
|
+
}
|
|
264
|
+
catch (e) {
|
|
265
|
+
console.error(`Invalid URL for same level parsing: ${url}`);
|
|
266
|
+
throw e;
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
async urlIsValid(url) {
|
|
270
|
+
try {
|
|
271
|
+
const response = await axios_1.default.head(url);
|
|
272
|
+
return response.status === 200;
|
|
273
|
+
}
|
|
274
|
+
catch (e) {
|
|
275
|
+
console.error(`Invalid URL: ${url}`);
|
|
276
|
+
return false;
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
async getLowerLevelLinks(url, rootURL, crawlDepth, scrapedURLs, regex) {
|
|
280
|
+
try {
|
|
281
|
+
console.log(`Scraping ${url}`);
|
|
282
|
+
if (scrapedURLs.has(url) || await this.urlIsValid(url) === false || crawlDepth < 0 || !this.CrawlSitesInLowerLevelDomain) {
|
|
283
|
+
return new Set();
|
|
284
|
+
}
|
|
285
|
+
let combinedLinks = new Set();
|
|
286
|
+
const extractedLinks = new Set();
|
|
287
|
+
const { data } = await axios_1.default.get(url);
|
|
288
|
+
const $ = cheerio.load(data);
|
|
289
|
+
$('a').each((_, element) => {
|
|
290
|
+
const link = $(element).attr('href');
|
|
291
|
+
if (link) {
|
|
292
|
+
const newURL = new url_1.URL(link, url).href;
|
|
293
|
+
if (newURL.startsWith(rootURL) && newURL !== url && !this.visitedURLs.has(newURL) && regex.test(newURL)) {
|
|
294
|
+
extractedLinks.add(newURL);
|
|
295
|
+
this.visitedURLs.add(newURL);
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
});
|
|
299
|
+
await this.delay(1000);
|
|
300
|
+
scrapedURLs.add(url);
|
|
301
|
+
if (crawlDepth === 0) {
|
|
302
|
+
return extractedLinks;
|
|
303
|
+
}
|
|
304
|
+
for (const subLink of extractedLinks) {
|
|
305
|
+
const lowerLevelLinks = await this.getLowerLevelLinks(subLink, rootURL, crawlDepth - 1, scrapedURLs, regex);
|
|
306
|
+
combinedLinks = new Set([...extractedLinks, ...lowerLevelLinks]);
|
|
307
|
+
}
|
|
308
|
+
return combinedLinks;
|
|
309
|
+
}
|
|
310
|
+
catch (e) {
|
|
311
|
+
console.error(`Failed to get links from ${url}`);
|
|
312
|
+
return new Set();
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
async delay(ms) {
|
|
316
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
317
|
+
}
|
|
318
|
+
};
|
|
319
|
+
exports.AutotagWebsite = AutotagWebsite;
|
|
320
|
+
exports.AutotagWebsite = AutotagWebsite = __decorate([
|
|
321
|
+
(0, global_1.RegisterClass)(Core_1.AutotagBase, 'AutotagWebsite'),
|
|
322
|
+
__metadata("design:paramtypes", [])
|
|
323
|
+
], AutotagWebsite);
|
|
324
|
+
//# sourceMappingURL=AutotagWebsite.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AutotagWebsite.js","sourceRoot":"","sources":["../../../../src/Websites/generic/AutotagWebsite.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,qCAAyC;AACzC,yCAAsE;AACtE,mDAAuD;AACvD,+CAAmE;AAEnE,iDAAmC;AACnC,kDAA0B;AAC1B,6BAA0B;AAC1B,oDAA4B;AAC5B,gBAAM,CAAC,MAAM,EAAE,CAAA;AAGR,IAAM,cAAc,GAApB,MAAM,cAAe,SAAQ,kBAAW;IACnC,WAAW,CAAW;IACtB,MAAM,CAAoB;IACxB,mBAAmB,CAAQ;IAC3B,+BAA+B,CAAU;IACzC,4BAA4B,CAAU;IACtC,QAAQ,CAAS;IACjB,OAAO,CAAS;IAChB,UAAU,CAAS;IACnB,WAAW,CAAc;IAEnC;QACI,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,0BAAiB,CAAC,QAAQ,CAAC;QACzC,IAAI,CAAC,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACzC,CAAC;IAES,cAAc;QACpB,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAOM,KAAK,CAAC,OAAO,CAAC,WAAqB;QACtC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,mBAAmB,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACvG,MAAM,cAAc,GAA0B,MAAM,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;QACjI,MAAM,qBAAqB,GAAwB,MAAM,IAAI,CAAC,wBAAwB,CAAC,cAAc,CAAC,CAAC;QACvG,MAAM,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,qBAAqB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAC5F,CAAC;IASM,KAAK,CAAC,wBAAwB,CAAC,cAAqC;QACvE,MAAM,qBAAqB,GAAwB,EAAE,CAAA;QAGrD,KAAK,MAAM,aAAa,IAAI,cAAc,EAAE,CAAC;YACzC,MAAM,sBAAsB,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,aAAa,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YACzG,IAAI,sBAAsB,EAAE,CAAC;gBAEzB,sBAAsB,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;oBAC1C,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;wBACb,IAAY,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;oBAC/B,CAAC;gBACL,CAAC,CAAC,CAAA;YACN,CAAC;YAED,MAAM,mBAAmB,GAAwB;gBAC7C,eAAe,EAAE,aAAa,CAAC,EAAE;gBACjC,IAAI,EAAE,aAAa,CAAC,IAAI;gBACxB,aAAa,EAAE,aAAa,CAAC,aAAa;gBAC1C,iBAAiB,EAAE,aAAa,CAAC,iBAAiB;gBAClD,mBAAmB,EAAE,aAAa,CAAC,mBAAmB;gBACtD,GAAG,EAAE,aAAa,CAAC,GAAG;aACzB,CAAA;YAED,IAAI,CAAC;gBAGD,MAAM,QAAQ,GAAW,mBAAmB,CAAC,GAAG,CAAC;gBAGjD,MAAM,OAAO,GAAW,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBAGjF,MAAM,KAAK,GAAW,IAAI,CAAC,UAAU,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC;gBAEzF,MAAM,mBAAmB,GAAa,MAAM,IAAI,CAAC,4BAA4B,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;gBACxG,MAAM,YAAY,GAAwB,MAAM,IAAI,CAAC,6BAA6B,CAAC,mBAAmB,EAAE,mBAAmB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;gBAC/I,IAAI,YAAY,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC1C,qBAAqB,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;gBAChD,CAAC;qBACI,CAAC;oBAEF,OAAO,CAAC,GAAG,CAAC,yDAAyD,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBACtG,CAAC;YACL,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,OAAO,CAAC,KAAK,CAAC,qCAAqC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACpF,CAAC;QACL,CAAC;QAED,OAAO,qBAAqB,CAAC;IACjC,CAAC;IAWS,KAAK,CAAC,6BAA6B,CAAC,gBAA0B,EAAE,mBAAwC,EAAE,WAAqB;QAErI,MAAM,iBAAiB,GAAwB,EAAE,CAAC;QAClD,KAAK,MAAM,eAAe,IAAI,gBAAgB,EAAE,CAAC;YAC7C,IAAI,CAAC;gBACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;gBAEtE,MAAM,EAAE,GAAG,IAAI,cAAO,EAAE,CAAC;gBACzB,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAoB;oBACjD;wBACI,UAAU,EAAE,eAAe;wBAC3B,WAAW,EAAE,eAAe,OAAO,GAAG;wBACtC,UAAU,EAAE,eAAe;qBAC9B;oBACD;wBACI,UAAU,EAAE,eAAe;wBAC3B,WAAW,EAAE,sBAAsB,mBAAmB,CAAC,eAAe,gBAAgB,eAAe,GAAG;wBACxG,UAAU,EAAE,eAAe;qBAC9B;iBACJ,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;gBAEpB,MAAM,8BAA8B,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;gBACjD,MAAM,yBAAyB,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;gBAE5C,IAAI,8BAA8B,CAAC,OAAO,IAAI,8BAA8B,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;oBAE1F,SAAS;gBACb,CAAC;qBAEI,IAAI,yBAAyB,CAAC,OAAO,IAAI,yBAAyB,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;oBAErF,MAAM,iBAAiB,GAAsB,yBAAyB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;oBAClF,MAAM,cAAc,GAAW,iBAAiB,CAAC,QAAQ,CAAA;oBAEzD,IAAI,cAAc,KAAK,OAAO,EAAE,CAAC;wBAE7B,MAAM,EAAE,GAAG,IAAI,eAAQ,EAAE,CAAC;wBAC1B,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,eAAe,CAAoB,eAAe,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;wBACnG,WAAW,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;wBACvC,WAAW,CAAC,QAAQ,GAAG,OAAO,CAAA;wBAC9B,WAAW,CAAC,IAAI,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,eAAe,CAAC,CAAA;wBAE3D,MAAM,WAAW,CAAC,IAAI,EAAE,CAAC;wBACzB,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;oBACxC,CAAC;gBACL,CAAC;qBACI,CAAC;oBAEF,MAAM,EAAE,GAAG,IAAI,eAAQ,EAAE,CAAC;oBAC1B,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,eAAe,CAAoB,eAAe,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;oBACnG,WAAW,CAAC,eAAe,GAAG,mBAAmB,CAAC,eAAe,CAAA;oBACjE,WAAW,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,CAAA;oBACpD,WAAW,CAAC,WAAW,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,yBAAyB,CAAC,mBAAmB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;oBAC5G,WAAW,CAAC,aAAa,GAAG,mBAAmB,CAAC,aAAa,CAAA;oBAC7D,WAAW,CAAC,iBAAiB,GAAG,mBAAmB,CAAC,iBAAiB,CAAA;oBACrE,WAAW,CAAC,mBAAmB,GAAG,mBAAmB,CAAC,mBAAmB,CAAA;oBACzE,WAAW,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAA;oBAC5E,WAAW,CAAC,GAAG,GAAG,eAAe,CAAA;oBACjC,WAAW,CAAC,IAAI,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,eAAe,CAAC,CAAA;oBAE3D,MAAM,WAAW,CAAC,IAAI,EAAE,CAAC;oBACzB,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBACxC,CAAC;YACD,CAAC;YAAA,OAAO,CAAC,EAAE,CAAC;gBACR,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YAClB,CAAC;QACT,CAAC;QACD,OAAO,iBAAiB,CAAC;IAC7B,CAAC;IAEM,KAAK,CAAC,gBAAgB,CAAC,GAAW;QACrC,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,eAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACtC,OAAO,IAAI,CAAC;IAChB,CAAC;IAEM,qBAAqB,CAAC,OAAY,EAAE,CAAqB;QAC5D,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;QAEvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;YACvB,IAAI,EAAE,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACrB,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC;YACtC,CAAC;iBAAM,IAAI,EAAE,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;gBAC3B,IAAI,IAAI,IAAI,GAAG,IAAI,CAAC,qBAAqB,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC;YAC5D,CAAC;QACL,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAOM,KAAK,CAAC,YAAY,CAAC,GAAW;QACjC,IAAI,CAAC;YACD,MAAM,WAAW,GAAW,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACpC,MAAM,IAAI,GAAW,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACjE,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,oBAAoB,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;YACjD,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IASS,KAAK,CAAC,4BAA4B,CAAC,GAAW,EAAE,OAAe,EAAE,KAAa;QAEpF,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,GAAG,EAAU,EAAE,KAAK,CAAC,CAAC;YACrF,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC;YAExD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACxC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,OAAO,CAAC,KAAK,CAAC,4BAA4B,GAAG,EAAE,CAAC,CAAC;YACjD,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IASS,KAAK,CAAC,gBAAgB,CAAC,GAAW,EAAE,OAAe;QACzD,IAAI,CAAC,IAAI,CAAC,+BAA+B,EAAE,CAAC;YACxC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC1B,OAAM;QACV,CAAC;QAGD,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC;YACxF,OAAM;QACV,CAAC;QAED,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAE1B,IAAI,CAAC;YACD,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,eAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAG7B,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;gBACvB,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrC,IAAI,IAAI,EAAE,CAAC;oBACP,MAAM,MAAM,GAAG,IAAI,SAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC;oBACvC,IAAI,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;wBAC9D,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBACjC,CAAC;gBACL,CAAC;YACL,CAAC,CAAC,CAAC;YACH,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC3B,CAAC;QACD,OAAO,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,KAAK,CAAC,4BAA4B,GAAG,EAAE,CAAC,CAAC;YACjD,OAAM;QACV,CAAC;IACL,CAAC;IAOS,eAAe,CAAC,GAAW;QACjC,IAAI,CAAC;YACD,MAAM,SAAS,GAAQ,IAAI,SAAG,CAAC,GAAG,CAAC,CAAC;YACpC,OAAO,SAAS,CAAC,QAAQ,KAAK,GAAG,IAAI,SAAS,CAAC,QAAQ,KAAK,EAAE,CAAC;QACnE,CAAC;QACD,OAAO,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,KAAK,CAAC,uCAAuC,GAAG,EAAE,CAAC,CAAC;YAC5D,MAAM,CAAC,CAAC;QACZ,CAAC;IACL,CAAC;IAES,WAAW,CAAC,GAAW;QAC7B,MAAM,SAAS,GAAQ,IAAI,SAAG,CAAC,GAAG,CAAC,CAAC;QACpC,MAAM,YAAY,GAAa,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;QACxF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,YAAY,CAAC,GAAG,EAAE,CAAC;QACvB,CAAC;QACD,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,GAAG,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjE,OAAO,QAAQ,CAAC;IACpB,CAAC;IAGS,WAAW,CAAC,GAAW;QAC7B,IAAI,CAAC;YACD,MAAM,SAAS,GAAQ,IAAI,SAAG,CAAC,GAAG,CAAC,CAAC;YACpC,MAAM,YAAY,GAAa,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;YACxF,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,GAAG,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC7D,OAAO,IAAI,CAAA;QACf,CAAC;QACD,OAAO,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,KAAK,CAAC,uCAAuC,GAAG,EAAE,CAAC,CAAC;YAC5D,MAAM,CAAC,CAAC;QACZ,CAAC;IACL,CAAC;IAES,KAAK,CAAC,UAAU,CAAC,GAAW;QAClC,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,eAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvC,OAAO,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC;QACnC,CAAC;QACD,OAAO,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,KAAK,CAAC,gBAAgB,GAAG,EAAE,CAAC,CAAC;YACrC,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;IAUS,KAAK,CAAC,kBAAkB,CAAC,GAAW,EAAE,OAAe,EAAE,UAAkB,EAAE,WAAwB,EAAE,KAAa;QAExH,IAAI,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,YAAY,GAAG,EAAE,CAAC,CAAC;YAE/B,IAAI,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,KAAK,IAAI,UAAU,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,4BAA4B,EAAE,CAAC;gBACvH,OAAO,IAAI,GAAG,EAAU,CAAC;YAC7B,CAAC;YAED,IAAI,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;YACtC,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;YAEzC,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,eAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAG7B,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;gBACvB,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrC,IAAI,IAAI,EAAE,CAAC;oBACP,MAAM,MAAM,GAAG,IAAI,SAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC;oBACvC,IAAI,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;wBACtG,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;wBAC3B,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBACjC,CAAC;gBACL,CAAC;YACL,CAAC,CAAC,CAAC;YACH,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACvB,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAGrB,IAAI,UAAU,KAAK,CAAC,EAAE,CAAC;gBACnB,OAAO,cAAc,CAAC;YAC1B,CAAC;YAED,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;gBAEnC,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,GAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;gBAC1G,aAAa,GAAG,IAAI,GAAG,CAAS,CAAC,GAAG,cAAc,EAAE,GAAG,eAAe,CAAC,CAAC,CAAC;YAC7E,CAAC;YACD,OAAO,aAAa,CAAC;QACzB,CAAC;QACD,OAAO,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,KAAK,CAAC,4BAA4B,GAAG,EAAE,CAAC,CAAC;YACjD,OAAO,IAAI,GAAG,EAAU,CAAC;QAC7B,CAAC;IACL,CAAC;IAES,KAAK,CAAC,KAAK,CAAC,EAAU;QAC5B,OAAO,IAAI,OAAO,CAAE,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAE,CAAC;IAC7D,CAAC;CACJ,CAAA;AA9XY,wCAAc;yBAAd,cAAc;IAD1B,IAAA,sBAAa,EAAC,kBAAW,EAAE,gBAAgB,CAAC;;GAChC,cAAc,CA8X1B"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/Websites/index.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("./generic/AutotagWebsite"), exports);
|
|
18
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/Websites/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,2DAAwC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,QAAQ,CAAC;AACvB,cAAc,UAAU,CAAC;AACzB,cAAc,mBAAmB,CAAC;AAClC,cAAc,WAAW,CAAC;AAC1B,cAAc,YAAY,CAAC;AAC3B,cAAc,gBAAgB,CAAA"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("./Core"), exports);
|
|
18
|
+
__exportStar(require("./Engine"), exports);
|
|
19
|
+
__exportStar(require("./LocalFileSystem"), exports);
|
|
20
|
+
__exportStar(require("./RSSFeed"), exports);
|
|
21
|
+
__exportStar(require("./Websites"), exports);
|
|
22
|
+
__exportStar(require("./CloudStorage"), exports);
|
|
23
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,yCAAuB;AACvB,2CAAyB;AACzB,oDAAkC;AAClC,4CAA0B;AAC1B,6CAA2B;AAC3B,iDAA8B"}
|