@memberjunction/content-autotagging 6.2.0-edge.0 → 6.2.0-edge.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/CloudStorage/providers/AutotagAzureBlob.d.ts +4 -0
- package/dist/CloudStorage/providers/AutotagAzureBlob.d.ts.map +1 -1
- package/dist/CloudStorage/providers/AutotagAzureBlob.js +13 -5
- package/dist/CloudStorage/providers/AutotagAzureBlob.js.map +1 -1
- package/dist/CloudStorage/providers/AutotagCloudStorage.d.ts +8 -8
- package/dist/CloudStorage/providers/AutotagCloudStorage.js +17 -17
- package/dist/Engine/generic/AutotagBaseEngine.d.ts +100 -13
- package/dist/Engine/generic/AutotagBaseEngine.d.ts.map +1 -1
- package/dist/Engine/generic/AutotagBaseEngine.js +232 -85
- package/dist/Engine/generic/AutotagBaseEngine.js.map +1 -1
- package/dist/Engine/generic/FieldPathResolver.d.ts +35 -2
- package/dist/Engine/generic/FieldPathResolver.d.ts.map +1 -1
- package/dist/Engine/generic/FieldPathResolver.js +53 -9
- package/dist/Engine/generic/FieldPathResolver.js.map +1 -1
- package/dist/Engine/generic/RunBudget.d.ts +22 -0
- package/dist/Engine/generic/RunBudget.d.ts.map +1 -1
- package/dist/Engine/generic/RunBudget.js +40 -8
- package/dist/Engine/generic/RunBudget.js.map +1 -1
- package/dist/Engine/generic/ScopeContextResolver.d.ts +4 -0
- package/dist/Engine/generic/ScopeContextResolver.d.ts.map +1 -1
- package/dist/Engine/generic/ScopeContextResolver.js +10 -2
- package/dist/Engine/generic/ScopeContextResolver.js.map +1 -1
- package/dist/Entity/generic/AutotagEntity.d.ts +12 -12
- package/dist/Entity/generic/AutotagEntity.js +24 -24
- package/dist/LocalFileSystem/generic/AutotagLocalFileSystem.d.ts +8 -0
- package/dist/LocalFileSystem/generic/AutotagLocalFileSystem.d.ts.map +1 -1
- package/dist/LocalFileSystem/generic/AutotagLocalFileSystem.js +23 -7
- package/dist/LocalFileSystem/generic/AutotagLocalFileSystem.js.map +1 -1
- package/dist/RSSFeed/generic/AutotagRSSFeed.d.ts +6 -6
- package/dist/RSSFeed/generic/AutotagRSSFeed.js +12 -12
- package/dist/Websites/generic/AutotagWebsite.d.ts +11 -0
- package/dist/Websites/generic/AutotagWebsite.d.ts.map +1 -1
- package/dist/Websites/generic/AutotagWebsite.js +24 -8
- package/dist/Websites/generic/AutotagWebsite.js.map +1 -1
- package/package.json +19 -19
|
@@ -45,11 +45,11 @@ let AutotagEntity = class AutotagEntity extends AutotagBase {
|
|
|
45
45
|
contentSources = contentSources.filter(s => contentSourceIDs.some(id => UUIDsEqual(id, s.ID)));
|
|
46
46
|
}
|
|
47
47
|
// Cache content source configs and set up taxonomy
|
|
48
|
-
await this.
|
|
48
|
+
await this.setupTaxonomyAndBridge(contentSources, contextUser);
|
|
49
49
|
const contentItemsToProcess = await this.SetContentItemsToProcess(contentSources);
|
|
50
50
|
// Also pick up previously failed items — ContentItems that exist but have
|
|
51
51
|
// no tags (LLM call failed due to rate limits, network errors, etc.)
|
|
52
|
-
const retryItems = await this.
|
|
52
|
+
const retryItems = await this.getUntaggedContentItems(contentSources);
|
|
53
53
|
if (retryItems.length > 0) {
|
|
54
54
|
LogStatus(`AutotagEntity: found ${retryItems.length} previously failed items to retry`);
|
|
55
55
|
}
|
|
@@ -76,7 +76,7 @@ let AutotagEntity = class AutotagEntity extends AutotagBase {
|
|
|
76
76
|
* Initialize the TagEngine, inject taxonomy into the prompt, and set up the
|
|
77
77
|
* bridge callback that links ContentItemTags to formal Tag + TaggedItem records.
|
|
78
78
|
*/
|
|
79
|
-
async
|
|
79
|
+
async setupTaxonomyAndBridge(contentSources, contextUser) {
|
|
80
80
|
// Cache source configs for bridge callback — use CodeGen-generated typed accessor
|
|
81
81
|
for (const source of contentSources) {
|
|
82
82
|
const normalizedID = NormalizeUUID(source.ID);
|
|
@@ -131,7 +131,7 @@ let AutotagEntity = class AutotagEntity extends AutotagBase {
|
|
|
131
131
|
}
|
|
132
132
|
// Set up the bridge callback for tag taxonomy linking
|
|
133
133
|
this.engine.OnContentItemTagSaved = async (contentItemTag, parentTagName, ctxUser) => {
|
|
134
|
-
await this.
|
|
134
|
+
await this.bridgeContentItemTagToTaxonomy(contentItemTag, parentTagName, ctxUser);
|
|
135
135
|
};
|
|
136
136
|
// Per-batch budget gate — pause the run when any source's budget is exceeded.
|
|
137
137
|
// Tally items per source first so MaxItemsPerRun ticks; then check all budgets.
|
|
@@ -161,7 +161,7 @@ let AutotagEntity = class AutotagEntity extends AutotagBase {
|
|
|
161
161
|
* Uses TagEngine.ResolveTag() for semantic matching with mode/threshold from source config.
|
|
162
162
|
* Creates a TaggedItem linking the resolved Tag to the source entity record.
|
|
163
163
|
*/
|
|
164
|
-
async
|
|
164
|
+
async bridgeContentItemTagToTaxonomy(contentItemTag, parentTagName, contextUser) {
|
|
165
165
|
// Find the source config for this content item
|
|
166
166
|
const rv = new RunView();
|
|
167
167
|
const ciResult = await rv.RunView({
|
|
@@ -282,7 +282,7 @@ let AutotagEntity = class AutotagEntity extends AutotagBase {
|
|
|
282
282
|
* Find ContentItems that exist for these sources but have no ContentItemTag records.
|
|
283
283
|
* These are items where the LLM tagging failed on a previous run (rate limits, network errors, etc.).
|
|
284
284
|
*/
|
|
285
|
-
async
|
|
285
|
+
async getUntaggedContentItems(contentSources) {
|
|
286
286
|
if (contentSources.length === 0)
|
|
287
287
|
return [];
|
|
288
288
|
const sourceIDs = contentSources.map(s => `'${s.ID}'`).join(',');
|
|
@@ -301,7 +301,7 @@ let AutotagEntity = class AutotagEntity extends AutotagBase {
|
|
|
301
301
|
const contentItemsToProcess = [];
|
|
302
302
|
for (const contentSource of contentSources) {
|
|
303
303
|
try {
|
|
304
|
-
const items = await this.
|
|
304
|
+
const items = await this.processContentSource(contentSource);
|
|
305
305
|
contentItemsToProcess.push(...items);
|
|
306
306
|
}
|
|
307
307
|
catch (e) {
|
|
@@ -315,7 +315,7 @@ let AutotagEntity = class AutotagEntity extends AutotagBase {
|
|
|
315
315
|
* Process a single entity-type ContentSource: load entity records modified since last run,
|
|
316
316
|
* render via EntityDocument template, create/update ERD and ContentItem records.
|
|
317
317
|
*/
|
|
318
|
-
async
|
|
318
|
+
async processContentSource(contentSource) {
|
|
319
319
|
const entityID = contentSource.EntityID;
|
|
320
320
|
const entityDocumentID = contentSource.EntityDocumentID;
|
|
321
321
|
if (!entityID || !entityDocumentID) {
|
|
@@ -323,12 +323,12 @@ let AutotagEntity = class AutotagEntity extends AutotagBase {
|
|
|
323
323
|
return [];
|
|
324
324
|
}
|
|
325
325
|
// Load the EntityDocument
|
|
326
|
-
const entityDocument = await this.
|
|
326
|
+
const entityDocument = await this.loadEntityDocument(entityDocumentID);
|
|
327
327
|
if (!entityDocument) {
|
|
328
328
|
LogError(`AutotagEntity: EntityDocument ${entityDocumentID} not found for source "${contentSource.Name}"`);
|
|
329
329
|
return [];
|
|
330
330
|
}
|
|
331
|
-
const templateText = await this.
|
|
331
|
+
const templateText = await this.getTemplateText(entityDocument);
|
|
332
332
|
if (!templateText) {
|
|
333
333
|
LogError(`AutotagEntity: no template content found for EntityDocument "${entityDocument.Name}"`);
|
|
334
334
|
return [];
|
|
@@ -342,16 +342,16 @@ let AutotagEntity = class AutotagEntity extends AutotagBase {
|
|
|
342
342
|
}
|
|
343
343
|
// Get records modified since last run
|
|
344
344
|
const lastRunDate = await this.engine.getContentSourceLastRunDate(contentSource.ID, this.contextUser);
|
|
345
|
-
const modifiedRecords = await this.
|
|
345
|
+
const modifiedRecords = await this.getModifiedRecords(entityInfo.Name, lastRunDate);
|
|
346
346
|
if (modifiedRecords.length === 0) {
|
|
347
347
|
LogStatus(`AutotagEntity: no modified records for entity "${entityInfo.Name}" since ${lastRunDate.toISOString()}`);
|
|
348
348
|
return [];
|
|
349
349
|
}
|
|
350
350
|
LogStatus(`AutotagEntity: processing ${modifiedRecords.length} modified records for entity "${entityInfo.Name}"`);
|
|
351
351
|
// Load existing ERD records for this entity document to enable upsert
|
|
352
|
-
const existingERDs = await this.
|
|
352
|
+
const existingERDs = await this.loadExistingERDs(entityDocumentID, entityID);
|
|
353
353
|
// Load existing ContentItems for this source to enable upsert
|
|
354
|
-
const existingContentItems = await this.
|
|
354
|
+
const existingContentItems = await this.loadExistingContentItems(contentSource.ID);
|
|
355
355
|
// Process each record: render template → create/update ERD → create/update ContentItem
|
|
356
356
|
const contentItems = [];
|
|
357
357
|
const parser = EntityDocumentTemplateParser.CreateInstance();
|
|
@@ -361,7 +361,7 @@ let AutotagEntity = class AutotagEntity extends AutotagBase {
|
|
|
361
361
|
const recordID = CompositeKey.FromEntityRecord(entityInfo, record).ToCompactURLSegment();
|
|
362
362
|
try {
|
|
363
363
|
const recordName = this.buildContentItemName(entityInfo, record);
|
|
364
|
-
const contentItem = await this.
|
|
364
|
+
const contentItem = await this.processSingleRecord(record, recordID, recordName, contentSource, entityDocument, templateText, parser, existingERDs, existingContentItems);
|
|
365
365
|
if (contentItem) {
|
|
366
366
|
contentItems.push(contentItem);
|
|
367
367
|
}
|
|
@@ -405,7 +405,7 @@ let AutotagEntity = class AutotagEntity extends AutotagBase {
|
|
|
405
405
|
/**
|
|
406
406
|
* Load an EntityDocument by ID using RunView.
|
|
407
407
|
*/
|
|
408
|
-
async
|
|
408
|
+
async loadEntityDocument(entityDocumentID) {
|
|
409
409
|
const rv = new RunView();
|
|
410
410
|
const result = await rv.RunView({
|
|
411
411
|
EntityName: 'MJ: Entity Documents',
|
|
@@ -422,7 +422,7 @@ let AutotagEntity = class AutotagEntity extends AutotagBase {
|
|
|
422
422
|
* Retrieves the template text from the EntityDocument's linked Template.
|
|
423
423
|
* Ensures TemplateEngineServer is configured before accessing Templates.
|
|
424
424
|
*/
|
|
425
|
-
async
|
|
425
|
+
async getTemplateText(entityDocument) {
|
|
426
426
|
if (!entityDocument.TemplateID)
|
|
427
427
|
return null;
|
|
428
428
|
// Ensure engine is loaded (no-op if already configured)
|
|
@@ -436,7 +436,7 @@ let AutotagEntity = class AutotagEntity extends AutotagBase {
|
|
|
436
436
|
/**
|
|
437
437
|
* Query entity records modified since lastRunDate using __mj_UpdatedAt.
|
|
438
438
|
*/
|
|
439
|
-
async
|
|
439
|
+
async getModifiedRecords(entityName, lastRunDate) {
|
|
440
440
|
const rv = new RunView();
|
|
441
441
|
const result = await rv.RunView({
|
|
442
442
|
EntityName: entityName,
|
|
@@ -452,7 +452,7 @@ let AutotagEntity = class AutotagEntity extends AutotagBase {
|
|
|
452
452
|
* Load existing EntityRecordDocument records for this entity document,
|
|
453
453
|
* keyed by normalized RecordID for fast lookup.
|
|
454
454
|
*/
|
|
455
|
-
async
|
|
455
|
+
async loadExistingERDs(entityDocumentID, entityID) {
|
|
456
456
|
const rv = new RunView();
|
|
457
457
|
const result = await rv.RunView({
|
|
458
458
|
EntityName: 'MJ: Entity Record Documents',
|
|
@@ -471,7 +471,7 @@ let AutotagEntity = class AutotagEntity extends AutotagBase {
|
|
|
471
471
|
* Load existing ContentItems for this content source, keyed by normalized
|
|
472
472
|
* EntityRecordDocumentID for fast lookup during upsert.
|
|
473
473
|
*/
|
|
474
|
-
async
|
|
474
|
+
async loadExistingContentItems(contentSourceID) {
|
|
475
475
|
const rv = new RunView();
|
|
476
476
|
const result = await rv.RunView({
|
|
477
477
|
EntityName: 'MJ: Content Items',
|
|
@@ -491,7 +491,7 @@ let AutotagEntity = class AutotagEntity extends AutotagBase {
|
|
|
491
491
|
/**
|
|
492
492
|
* Process a single entity record: render template, create/update ERD, create/update ContentItem.
|
|
493
493
|
*/
|
|
494
|
-
async
|
|
494
|
+
async processSingleRecord(record, recordID, recordName, contentSource, entityDocument, templateText, parser, existingERDs, existingContentItems) {
|
|
495
495
|
const entityID = contentSource.EntityID;
|
|
496
496
|
// 1. Render the template for this record
|
|
497
497
|
const renderedText = await parser.Parse(templateText, entityID, record, this.contextUser);
|
|
@@ -500,17 +500,17 @@ let AutotagEntity = class AutotagEntity extends AutotagBase {
|
|
|
500
500
|
return null;
|
|
501
501
|
}
|
|
502
502
|
// 2. Create or update EntityRecordDocument
|
|
503
|
-
const erd = await this.
|
|
503
|
+
const erd = await this.upsertEntityRecordDocument(entityID, recordID, entityDocument, renderedText, existingERDs);
|
|
504
504
|
// 3. Compute checksum to detect content changes
|
|
505
505
|
const checksum = await this.engine.getChecksumFromText(renderedText);
|
|
506
506
|
// 4. Create or update ContentItem linked to ERD
|
|
507
|
-
const contentItem = await this.
|
|
507
|
+
const contentItem = await this.upsertContentItem(contentSource, erd, renderedText, checksum, record, recordName, existingContentItems);
|
|
508
508
|
return contentItem;
|
|
509
509
|
}
|
|
510
510
|
/**
|
|
511
511
|
* Create or update an EntityRecordDocument snapshot.
|
|
512
512
|
*/
|
|
513
|
-
async
|
|
513
|
+
async upsertEntityRecordDocument(entityID, recordID, entityDocument, documentText, existingERDs) {
|
|
514
514
|
const md = this.ProviderToUse;
|
|
515
515
|
let erd = existingERDs.get(recordID);
|
|
516
516
|
if (erd) {
|
|
@@ -542,7 +542,7 @@ let AutotagEntity = class AutotagEntity extends AutotagBase {
|
|
|
542
542
|
* Only creates a new ContentItem if the checksum changed (content actually changed)
|
|
543
543
|
* or if no ContentItem exists yet for this ERD.
|
|
544
544
|
*/
|
|
545
|
-
async
|
|
545
|
+
async upsertContentItem(contentSource, erd, text, checksum, record, recordName, existingContentItems) {
|
|
546
546
|
const md = this.ProviderToUse;
|
|
547
547
|
const erdNormalizedID = NormalizeUUID(erd.ID);
|
|
548
548
|
const existing = existingContentItems.get(erdNormalizedID);
|
|
@@ -9,6 +9,8 @@ export declare class AutotagLocalFileSystem extends AutotagBase {
|
|
|
9
9
|
protected contentSourceTypeID: string;
|
|
10
10
|
static _openAI: OpenAI;
|
|
11
11
|
constructor();
|
|
12
|
+
GetContextUser(): UserInfo | null;
|
|
13
|
+
/** @deprecated Use {@link GetContextUser}. */
|
|
12
14
|
getContextUser(): UserInfo | null;
|
|
13
15
|
/**
|
|
14
16
|
* Implemented abstract method from the AutotagBase class. that runs the entire autotagging process. This method is the entry point for the autotagging process.
|
|
@@ -23,6 +25,8 @@ export declare class AutotagLocalFileSystem extends AutotagBase {
|
|
|
23
25
|
* @returns - An array of content source items that have been modified or added after the most recent process run for that content source
|
|
24
26
|
*/
|
|
25
27
|
SetContentItemsToProcess(contentSources: MJContentSourceEntity[]): Promise<MJContentItemEntity[]>;
|
|
28
|
+
SetContentSourceParams(contentSource: MJContentSourceEntity): Promise<ContentSourceParams>;
|
|
29
|
+
/** @deprecated Use {@link SetContentSourceParams}. */
|
|
26
30
|
setContentSourceParams(contentSource: MJContentSourceEntity): Promise<ContentSourceParams>;
|
|
27
31
|
/**
|
|
28
32
|
* Given a content source and last run date, recursively traverse through the directory and return a
|
|
@@ -33,7 +37,11 @@ export declare class AutotagLocalFileSystem extends AutotagBase {
|
|
|
33
37
|
* @returns
|
|
34
38
|
*/
|
|
35
39
|
SetNewAndModifiedContentItems(contentSourceParams: ContentSourceParams, lastRunDate: Date, contextUser: UserInfo): Promise<MJContentItemEntity[]>;
|
|
40
|
+
SetAddedContentItem(filePath: string, contentSourceParams: ContentSourceParams): Promise<MJContentItemEntity>;
|
|
41
|
+
/** @deprecated Use {@link SetAddedContentItem}. */
|
|
36
42
|
setAddedContentItem(filePath: string, contentSourceParams: ContentSourceParams): Promise<MJContentItemEntity>;
|
|
43
|
+
SetModifiedContentItem(filePath: string, contentSourceParams: ContentSourceParams): Promise<MJContentItemEntity>;
|
|
44
|
+
/** @deprecated Use {@link SetModifiedContentItem}. */
|
|
37
45
|
setModifiedContentItem(filePath: string, contentSourceParams: ContentSourceParams): Promise<MJContentItemEntity>;
|
|
38
46
|
}
|
|
39
47
|
//# sourceMappingURL=AutotagLocalFileSystem.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AutotagLocalFileSystem.d.ts","sourceRoot":"","sources":["../../../src/LocalFileSystem/generic/AutotagLocalFileSystem.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,uBAAuB,EAAE,MAAM,YAAY,CAAC;AAClE,OAAO,EAAqB,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACtE,OAAO,EAAE,iBAAiB,EAAE,QAAQ,EAAqB,MAAM,sBAAsB,CAAC;AACtF,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AAC3F,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAKhC,qBACa,sBAAuB,SAAQ,WAAW;IACnD,OAAO,CAAC,WAAW,CAAW;IAC9B,OAAO,CAAC,MAAM,CAAoB;IAClC,SAAS,CAAC,mBAAmB,EAAE,MAAM,CAAA;IACrC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC;;IAOhB,cAAc,IAAI,QAAQ,GAAG,IAAI;IAIxC;;;;OAIG;IACU,OAAO,CAAC,WAAW,EAAE,QAAQ,EAAE,UAAU,CAAC,EAAE,uBAAuB,EAAE,gBAAgB,CAAC,EAAE,MAAM,EAAE,EAAE,QAAQ,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC;IAU7J;;;;;MAKE;IACW,wBAAwB,CAAC,cAAc,EAAE,qBAAqB,EAAE,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IA+BjG,sBAAsB,CAAC,aAAa,EAAE,qBAAqB;IAwBxE;;;;;;;OAOG;IACU,6BAA6B,CAAC,mBAAmB,EAAE,mBAAmB,EAAE,WAAW,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IA+BjJ,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,mBAAmB,EAAE,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"AutotagLocalFileSystem.d.ts","sourceRoot":"","sources":["../../../src/LocalFileSystem/generic/AutotagLocalFileSystem.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,uBAAuB,EAAE,MAAM,YAAY,CAAC;AAClE,OAAO,EAAqB,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACtE,OAAO,EAAE,iBAAiB,EAAE,QAAQ,EAAqB,MAAM,sBAAsB,CAAC;AACtF,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AAC3F,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAKhC,qBACa,sBAAuB,SAAQ,WAAW;IACnD,OAAO,CAAC,WAAW,CAAW;IAC9B,OAAO,CAAC,MAAM,CAAoB;IAClC,SAAS,CAAC,mBAAmB,EAAE,MAAM,CAAA;IACrC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC;;IAOhB,cAAc,IAAI,QAAQ,GAAG,IAAI;IAIxC,8CAA8C;IACvC,cAAc,IAAI,QAAQ,GAAG,IAAI;IAIxC;;;;OAIG;IACU,OAAO,CAAC,WAAW,EAAE,QAAQ,EAAE,UAAU,CAAC,EAAE,uBAAuB,EAAE,gBAAgB,CAAC,EAAE,MAAM,EAAE,EAAE,QAAQ,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC;IAU7J;;;;;MAKE;IACW,wBAAwB,CAAC,cAAc,EAAE,qBAAqB,EAAE,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IA+BjG,sBAAsB,CAAC,aAAa,EAAE,qBAAqB;IAwBxE,sDAAsD;IACzC,sBAAsB,CAAC,aAAa,EAAE,qBAAqB;IAIxE;;;;;;;OAOG;IACU,6BAA6B,CAAC,mBAAmB,EAAE,mBAAmB,EAAE,WAAW,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IA+BjJ,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,mBAAmB,EAAE,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,CAAC;IAuB1H,mDAAmD;IACtC,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,mBAAmB,EAAE,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,CAAC;IAI7G,sBAAsB,CAAC,QAAQ,EAAE,MAAM,EAAE,mBAAmB,EAAE,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,CAAC;IAiB7H,sDAAsD;IACzC,sBAAsB,CAAC,QAAQ,EAAE,MAAM,EAAE,mBAAmB,EAAE,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,CAAC;CAGhI"}
|
|
@@ -19,9 +19,13 @@ let AutotagLocalFileSystem = class AutotagLocalFileSystem extends AutotagBase {
|
|
|
19
19
|
super();
|
|
20
20
|
this.engine = AutotagBaseEngine.Instance;
|
|
21
21
|
}
|
|
22
|
-
|
|
22
|
+
GetContextUser() {
|
|
23
23
|
return this.contextUser;
|
|
24
24
|
}
|
|
25
|
+
/** @deprecated Use {@link GetContextUser}. */
|
|
26
|
+
getContextUser() {
|
|
27
|
+
return this.GetContextUser();
|
|
28
|
+
}
|
|
25
29
|
/**
|
|
26
30
|
* Implemented abstract method from the AutotagBase class. that runs the entire autotagging process. This method is the entry point for the autotagging process.
|
|
27
31
|
* It initializes the connection, retrieves the content sources corresponding to the content source type, sets the content items that we want to process,
|
|
@@ -48,7 +52,7 @@ let AutotagLocalFileSystem = class AutotagLocalFileSystem extends AutotagBase {
|
|
|
48
52
|
for (const contentSource of contentSources) {
|
|
49
53
|
// First check that the directory exists
|
|
50
54
|
if (fs.existsSync(contentSource.URL)) {
|
|
51
|
-
const contentSourceParams = await this.
|
|
55
|
+
const contentSourceParams = await this.SetContentSourceParams(contentSource);
|
|
52
56
|
const lastRunDate = await this.engine.getContentSourceLastRunDate(contentSourceParams.contentSourceID, this.contextUser);
|
|
53
57
|
// Traverse through all the files in the directory
|
|
54
58
|
if (lastRunDate) {
|
|
@@ -71,7 +75,7 @@ let AutotagLocalFileSystem = class AutotagLocalFileSystem extends AutotagBase {
|
|
|
71
75
|
}
|
|
72
76
|
return contentItemsToProcess;
|
|
73
77
|
}
|
|
74
|
-
async
|
|
78
|
+
async SetContentSourceParams(contentSource) {
|
|
75
79
|
// If content source parameters were provided, set them. Otherwise, use the default values.
|
|
76
80
|
const contentSourceParamsMap = await this.engine.getContentSourceParams(contentSource, this.contextUser);
|
|
77
81
|
if (contentSourceParamsMap) {
|
|
@@ -92,6 +96,10 @@ let AutotagLocalFileSystem = class AutotagLocalFileSystem extends AutotagBase {
|
|
|
92
96
|
};
|
|
93
97
|
return contentSourceParams;
|
|
94
98
|
}
|
|
99
|
+
/** @deprecated Use {@link SetContentSourceParams}. */
|
|
100
|
+
async setContentSourceParams(contentSource) {
|
|
101
|
+
return this.SetContentSourceParams(contentSource);
|
|
102
|
+
}
|
|
95
103
|
/**
|
|
96
104
|
* Given a content source and last run date, recursively traverse through the directory and return a
|
|
97
105
|
* list of content source items that have been modified or added after the last run date.
|
|
@@ -116,19 +124,19 @@ let AutotagLocalFileSystem = class AutotagLocalFileSystem extends AutotagBase {
|
|
|
116
124
|
const changedDate = new Date(stats.ctime.toUTCString());
|
|
117
125
|
if (changedDate > lastRunDate) {
|
|
118
126
|
// The file has been added, create a new record for this file
|
|
119
|
-
const contentItem = await this.
|
|
127
|
+
const contentItem = await this.SetAddedContentItem(filePath, contentSourceParams);
|
|
120
128
|
contentItems.push(contentItem); // Content item was added, add to list
|
|
121
129
|
}
|
|
122
130
|
else if (modifiedDate > lastRunDate) {
|
|
123
131
|
// The file's contents has been, update the record for this file
|
|
124
|
-
const contentItem = await this.
|
|
132
|
+
const contentItem = await this.SetModifiedContentItem(filePath, contentSourceParams);
|
|
125
133
|
contentItems.push(contentItem);
|
|
126
134
|
}
|
|
127
135
|
}
|
|
128
136
|
}
|
|
129
137
|
return contentItems;
|
|
130
138
|
}
|
|
131
|
-
async
|
|
139
|
+
async SetAddedContentItem(filePath, contentSourceParams) {
|
|
132
140
|
const md = this.ProviderToUse;
|
|
133
141
|
const text = await this.engine.parseFileFromPath(filePath);
|
|
134
142
|
const contentItem = await md.GetEntityObject('MJ: Content Items', this.contextUser);
|
|
@@ -149,7 +157,11 @@ let AutotagLocalFileSystem = class AutotagLocalFileSystem extends AutotagBase {
|
|
|
149
157
|
throw new Error('Failed to save content item');
|
|
150
158
|
}
|
|
151
159
|
}
|
|
152
|
-
|
|
160
|
+
/** @deprecated Use {@link SetAddedContentItem}. */
|
|
161
|
+
async setAddedContentItem(filePath, contentSourceParams) {
|
|
162
|
+
return this.SetAddedContentItem(filePath, contentSourceParams);
|
|
163
|
+
}
|
|
164
|
+
async SetModifiedContentItem(filePath, contentSourceParams) {
|
|
153
165
|
const md = this.ProviderToUse;
|
|
154
166
|
const contentItem = await md.GetEntityObject('MJ: Content Items', this.contextUser);
|
|
155
167
|
const contentItemID = await this.engine.getContentItemIDFromURL(contentSourceParams, this.contextUser);
|
|
@@ -164,6 +176,10 @@ let AutotagLocalFileSystem = class AutotagLocalFileSystem extends AutotagBase {
|
|
|
164
176
|
throw new Error('Failed to save content item');
|
|
165
177
|
}
|
|
166
178
|
}
|
|
179
|
+
/** @deprecated Use {@link SetModifiedContentItem}. */
|
|
180
|
+
async setModifiedContentItem(filePath, contentSourceParams) {
|
|
181
|
+
return this.SetModifiedContentItem(filePath, contentSourceParams);
|
|
182
|
+
}
|
|
167
183
|
};
|
|
168
184
|
AutotagLocalFileSystem = __decorate([
|
|
169
185
|
RegisterClass(AutotagBase, 'AutotagLocalFileSystem'),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AutotagLocalFileSystem.js","sourceRoot":"","sources":["../../../src/LocalFileSystem/generic/AutotagLocalFileSystem.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,WAAW,EAA2B,MAAM,YAAY,CAAC;AAClE,OAAO,EAAE,iBAAiB,EAAuB,MAAM,cAAc,CAAC;AAItE,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;AAGvB,IAAM,sBAAsB,GAA5B,MAAM,sBAAuB,SAAQ,WAAW;IAMnD;QACI,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,iBAAiB,CAAC,QAAQ,CAAC;IAC7C,CAAC;IAEM,cAAc;QACjB,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,OAAO,CAAC,WAAqB,EAAE,UAAoC,EAAE,gBAA2B,EAAE,QAA4B;QACvI,IAAI,QAAQ;YAAE,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;QACxC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,mBAAmB,CAAC,CAAC;QACzF,MAAM,cAAc,GAA4B,MAAM,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC;QACzI,MAAM,qBAAqB,GAA0B,MAAM,IAAI,CAAC,wBAAwB,CAAC,cAAc,CAAC,CAAA;QACxG,MAAM,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,qBAAqB,EAAE,IAAI,CAAC,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;QAC1H,OAAO,qBAAqB,CAAC,MAAM,CAAC;IACxC,CAAC;IAED;;;;;MAKE;IACK,KAAK,CAAC,wBAAwB,CAAC,cAAuC;QACzE,MAAM,qBAAqB,GAA0B,EAAE,CAAA;QAEvD,KAAK,MAAM,aAAa,IAAI,cAAc,EAAE,CAAC;YACzC,wCAAwC;YACxC,IAAI,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC;gBACnC,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,aAAa,CAAC,CAAC;gBAC7E,MAAM,WAAW,GAAS,MAAM,IAAI,CAAC,MAAM,CAAC,2BAA2B,CAAC,mBAAmB,CAAC,eAAe,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;gBAE9H,kDAAkD;gBAClD,IAAI,WAAW,EAAE,CAAC;oBAClB,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,mBAAmB,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;oBAC9G,IAAI,YAAY,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC1C,qBAAqB,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;oBAChD,CAAC;yBACI,CAAC;wBACF,oCAAoC;wBACpC,OAAO,CAAC,GAAG,CAAC,yDAAyD,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;oBACtG,CAAC;gBACL,CAAC;qBACI,CAAC;oBACN,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;gBACzC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACJ,OAAO,CAAC,GAAG,CAAC,0BAA0B,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC;YAChE,CAAC;QACL,CAAC;QAED,OAAO,qBAAqB,CAAC;IACjC,CAAC;IAEM,KAAK,CAAC,sBAAsB,CAAC,aAAoC;QACpE,2FAA2F;QAC3F,MAAM,sBAAsB,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,aAAa,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACzG,IAAI,sBAAsB,EAAE,CAAC;YACzB,wDAAwD;YACxD,sBAAsB,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;gBAC1C,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;oBACb,IAAY,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;gBAC/B,CAAC;YACL,CAAC,CAAC,CAAA;QACN,CAAC;QAED,MAAM,mBAAmB,GAAwB;YAC7C,eAAe,EAAE,aAAa,CAAC,EAAE;YACjC,IAAI,EAAE,aAAa,CAAC,IAAI;YACxB,aAAa,EAAE,aAAa,CAAC,aAAa;YAC1C,mBAAmB,EAAE,aAAa,CAAC,mBAAmB;YACtD,iBAAiB,EAAE,aAAa,CAAC,iBAAiB;YAClD,GAAG,EAAE,aAAa,CAAC,GAAG;SACzB,CAAA;QAED,OAAO,mBAAmB,CAAC;IAC/B,CAAC;IAED;;;;;;;OAOG;IACI,KAAK,CAAC,6BAA6B,CAAC,mBAAwC,EAAE,WAAiB,EAAE,WAAqB;QACzH,MAAM,YAAY,GAA0B,EAAE,CAAA;QAC9C,IAAI,iBAAiB,GAAG,mBAAmB,CAAC,GAAG,CAAA;QAC/C,MAAM,YAAY,GAAG,EAAE,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAA;QAEtD,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;YAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAAA;YACnD,MAAM,KAAK,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;YACnC,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACtB,mBAAmB,CAAC,GAAG,GAAG,QAAQ,CAAA;gBAClC,MAAM,IAAI,CAAC,6BAA6B,CAAC,mBAAmB,EAAE,WAAW,EAAE,WAAW,CAAC,CAAA;YAC3F,CAAC;iBAEI,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;gBACtB,MAAM,YAAY,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAA;gBACxD,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAA;gBACvD,IAAI,WAAW,GAAG,WAAW,EAAE,CAAC;oBAC5B,6DAA6D;oBAC7D,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EAAE,mBAAmB,CAAC,CAAC;oBAClF,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,sCAAsC;gBAC1E,CAAC;qBACI,IAAI,YAAY,GAAG,WAAW,EAAE,CAAC;oBAClC,iEAAiE;oBACjE,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,QAAQ,EAAE,mBAAmB,CAAC,CAAC;oBACrF,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBACnC,CAAC;YACL,CAAC;QACL,CAAC;QACD,OAAO,YAAY,CAAC;IACxB,CAAC;IAEM,KAAK,CAAC,mBAAmB,CAAC,QAAgB,EAAE,mBAAwC;QACvF,MAAM,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC;QAC9B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAC3D,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,eAAe,CAAsB,mBAAmB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACzG,WAAW,CAAC,SAAS,EAAE,CAAC;QACxB,WAAW,CAAC,eAAe,GAAG,mBAAmB,CAAC,eAAe,CAAA;QACjE,WAAW,CAAC,IAAI,GAAG,mBAAmB,CAAC,IAAI,CAAA;QAC3C,WAAW,CAAC,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAA;QACpF,WAAW,CAAC,aAAa,GAAG,mBAAmB,CAAC,aAAa,CAAA;QAC7D,WAAW,CAAC,iBAAiB,GAAG,mBAAmB,CAAC,iBAAiB,CAAA;QACrE,WAAW,CAAC,mBAAmB,GAAG,mBAAmB,CAAC,mBAAmB,CAAA;QACzE,WAAW,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAA;QAClE,WAAW,CAAC,GAAG,GAAG,mBAAmB,CAAC,GAAG,CAAA;QACzC,WAAW,CAAC,IAAI,GAAG,IAAI,CAAA;QAEvB,IAAG,MAAM,WAAW,CAAC,IAAI,EAAE,EAAC,CAAC;YACzB,OAAO,WAAW,CAAC;QACvB,CAAC;aACI,CAAC;YACF,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACnD,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,sBAAsB,CAAC,QAAgB,EAAE,mBAAwC;QAC1F,MAAM,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC;QAC9B,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,eAAe,CAAsB,mBAAmB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACzG,MAAM,aAAa,GAAW,MAAM,IAAI,CAAC,MAAM,CAAC,uBAAuB,CAAC,mBAAmB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAC/G,MAAM,WAAW,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACtC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAC3D,WAAW,CAAC,IAAI,GAAG,IAAI,CAAA;QACvB,WAAW,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAA;QAElE,IAAG,MAAM,WAAW,CAAC,IAAI,EAAE,EAAC,CAAC;YACzB,OAAO,WAAW,CAAC;QACvB,CAAC;aACI,CAAC;YACF,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACnD,CAAC;IACL,CAAC;CACJ,CAAA;
|
|
1
|
+
{"version":3,"file":"AutotagLocalFileSystem.js","sourceRoot":"","sources":["../../../src/LocalFileSystem/generic/AutotagLocalFileSystem.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,WAAW,EAA2B,MAAM,YAAY,CAAC;AAClE,OAAO,EAAE,iBAAiB,EAAuB,MAAM,cAAc,CAAC;AAItE,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;AAGvB,IAAM,sBAAsB,GAA5B,MAAM,sBAAuB,SAAQ,WAAW;IAMnD;QACI,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,iBAAiB,CAAC,QAAQ,CAAC;IAC7C,CAAC;IAEM,cAAc;QACjB,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED,8CAA8C;IACvC,cAAc;QACjB,OAAO,IAAI,CAAC,cAAc,EAAE,CAAC;IACjC,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,OAAO,CAAC,WAAqB,EAAE,UAAoC,EAAE,gBAA2B,EAAE,QAA4B;QACvI,IAAI,QAAQ;YAAE,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;QACxC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,mBAAmB,CAAC,CAAC;QACzF,MAAM,cAAc,GAA4B,MAAM,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC;QACzI,MAAM,qBAAqB,GAA0B,MAAM,IAAI,CAAC,wBAAwB,CAAC,cAAc,CAAC,CAAA;QACxG,MAAM,IAAI,CAAC,MAAM,CAAC,4BAA4B,CAAC,qBAAqB,EAAE,IAAI,CAAC,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;QAC1H,OAAO,qBAAqB,CAAC,MAAM,CAAC;IACxC,CAAC;IAED;;;;;MAKE;IACK,KAAK,CAAC,wBAAwB,CAAC,cAAuC;QACzE,MAAM,qBAAqB,GAA0B,EAAE,CAAA;QAEvD,KAAK,MAAM,aAAa,IAAI,cAAc,EAAE,CAAC;YACzC,wCAAwC;YACxC,IAAI,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC;gBACnC,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,aAAa,CAAC,CAAC;gBAC7E,MAAM,WAAW,GAAS,MAAM,IAAI,CAAC,MAAM,CAAC,2BAA2B,CAAC,mBAAmB,CAAC,eAAe,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;gBAE9H,kDAAkD;gBAClD,IAAI,WAAW,EAAE,CAAC;oBAClB,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,mBAAmB,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;oBAC9G,IAAI,YAAY,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC1C,qBAAqB,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;oBAChD,CAAC;yBACI,CAAC;wBACF,oCAAoC;wBACpC,OAAO,CAAC,GAAG,CAAC,yDAAyD,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;oBACtG,CAAC;gBACL,CAAC;qBACI,CAAC;oBACN,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;gBACzC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACJ,OAAO,CAAC,GAAG,CAAC,0BAA0B,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC;YAChE,CAAC;QACL,CAAC;QAED,OAAO,qBAAqB,CAAC;IACjC,CAAC;IAEM,KAAK,CAAC,sBAAsB,CAAC,aAAoC;QACpE,2FAA2F;QAC3F,MAAM,sBAAsB,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,aAAa,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACzG,IAAI,sBAAsB,EAAE,CAAC;YACzB,wDAAwD;YACxD,sBAAsB,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;gBAC1C,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;oBACb,IAAY,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;gBAC/B,CAAC;YACL,CAAC,CAAC,CAAA;QACN,CAAC;QAED,MAAM,mBAAmB,GAAwB;YAC7C,eAAe,EAAE,aAAa,CAAC,EAAE;YACjC,IAAI,EAAE,aAAa,CAAC,IAAI;YACxB,aAAa,EAAE,aAAa,CAAC,aAAa;YAC1C,mBAAmB,EAAE,aAAa,CAAC,mBAAmB;YACtD,iBAAiB,EAAE,aAAa,CAAC,iBAAiB;YAClD,GAAG,EAAE,aAAa,CAAC,GAAG;SACzB,CAAA;QAED,OAAO,mBAAmB,CAAC;IAC/B,CAAC;IAED,sDAAsD;IAC/C,KAAK,CAAC,sBAAsB,CAAC,aAAoC;QACpE,OAAO,IAAI,CAAC,sBAAsB,CAAC,aAAa,CAAC,CAAC;IACtD,CAAC;IAED;;;;;;;OAOG;IACI,KAAK,CAAC,6BAA6B,CAAC,mBAAwC,EAAE,WAAiB,EAAE,WAAqB;QACzH,MAAM,YAAY,GAA0B,EAAE,CAAA;QAC9C,IAAI,iBAAiB,GAAG,mBAAmB,CAAC,GAAG,CAAA;QAC/C,MAAM,YAAY,GAAG,EAAE,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAA;QAEtD,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;YAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAAA;YACnD,MAAM,KAAK,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;YACnC,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACtB,mBAAmB,CAAC,GAAG,GAAG,QAAQ,CAAA;gBAClC,MAAM,IAAI,CAAC,6BAA6B,CAAC,mBAAmB,EAAE,WAAW,EAAE,WAAW,CAAC,CAAA;YAC3F,CAAC;iBAEI,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;gBACtB,MAAM,YAAY,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAA;gBACxD,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAA;gBACvD,IAAI,WAAW,GAAG,WAAW,EAAE,CAAC;oBAC5B,6DAA6D;oBAC7D,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EAAE,mBAAmB,CAAC,CAAC;oBAClF,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,sCAAsC;gBAC1E,CAAC;qBACI,IAAI,YAAY,GAAG,WAAW,EAAE,CAAC;oBAClC,iEAAiE;oBACjE,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,QAAQ,EAAE,mBAAmB,CAAC,CAAC;oBACrF,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBACnC,CAAC;YACL,CAAC;QACL,CAAC;QACD,OAAO,YAAY,CAAC;IACxB,CAAC;IAEM,KAAK,CAAC,mBAAmB,CAAC,QAAgB,EAAE,mBAAwC;QACvF,MAAM,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC;QAC9B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAC3D,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,eAAe,CAAsB,mBAAmB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACzG,WAAW,CAAC,SAAS,EAAE,CAAC;QACxB,WAAW,CAAC,eAAe,GAAG,mBAAmB,CAAC,eAAe,CAAA;QACjE,WAAW,CAAC,IAAI,GAAG,mBAAmB,CAAC,IAAI,CAAA;QAC3C,WAAW,CAAC,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAA;QACpF,WAAW,CAAC,aAAa,GAAG,mBAAmB,CAAC,aAAa,CAAA;QAC7D,WAAW,CAAC,iBAAiB,GAAG,mBAAmB,CAAC,iBAAiB,CAAA;QACrE,WAAW,CAAC,mBAAmB,GAAG,mBAAmB,CAAC,mBAAmB,CAAA;QACzE,WAAW,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAA;QAClE,WAAW,CAAC,GAAG,GAAG,mBAAmB,CAAC,GAAG,CAAA;QACzC,WAAW,CAAC,IAAI,GAAG,IAAI,CAAA;QAEvB,IAAG,MAAM,WAAW,CAAC,IAAI,EAAE,EAAC,CAAC;YACzB,OAAO,WAAW,CAAC;QACvB,CAAC;aACI,CAAC;YACF,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACnD,CAAC;IACL,CAAC;IAED,mDAAmD;IAC5C,KAAK,CAAC,mBAAmB,CAAC,QAAgB,EAAE,mBAAwC;QACvF,OAAO,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EAAE,mBAAmB,CAAC,CAAC;IACnE,CAAC;IAEM,KAAK,CAAC,sBAAsB,CAAC,QAAgB,EAAE,mBAAwC;QAC1F,MAAM,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC;QAC9B,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,eAAe,CAAsB,mBAAmB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACzG,MAAM,aAAa,GAAW,MAAM,IAAI,CAAC,MAAM,CAAC,uBAAuB,CAAC,mBAAmB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAC/G,MAAM,WAAW,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACtC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAC3D,WAAW,CAAC,IAAI,GAAG,IAAI,CAAA;QACvB,WAAW,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAA;QAElE,IAAG,MAAM,WAAW,CAAC,IAAI,EAAE,EAAC,CAAC;YACzB,OAAO,WAAW,CAAC;QACvB,CAAC;aACI,CAAC;YACF,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACnD,CAAC;IACL,CAAC;IAED,sDAAsD;IAC/C,KAAK,CAAC,sBAAsB,CAAC,QAAgB,EAAE,mBAAwC;QAC1F,OAAO,IAAI,CAAC,sBAAsB,CAAC,QAAQ,EAAE,mBAAmB,CAAC,CAAC;IACtE,CAAC;CACJ,CAAA;AA7LY,sBAAsB;IADlC,aAAa,CAAC,WAAW,EAAE,wBAAwB,CAAC;;GACxC,sBAAsB,CA6LlC"}
|
|
@@ -24,12 +24,12 @@ export declare class AutotagRSSFeed extends AutotagBase {
|
|
|
24
24
|
* Process a single content source: parse the RSS feed, detect new/modified
|
|
25
25
|
* items, fetch full article text, and create/update ContentItems.
|
|
26
26
|
*/
|
|
27
|
-
private
|
|
27
|
+
private processContentSource;
|
|
28
28
|
/**
|
|
29
29
|
* Process a single RSS feed item: fetch full article text, compute checksum,
|
|
30
30
|
* create or update the ContentItem.
|
|
31
31
|
*/
|
|
32
|
-
private
|
|
32
|
+
private processSingleFeedItem;
|
|
33
33
|
/**
|
|
34
34
|
* Fetch the full article text for an RSS item.
|
|
35
35
|
*
|
|
@@ -38,12 +38,12 @@ export declare class AutotagRSSFeed extends AutotagBase {
|
|
|
38
38
|
* 2. Otherwise, follow the item's link URL and extract text with Cheerio
|
|
39
39
|
* 3. Fall back to the RSS description if link fetching fails
|
|
40
40
|
*/
|
|
41
|
-
private
|
|
41
|
+
private fetchArticleText;
|
|
42
42
|
/**
|
|
43
43
|
* Fetch a web page and extract its main text content using Cheerio.
|
|
44
44
|
* Strips navigation, headers, footers, scripts, and styles.
|
|
45
45
|
*/
|
|
46
|
-
private
|
|
46
|
+
private fetchAndParseWebPage;
|
|
47
47
|
/**
|
|
48
48
|
* Parse an RSS/Atom feed URL and return structured items.
|
|
49
49
|
* The content field is HTML-stripped via the engine's parseHTML.
|
|
@@ -52,10 +52,10 @@ export declare class AutotagRSSFeed extends AutotagBase {
|
|
|
52
52
|
/**
|
|
53
53
|
* Check if a URL is reachable via HTTP HEAD request.
|
|
54
54
|
*/
|
|
55
|
-
private
|
|
55
|
+
private urlIsValid;
|
|
56
56
|
/**
|
|
57
57
|
* Load existing ContentItems for this source, keyed by lowercase URL for upsert.
|
|
58
58
|
*/
|
|
59
|
-
private
|
|
59
|
+
private loadExistingContentItems;
|
|
60
60
|
}
|
|
61
61
|
//# sourceMappingURL=AutotagRSSFeed.d.ts.map
|
|
@@ -69,7 +69,7 @@ let AutotagRSSFeed = class AutotagRSSFeed extends AutotagBase {
|
|
|
69
69
|
const contentItemsToProcess = [];
|
|
70
70
|
for (const contentSource of contentSources) {
|
|
71
71
|
try {
|
|
72
|
-
const items = await this.
|
|
72
|
+
const items = await this.processContentSource(contentSource);
|
|
73
73
|
contentItemsToProcess.push(...items);
|
|
74
74
|
}
|
|
75
75
|
catch (e) {
|
|
@@ -83,7 +83,7 @@ let AutotagRSSFeed = class AutotagRSSFeed extends AutotagBase {
|
|
|
83
83
|
* Process a single content source: parse the RSS feed, detect new/modified
|
|
84
84
|
* items, fetch full article text, and create/update ContentItems.
|
|
85
85
|
*/
|
|
86
|
-
async
|
|
86
|
+
async processContentSource(contentSource) {
|
|
87
87
|
const contentSourceParams = {
|
|
88
88
|
contentSourceID: contentSource.ID,
|
|
89
89
|
name: contentSource.Name ?? '',
|
|
@@ -100,14 +100,14 @@ let AutotagRSSFeed = class AutotagRSSFeed extends AutotagBase {
|
|
|
100
100
|
}
|
|
101
101
|
LogStatus(`[RSS] Parsed ${allRSSItems.length} items from "${contentSource.Name}"`);
|
|
102
102
|
// Load existing content items for upsert by URL
|
|
103
|
-
const existingItems = await this.
|
|
103
|
+
const existingItems = await this.loadExistingContentItems(contentSourceParams.contentSourceID);
|
|
104
104
|
LogStatus(`[RSS] ${existingItems.size} existing items for "${contentSource.Name}"`);
|
|
105
105
|
const items = [];
|
|
106
106
|
for (let idx = 0; idx < allRSSItems.length; idx++) {
|
|
107
107
|
const rssItem = allRSSItems[idx];
|
|
108
108
|
try {
|
|
109
109
|
LogStatus(`[RSS] Processing item ${idx + 1}/${allRSSItems.length}: "${rssItem.title?.substring(0, 60) ?? 'untitled'}"...`);
|
|
110
|
-
const item = await this.
|
|
110
|
+
const item = await this.processSingleFeedItem(rssItem, contentSourceParams, existingItems);
|
|
111
111
|
if (item) {
|
|
112
112
|
items.push(item);
|
|
113
113
|
LogStatus(`[RSS] Item ${idx + 1} created/updated (text: ${item.Text?.length ?? 0} chars)`);
|
|
@@ -128,9 +128,9 @@ let AutotagRSSFeed = class AutotagRSSFeed extends AutotagBase {
|
|
|
128
128
|
* Process a single RSS feed item: fetch full article text, compute checksum,
|
|
129
129
|
* create or update the ContentItem.
|
|
130
130
|
*/
|
|
131
|
-
async
|
|
131
|
+
async processSingleFeedItem(rssItem, contentSourceParams, existingItems) {
|
|
132
132
|
// Fetch full article text from the link URL
|
|
133
|
-
const articleText = await this.
|
|
133
|
+
const articleText = await this.fetchArticleText(rssItem);
|
|
134
134
|
if (!articleText || articleText.trim().length === 0) {
|
|
135
135
|
return null;
|
|
136
136
|
}
|
|
@@ -176,7 +176,7 @@ let AutotagRSSFeed = class AutotagRSSFeed extends AutotagBase {
|
|
|
176
176
|
* 2. Otherwise, follow the item's link URL and extract text with Cheerio
|
|
177
177
|
* 3. Fall back to the RSS description if link fetching fails
|
|
178
178
|
*/
|
|
179
|
-
async
|
|
179
|
+
async fetchArticleText(rssItem) {
|
|
180
180
|
// 1. Prefer inline content if it's substantial (> 200 chars after HTML stripping)
|
|
181
181
|
if (rssItem.content && rssItem.content.trim().length > 200) {
|
|
182
182
|
return rssItem.content; // Already HTML-parsed by parseRSSFeed
|
|
@@ -184,7 +184,7 @@ let AutotagRSSFeed = class AutotagRSSFeed extends AutotagBase {
|
|
|
184
184
|
// 2. Follow the link URL to get the full article
|
|
185
185
|
if (rssItem.link) {
|
|
186
186
|
try {
|
|
187
|
-
const fullText = await this.
|
|
187
|
+
const fullText = await this.fetchAndParseWebPage(rssItem.link);
|
|
188
188
|
if (fullText && fullText.trim().length > 100) {
|
|
189
189
|
return fullText;
|
|
190
190
|
}
|
|
@@ -205,7 +205,7 @@ let AutotagRSSFeed = class AutotagRSSFeed extends AutotagBase {
|
|
|
205
205
|
* Fetch a web page and extract its main text content using Cheerio.
|
|
206
206
|
* Strips navigation, headers, footers, scripts, and styles.
|
|
207
207
|
*/
|
|
208
|
-
async
|
|
208
|
+
async fetchAndParseWebPage(url) {
|
|
209
209
|
const response = await HttpGet(url, {
|
|
210
210
|
Timeout: 8000,
|
|
211
211
|
ResponseType: 'text',
|
|
@@ -224,7 +224,7 @@ let AutotagRSSFeed = class AutotagRSSFeed extends AutotagBase {
|
|
|
224
224
|
* The content field is HTML-stripped via the engine's parseHTML.
|
|
225
225
|
*/
|
|
226
226
|
async ParseRSSFeed(url) {
|
|
227
|
-
if (!await this.
|
|
227
|
+
if (!await this.urlIsValid(url)) {
|
|
228
228
|
LogError(`AutotagRSSFeed: invalid feed URL: ${url}`);
|
|
229
229
|
return [];
|
|
230
230
|
}
|
|
@@ -261,7 +261,7 @@ let AutotagRSSFeed = class AutotagRSSFeed extends AutotagBase {
|
|
|
261
261
|
/**
|
|
262
262
|
* Check if a URL is reachable via HTTP HEAD request.
|
|
263
263
|
*/
|
|
264
|
-
async
|
|
264
|
+
async urlIsValid(url) {
|
|
265
265
|
try {
|
|
266
266
|
// ThrowOnError off so a 3xx/4xx is reported by status rather than as a throw.
|
|
267
267
|
const response = await HttpHead(url, { Timeout: 10000, ThrowOnError: false });
|
|
@@ -274,7 +274,7 @@ let AutotagRSSFeed = class AutotagRSSFeed extends AutotagBase {
|
|
|
274
274
|
/**
|
|
275
275
|
* Load existing ContentItems for this source, keyed by lowercase URL for upsert.
|
|
276
276
|
*/
|
|
277
|
-
async
|
|
277
|
+
async loadExistingContentItems(contentSourceID) {
|
|
278
278
|
const rv = new RunView();
|
|
279
279
|
const result = await rv.RunView({
|
|
280
280
|
EntityName: 'MJ: Content Items',
|
|
@@ -130,7 +130,11 @@ export declare class AutotagWebsite extends AutotagBase {
|
|
|
130
130
|
* legitimate pages here.
|
|
131
131
|
*/
|
|
132
132
|
protected processSingleURL(url: string, contentSourceParams: ContentSourceParams): Promise<MJContentItemEntity | null>;
|
|
133
|
+
FetchPageContent(url: string): Promise<string>;
|
|
134
|
+
/** @deprecated Use {@link FetchPageContent}. */
|
|
133
135
|
fetchPageContent(url: string): Promise<string>;
|
|
136
|
+
GetTextWithLineBreaks(element: AnyNode, $: cheerio.CheerioAPI): string;
|
|
137
|
+
/** @deprecated Use {@link GetTextWithLineBreaks}. */
|
|
134
138
|
getTextWithLineBreaks(element: AnyNode, $: cheerio.CheerioAPI): string;
|
|
135
139
|
/**
|
|
136
140
|
* Pure helper: extract clean body text from raw HTML. No IO. Exposed as
|
|
@@ -150,6 +154,11 @@ export declare class AutotagWebsite extends AutotagBase {
|
|
|
150
154
|
* extracted text is what users actually mean by "did the content
|
|
151
155
|
* change?"
|
|
152
156
|
*/
|
|
157
|
+
FetchAndExtract(url: string): Promise<{
|
|
158
|
+
text: string;
|
|
159
|
+
checksum: string;
|
|
160
|
+
}>;
|
|
161
|
+
/** @deprecated Use {@link FetchAndExtract}. */
|
|
153
162
|
fetchAndExtract(url: string): Promise<{
|
|
154
163
|
text: string;
|
|
155
164
|
checksum: string;
|
|
@@ -159,6 +168,8 @@ export declare class AutotagWebsite extends AutotagBase {
|
|
|
159
168
|
* that just want the text — internal change-detection now uses
|
|
160
169
|
* `fetchAndExtract` to avoid redundant fetches.
|
|
161
170
|
*/
|
|
171
|
+
ParseWebPage(url: string): Promise<string>;
|
|
172
|
+
/** @deprecated Use {@link ParseWebPage}. */
|
|
162
173
|
parseWebPage(url: string): Promise<string>;
|
|
163
174
|
/**
|
|
164
175
|
* Streaming variant: yields each newly-discovered URL as the crawler finds it,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AutotagWebsite.d.ts","sourceRoot":"","sources":["../../../src/Websites/generic/AutotagWebsite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,uBAAuB,EAAE,MAAM,YAAY,CAAC;AAClE,OAAO,EAAqB,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACtE,OAAO,EAAE,2BAA2B,EAAE,MAAM,oCAAoC,CAAC;AACjF,OAAO,EAAE,SAAS,EAAE,MAAM,gCAAgC,CAAC;AAE3D,OAAO,EAAE,iBAAiB,EAAE,QAAQ,EAAgC,MAAM,sBAAsB,CAAC;AACjG,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAqD,MAAM,+BAA+B,CAAC;AAC9I,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAM1C,qBACa,cAAe,SAAQ,WAAW;IAC3C,OAAO,CAAC,WAAW,CAAW;IAC9B,OAAO,CAAC,MAAM,CAAoB;IAClC,SAAS,CAAC,mBAAmB,EAAE,MAAM,CAAA;IAMrC,SAAS,CAAC,+BAA+B,EAAE,OAAO,CAAS;IAC3D,SAAS,CAAC,4BAA4B,EAAE,OAAO,CAAQ;IACvD,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAK;IAG/B,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IACtC,SAAS,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC,SAAS,CAAC,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAEnC;;;;;;;OAOG;IACH,SAAS,CAAC,eAAe,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,CAAa;;IAQ9D;;;;OAIG;IACH,SAAS,CAAC,yBAAyB,IAAI,IAAI;IAQ3C;;;;;;;;;;;OAWG;IACH;;;;;;;;OAQG;IACH,SAAS,CAAC,yBAAyB,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,2BAA2B,CAAC,GAAG,IAAI;IAsB3F,OAAO,CAAC,aAAa;IASrB,SAAS,CAAC,4BAA4B,CAAC,MAAM,EAAE,qBAAqB,GAAG,IAAI;IAU3E;;;;;;;OAOG;cACa,eAAe,CAAC,cAAc,EAAE,qBAAqB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAwBvF,OAAO,CAAC,YAAY;IAUpB;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAQxB;;;;;OAKG;IACH,SAAS,CAAC,iBAAiB,IAAI,IAAI;IAsBnC,SAAS,CAAC,cAAc,IAAI,QAAQ;IAIpC;;;;OAIG;IACU,OAAO,CAAC,WAAW,EAAE,QAAQ,EAAE,UAAU,CAAC,EAAE,uBAAuB,EAAE,gBAAgB,CAAC,EAAE,MAAM,EAAE,EAAE,QAAQ,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC;IAiD7J;;;;;;;OAOG;IACW,2BAA2B,CAAC,cAAc,EAAE,qBAAqB,EAAE,GAAG,aAAa,CAAC,mBAAmB,CAAC;IAgEtH;;;;;;;OAOG;IACU,wBAAwB,CAAC,cAAc,EAAE,qBAAqB,EAAE,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAQ9G;;;;;;;;OAQG;IACH;;;;OAIG;cACa,6BAA6B,CAAC,gBAAgB,EAAE,MAAM,EAAE,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAa1K;;;;;;;;;;OAUG;cACa,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,mBAAmB,EAAE,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC;IA8D/G,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"AutotagWebsite.d.ts","sourceRoot":"","sources":["../../../src/Websites/generic/AutotagWebsite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,uBAAuB,EAAE,MAAM,YAAY,CAAC;AAClE,OAAO,EAAqB,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACtE,OAAO,EAAE,2BAA2B,EAAE,MAAM,oCAAoC,CAAC;AACjF,OAAO,EAAE,SAAS,EAAE,MAAM,gCAAgC,CAAC;AAE3D,OAAO,EAAE,iBAAiB,EAAE,QAAQ,EAAgC,MAAM,sBAAsB,CAAC;AACjG,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAqD,MAAM,+BAA+B,CAAC;AAC9I,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAM1C,qBACa,cAAe,SAAQ,WAAW;IAC3C,OAAO,CAAC,WAAW,CAAW;IAC9B,OAAO,CAAC,MAAM,CAAoB;IAClC,SAAS,CAAC,mBAAmB,EAAE,MAAM,CAAA;IAMrC,SAAS,CAAC,+BAA+B,EAAE,OAAO,CAAS;IAC3D,SAAS,CAAC,4BAA4B,EAAE,OAAO,CAAQ;IACvD,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAK;IAG/B,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IACtC,SAAS,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC,SAAS,CAAC,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAEnC;;;;;;;OAOG;IACH,SAAS,CAAC,eAAe,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,CAAa;;IAQ9D;;;;OAIG;IACH,SAAS,CAAC,yBAAyB,IAAI,IAAI;IAQ3C;;;;;;;;;;;OAWG;IACH;;;;;;;;OAQG;IACH,SAAS,CAAC,yBAAyB,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,2BAA2B,CAAC,GAAG,IAAI;IAsB3F,OAAO,CAAC,aAAa;IASrB,SAAS,CAAC,4BAA4B,CAAC,MAAM,EAAE,qBAAqB,GAAG,IAAI;IAU3E;;;;;;;OAOG;cACa,eAAe,CAAC,cAAc,EAAE,qBAAqB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAwBvF,OAAO,CAAC,YAAY;IAUpB;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAQxB;;;;;OAKG;IACH,SAAS,CAAC,iBAAiB,IAAI,IAAI;IAsBnC,SAAS,CAAC,cAAc,IAAI,QAAQ;IAIpC;;;;OAIG;IACU,OAAO,CAAC,WAAW,EAAE,QAAQ,EAAE,UAAU,CAAC,EAAE,uBAAuB,EAAE,gBAAgB,CAAC,EAAE,MAAM,EAAE,EAAE,QAAQ,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC;IAiD7J;;;;;;;OAOG;IACW,2BAA2B,CAAC,cAAc,EAAE,qBAAqB,EAAE,GAAG,aAAa,CAAC,mBAAmB,CAAC;IAgEtH;;;;;;;OAOG;IACU,wBAAwB,CAAC,cAAc,EAAE,qBAAqB,EAAE,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAQ9G;;;;;;;;OAQG;IACH;;;;OAIG;cACa,6BAA6B,CAAC,gBAAgB,EAAE,MAAM,EAAE,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAa1K;;;;;;;;;;OAUG;cACa,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,mBAAmB,EAAE,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC;IA8D/G,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAK3D,gDAAgD;IACnC,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAIpD,qBAAqB,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,CAAC,UAAU,GAAG,MAAM;IAgB7E,qDAAqD;IAC9C,qBAAqB,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,CAAC,UAAU,GAAG,MAAM;IAI7E;;;;OAIG;IACH,SAAS,CAAC,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAOnD;;;;;;;;;;;OAWG;IACU,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IAOtF,+CAA+C;IAClC,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IAItF;;;;OAIG;IACU,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAWvD,4CAA4C;IAC/B,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAIvD;;;;;;OAMG;cACc,+BAA+B,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;IAiBpH;;OAEG;cACa,4BAA4B,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAQ5G;;;OAGG;cACc,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;IA8CxG;;;OAGG;cACa,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAK5F;;;;OAIG;IACH,SAAS,CAAC,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAW/C,SAAS,CAAC,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAW1C,SAAS,CAAC,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAa1C;;;;;;;;;;OAUG;IACH,SAAS,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;cAe5B,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAWzD;;;;;;;;;OASG;cACc,qBAAqB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;IA8CxJ;;;OAGG;cACa,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;cAQnI,KAAK,CAAC,EAAE,EAAE,MAAM;CAGnC"}
|
|
@@ -375,7 +375,7 @@ let AutotagWebsite = class AutotagWebsite extends AutotagBase {
|
|
|
375
375
|
* legitimate pages here.
|
|
376
376
|
*/
|
|
377
377
|
async processSingleURL(url, contentSourceParams) {
|
|
378
|
-
const { text, checksum: newHash } = await this.
|
|
378
|
+
const { text, checksum: newHash } = await this.FetchAndExtract(url);
|
|
379
379
|
const rv = new RunView();
|
|
380
380
|
const results = await rv.RunViews([
|
|
381
381
|
{
|
|
@@ -430,11 +430,15 @@ let AutotagWebsite = class AutotagWebsite extends AutotagBase {
|
|
|
430
430
|
}
|
|
431
431
|
return contentItem;
|
|
432
432
|
}
|
|
433
|
-
async
|
|
433
|
+
async FetchPageContent(url) {
|
|
434
434
|
const { Data: data } = await HttpGet(url, { ResponseType: 'text' });
|
|
435
435
|
return data;
|
|
436
436
|
}
|
|
437
|
-
|
|
437
|
+
/** @deprecated Use {@link FetchPageContent}. */
|
|
438
|
+
async fetchPageContent(url) {
|
|
439
|
+
return this.FetchPageContent(url);
|
|
440
|
+
}
|
|
441
|
+
GetTextWithLineBreaks(element, $) {
|
|
438
442
|
let text = '';
|
|
439
443
|
const children = $(element).contents();
|
|
440
444
|
for (let i = 0; i < children.length; i++) {
|
|
@@ -443,11 +447,15 @@ let AutotagWebsite = class AutotagWebsite extends AutotagBase {
|
|
|
443
447
|
text += $(el).text().trim() + ' ';
|
|
444
448
|
}
|
|
445
449
|
else if (el.type === 'tag') {
|
|
446
|
-
text += '\n' + this.
|
|
450
|
+
text += '\n' + this.GetTextWithLineBreaks(el, $) + '\n';
|
|
447
451
|
}
|
|
448
452
|
}
|
|
449
453
|
return text;
|
|
450
454
|
}
|
|
455
|
+
/** @deprecated Use {@link GetTextWithLineBreaks}. */
|
|
456
|
+
getTextWithLineBreaks(element, $) {
|
|
457
|
+
return this.GetTextWithLineBreaks(element, $);
|
|
458
|
+
}
|
|
451
459
|
/**
|
|
452
460
|
* Pure helper: extract clean body text from raw HTML. No IO. Exposed as
|
|
453
461
|
* a protected method so subclasses and unit tests can exercise it without
|
|
@@ -458,7 +466,7 @@ let AutotagWebsite = class AutotagWebsite extends AutotagBase {
|
|
|
458
466
|
const body = $('body')[0];
|
|
459
467
|
if (!body)
|
|
460
468
|
return '';
|
|
461
|
-
return this.
|
|
469
|
+
return this.GetTextWithLineBreaks(body, $);
|
|
462
470
|
}
|
|
463
471
|
/**
|
|
464
472
|
* Fetch a URL once, extract clean text, and compute a stable checksum
|
|
@@ -472,20 +480,24 @@ let AutotagWebsite = class AutotagWebsite extends AutotagBase {
|
|
|
472
480
|
* extracted text is what users actually mean by "did the content
|
|
473
481
|
* change?"
|
|
474
482
|
*/
|
|
475
|
-
async
|
|
483
|
+
async FetchAndExtract(url) {
|
|
476
484
|
const { Data: data } = await HttpGet(url, { ResponseType: 'text' });
|
|
477
485
|
const text = this.extractTextFromHTML(String(data));
|
|
478
486
|
const checksum = await this.engine.getChecksumFromText(text);
|
|
479
487
|
return { text, checksum };
|
|
480
488
|
}
|
|
489
|
+
/** @deprecated Use {@link FetchAndExtract}. */
|
|
490
|
+
async fetchAndExtract(url) {
|
|
491
|
+
return this.FetchAndExtract(url);
|
|
492
|
+
}
|
|
481
493
|
/**
|
|
482
494
|
* Given a URL, extracts text from a webpage. Kept for external callers
|
|
483
495
|
* that just want the text — internal change-detection now uses
|
|
484
496
|
* `fetchAndExtract` to avoid redundant fetches.
|
|
485
497
|
*/
|
|
486
|
-
async
|
|
498
|
+
async ParseWebPage(url) {
|
|
487
499
|
try {
|
|
488
|
-
const pageContent = await this.
|
|
500
|
+
const pageContent = await this.FetchPageContent(url);
|
|
489
501
|
return this.extractTextFromHTML(pageContent);
|
|
490
502
|
}
|
|
491
503
|
catch (error) {
|
|
@@ -493,6 +505,10 @@ let AutotagWebsite = class AutotagWebsite extends AutotagBase {
|
|
|
493
505
|
return '';
|
|
494
506
|
}
|
|
495
507
|
}
|
|
508
|
+
/** @deprecated Use {@link ParseWebPage}. */
|
|
509
|
+
async parseWebPage(url) {
|
|
510
|
+
return this.ParseWebPage(url);
|
|
511
|
+
}
|
|
496
512
|
/**
|
|
497
513
|
* Streaming variant: yields each newly-discovered URL as the crawler finds it,
|
|
498
514
|
* so downstream consumers (the content-item streamer that feeds the LLM
|