npm - @comunica/actor-query-source-identify-hypermedia - Versions diffs - 3.2.2 → 3.3.0 - Mend

@comunica/actor-query-source-identify-hypermedia 3.2.2 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (9) hide show

package/lib/LinkedRdfSourcesAsyncRdfIterator.d.ts CHANGED Viewed

@@ -1,6 +1,5 @@
-import type { ILink } from '@comunica/bus-rdf-resolve-hypermedia-links';
 import type { ILinkQueue } from '@comunica/bus-rdf-resolve-hypermedia-links-queue';
-import type { IQuerySource, IActionContext, MetadataBindings, IQueryBindingsOptions } from '@comunica/types';
+import type { ILink, IQuerySource, IActionContext, MetadataBindings, IQueryBindingsOptions } from '@comunica/types';
 import type * as RDF from '@rdfjs/types';
 import type { BufferedIteratorOptions } from 'asynciterator';
 import { BufferedIterator } from 'asynciterator';
@@ -35,7 +34,7 @@ export declare abstract class LinkedRdfSourcesAsyncRdfIterator extends BufferedI
      * Determine the links to be followed from the current source given its metadata.
      * @param metadata The metadata of a source.
      */
-    protected abstract getSourceLinks(metadata: Record<string, any>): Promise<ILink[]>;
+    protected abstract getSourceLinks(metadata: Record<string, any>, startSource: ISourceState): Promise<ILink[]>;
     _read(count: number, done: () => void): void;
     protected canStartNewIterator(): boolean;
     protected canStartNewIteratorConsiderReadable(): boolean;

package/lib/LinkedRdfSourcesAsyncRdfIterator.js CHANGED Viewed

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.LinkedRdfSourcesAsyncRdfIterator = void 0;
+const context_entries_1 = require("@comunica/context-entries");
 const metadata_1 = require("@comunica/metadata");
 const asynciterator_1 = require("asynciterator");
 class LinkedRdfSourcesAsyncRdfIterator extends asynciterator_1.BufferedIterator {
@@ -189,7 +190,7 @@ class LinkedRdfSourcesAsyncRdfIterator extends asynciterator_1.BufferedIterator
                         });
                     }
                     // Determine next urls, which will eventually become a next-next source.
-                    this.getSourceLinks(returnMetadata)
+                    this.getSourceLinks(returnMetadata, startSource)
                         .then((nextUrls) => Promise.all(nextUrls))
                         .then(async (nextUrls) => {
                         // Append all next URLs to our queue
@@ -239,6 +240,14 @@ class LinkedRdfSourcesAsyncRdfIterator extends asynciterator_1.BufferedIterator
                     this.iteratorsPendingCreation++;
                     this.sourceStateGetter(nextLink, handledDatasets)
                         .then((nextSourceState) => {
+                        // If we find a statistic tracking dereference events we emit the relevant data
+                        const statisticDereferenceLinks = this.context.get(context_entries_1.KeysStatistics.dereferencedLinks);
+                        if (statisticDereferenceLinks) {
+                            statisticDereferenceLinks.updateStatistic({
+                                url: nextSourceState.link.url,
+                                metadata: { ...nextSourceState.metadata, ...nextSourceState.link.metadata },
+                            }, nextSourceState.source);
+                        }
                         this.iteratorsPendingCreation--;
                         this.startIterator(nextSourceState);
                     })

package/lib/LinkedRdfSourcesAsyncRdfIterator.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"LinkedRdfSourcesAsyncRdfIterator.js","sourceRoot":"","sources":["LinkedRdfSourcesAsyncRdfIterator.ts"],"names":[],"mappings":";;;AAEA,iDAA6D;AAI7D,iDAAiD;AAGjD,MAAsB,gCAAiC,SAAQ,gCAA8B;IAkB3F,YACE,SAAiB,EACjB,SAA4B,EAC5B,oBAAuD,EACvD,OAAuB,EACvB,QAAgB,EAChB,YAAoB,EACpB,iBAAoC,EACpC,OAAiC;QAEjC,KAAK,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;QAlBhC,YAAO,GAAG,KAAK,CAAC;QACT,qBAAgB,GAAkC,EAAE,CAAC;QAC9D,6BAAwB,GAAG,CAAC,CAAC;QAC7B,gCAA2B,GAAG,CAAC,CAAC;QACxC,wDAAwD;QAChD,wBAAmB,GAA0C,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAc9F,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;QACjD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAE3C,IAAI,IAAI,CAAC,YAAY,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,mFAAmF,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;QAC1H,CAAC;IACH,CAAC;IAED;;OAEG;IACI,SAAS;QACd,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;IAEe,WAAW,CAAI,YAAoB,EAAE,QAA6B;QAChF,IAAI,YAAY,KAAK,UAAU,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACjD,0GAA0G;YAC1G,iDAAiD;YACjD,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBAC5B,IAAI,CAAC,iBAAiB,GAAG,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;oBACvD,IAAI,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC;yBAC/C,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE;wBACpB,6EAA6E;wBAC7E,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;wBACtF,cAAc,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,QAA0B,EAAE,EAAE;4BACpE,QAAQ,CAAC,KAAK,GAAG,IAAI,kCAAuB,EAAE,CAAC;4BAC/C,cAAc,CAAC,OAAO,EAAE,CAAC;4BACzB,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,QAAQ,EAAE,QAAQ,CAAC;iCACpD,IAAI,CAAC,CAAC,mBAAmB,EAAE,EAAE;gCAC5B,yDAAyD;gCACzD,MAAM,cAAc,GAAG,EAAE,GAAG,WAAW,CAAC,QAAQ,EAAE,GAAG,QAAQ,EAAE,GAAG,mBAAmB,EAAE,CAAC;gCACxF,OAAO,CAAC,cAAc,CAAC,CAAC;4BAC1B,CAAC,CAAC;iCACD,KAAK,CAAC,GAAG,EAAE;gCACV,OAAO,CAAC;oCACN,GAAG,WAAW,CAAC,QAAQ;oCACvB,KAAK,EAAE,IAAI,kCAAuB,EAAE;iCACrC,CAAC,CAAC;4BACL,CAAC,CAAC,CAAC;wBACP,CAAC,CAAC,CAAC;oBACL,CAAC,CAAC;yBACD,KAAK,CAAC,MAAM,CAAC,CAAC;gBACnB,CAAC,CAAC,CAAC;YACL,CAAC;YACD,IAAI,CAAC,iBAAiB;iBACnB,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;iBACxD,KAAK,CAAC,GAAG,EAAE;gBACV,gBAAgB;YAClB,CAAC,CAAC,CAAC;QACP,CAAC;QACD,OAAO,KAAK,CAAC,WAAW,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;IACnD,CAAC;IAEkB,IAAI,CAAC,OAAiB;QACvC,8BAA8B;QAC9B,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACvC,EAAE,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACtB,CAAC;IAce,KAAK,CAAC,KAAa,EAAE,IAAgB;QACnD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,kCAAkC;YAClC,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC7C,OAAO,KAAK,GAAG,CAAC,EAAE,CAAC;oBACjB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;oBAC7B,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;wBAClB,MAAM;oBACR,CAAC;yBAAM,CAAC;wBACN,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBACnB,CAAC;gBACH,CAAC;gBACD,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;oBACf,MAAM;gBACR,CAAC;YACH,CAAC;YAED,mCAAmC;YACnC,IAAI,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC,mBAAmB,EAAE,EAAE,CAAC;gBAC7C,gFAAgF;gBAChF,wDAAwD;gBACxD,mDAAmD;gBACnD,IAAI,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC;qBAC/C,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE;oBACpB,IAAI,CAAC,yBAAyB,CAAC,WAAW,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;oBACnE,IAAI,EAAE,CAAC;gBACT,CAAC,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACN,IAAI,EAAE,CAAC;YACT,CAAC;QACH,CAAC;aAAM,CAAC;YACN,0DAA0D;YAC1D,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YAEpB,4DAA4D;YAC5D,IAAI,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC;iBAC/C,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE;gBACpB,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;gBAChC,IAAI,EAAE,CAAC;YACT,CAAC,CAAC;gBACF,4EAA4E;iBAC3E,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAES,mBAAmB;QAC3B,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC,2BAA2B,CAAC;YACtG,IAAI,CAAC,YAAY,IAAI,CAAC,CAAC,IAAI,CAAC,mCAAmC,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACzF,CAAC;IAES,mCAAmC;QAC3C,OAAO,IAAI,CAAC;IACd,CAAC;IAES,mBAAmB;QAC3B,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC,2BAA2B,CAAC,GAAG,CAAC,CAAC;IAC/G,CAAC;IAaD;;;;OAIG;IACO,aAAa,CAAC,WAAyB;QAC/C,sDAAsD;QACtD,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAC3G,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACrC,IAAI,gBAAgB,GAAG,KAAK,CAAC;YAC7B,IAAI,gBAAgB,GAAG,KAAK,CAAC;YAE7B,+CAA+C;YACzC,QAAS,CAAC,YAAY,GAAG,IAAI,CAAC;YACpC,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;YAC5D,QAAQ,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YAClD,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACtB,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;gBAEzE,mEAAmE;gBACnE,8FAA8F;gBAC9F,gBAAgB,GAAG,IAAI,CAAC;gBACxB,IAAI,CAAC,gBAAgB,EAAE,CAAC;oBACtB,IAAI,CAAC,2BAA2B,EAAE,CAAC;gBACrC,CAAC;gBAED,yEAAyE;gBACzE,IAAI,gBAAgB,EAAE,CAAC;oBACrB,IAAI,CAAC,yBAAyB,CAAC,WAAW,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;gBACpE,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,wCAAwC;YACxC,gDAAgD;YAChD,QAAQ,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,QAA0B,EAAE,EAAE;gBAC9D,iCAAiC;gBACjC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB;qBAChD,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC,KAAK,IAAG,EAAE;oBACnC,IAAI,CAAC,gBAAgB,EAAE,CAAC;wBACtB,gBAAgB,GAAG,WAAW,CAAC,QAAQ,CAAC;oBAC1C,CAAC;oBACD,OAAO,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,QAAQ,CAAC,CAAC;gBAC7D,CAAC,CAAC,EAAE;qBACD,IAAI,CAAC,CAAC,mBAAmB,EAAE,EAAE;oBAC5B,yDAAyD;oBACzD,MAAM,cAAc,GAAG,EAAE,GAAG,WAAW,CAAC,QAAQ,EAAE,GAAG,QAAQ,EAAE,GAAG,mBAAmB,EAAE,CAAC;oBAExF,4BAA4B;oBAC5B,cAAc,CAAC,KAAK,GAAG,IAAI,kCAAuB,EAAE,CAAC;oBAErD,iEAAiE;oBACjE,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC,CAAC;oBAEpC,oCAAoC;oBACpC,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;wBAC3B,IAAI,CAAC,iBAAiB;6BACnB,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;6BACjD,KAAK,CAAC,GAAG,EAAE;4BACV,gBAAgB;wBAClB,CAAC,CAAC,CAAC;oBACP,CAAC;oBAED,wEAAwE;oBACxE,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC;yBAChC,IAAI,CAAC,CAAC,QAAiB,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;yBAClD,IAAI,CAAC,KAAK,EAAC,QAAiB,EAAE,EAAE;wBAC/B,oCAAoC;wBACpC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;wBAC5C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;4BAC/B,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC;wBAC5C,CAAC;wBAED,gBAAgB,GAAG,IAAI,CAAC;wBACxB,IAAI,gBAAgB,EAAE,CAAC;4BACrB,IAAI,CAAC,2BAA2B,EAAE,CAAC;wBACrC,CAAC;wBAED,IAAI,CAAC,yBAAyB,CAAC,WAAW,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;oBACpE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;oBAEzC,OAAO,cAAc,CAAC;gBACxB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;oBACpB,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;oBACpB,OAAyB,EAAE,CAAC;gBAC9B,CAAC,CAAC,CAAC;YACP,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,SAAkB,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO,CAAS,SAAS,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAES,cAAc,CAAC,WAA6B;QACpD,MAAM,oBAAoB,GAAG,IAAI,CAAC,WAAW,CAAmB,UAAU,CAAC,CAAC;QAC5E,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;QAC1C,oBAAoB,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC;IAC3C,CAAC;IAES,SAAS;QACjB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;IACpB,CAAC;IAED;;;;;;OAMG;IACO,yBAAyB,CAAC,eAAwC,EAAE,QAAiB;QAC7F,IAAI,CAAC,YAAY,EAAE;aAChB,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE;YAClB,2CAA2C;YAC3C,OAAO,IAAI,CAAC,mBAAmB,EAAE,IAAI,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC;gBACtD,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,EAAE,CAAC;gBACjC,IAAI,QAAQ,EAAE,CAAC;oBACb,IAAI,CAAC,wBAAwB,EAAE,CAAC;oBAChC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,eAAe,CAAC;yBAC9C,IAAI,CAAC,CAAC,eAAe,EAAE,EAAE;wBACxB,IAAI,CAAC,wBAAwB,EAAE,CAAC;wBAChC,IAAI,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC;oBACtC,CAAC,CAAC;yBACD,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC/C,CAAC;qBAAM,CAAC;oBACN,MAAM;gBACR,CAAC;YACH,CAAC;YAED,sDAAsD;YACtD,IAAI,QAAQ,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,CAAC;gBAClD,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,CAAC;QACH,CAAC,CAAC;aACD,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IACzC,CAAC;IAES,WAAW,CAAC,SAAqB,EAAE,kBAA2B;QACtE,OAAO,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC;IAC5D,CAAC;CACF;AAjUD,4EAiUC","sourcesContent":["import type { ILink } from '@comunica/bus-rdf-resolve-hypermedia-links';\nimport type { ILinkQueue } from '@comunica/bus-rdf-resolve-hypermedia-links-queue';\nimport { MetadataValidationState } from '@comunica/metadata';\nimport type { IQuerySource, IActionContext, MetadataBindings, IQueryBindingsOptions } from '@comunica/types';\nimport type * as RDF from '@rdfjs/types';\nimport type { AsyncIterator, BufferedIteratorOptions } from 'asynciterator';\nimport { BufferedIterator } from 'asynciterator';\nimport type { Algebra } from 'sparqlalgebrajs';\n\nexport abstract class LinkedRdfSourcesAsyncRdfIterator extends BufferedIterator<RDF.Bindings> {\n protected readonly operation: Algebra.Operation;\n protected readonly queryBindingsOptions: IQueryBindingsOptions \| undefined;\n protected readonly context: IActionContext;\n\n private readonly cacheSize: number;\n protected readonly firstUrl: string;\n private readonly maxIterators: number;\n private readonly sourceStateGetter: SourceStateGetter;\n\n protected started = false;\n private readonly currentIterators: AsyncIterator<RDF.Bindings>[] = [];\n private iteratorsPendingCreation = 0;\n private iteratorsPendingTermination = 0;\n // eslint-disable-next-line unicorn/no-useless-undefined\n private accumulatedMetadata: Promise<MetadataBindings \| undefined> = Promise.resolve(undefined);\n private preflightMetadata: Promise<MetadataBindings> \| undefined;\n\n public constructor(\n cacheSize: number,\n operation: Algebra.Operation,\n queryBindingsOptions: IQueryBindingsOptions \| undefined,\n context: IActionContext,\n firstUrl: string,\n maxIterators: number,\n sourceStateGetter: SourceStateGetter,\n options?: BufferedIteratorOptions,\n ) {\n super({ autoStart: false, ...options });\n this._reading = false;\n this.cacheSize = cacheSize;\n this.operation = operation;\n this.queryBindingsOptions = queryBindingsOptions;\n this.context = context;\n this.firstUrl = firstUrl;\n this.maxIterators = maxIterators;\n this.sourceStateGetter = sourceStateGetter;\n\n if (this.maxIterators <= 0) {\n throw new Error(`LinkedRdfSourcesAsyncRdfIterator.maxIterators must be larger than zero, but got ${this.maxIterators}`);\n }\n }\n\n /*\n Start filling the buffer of this iterator.\n /\n public kickstart(): void {\n if (!this.started) {\n this._fillBufferAsync();\n }\n }\n\n public override getProperty<P>(propertyName: string, callback?: (value: P) => void): P \| undefined {\n if (propertyName === 'metadata' && !this.started) {\n // If the iterator has not started yet, forcefully fetch the metadata from the source without starting the\n // iterator. This way, we keep the iterator lazy.\n if (!this.preflightMetadata) {\n this.preflightMetadata = new Promise((resolve, reject) => {\n this.sourceStateGetter({ url: this.firstUrl }, {})\n .then((sourceState) => {\n // Don't pass query options, as we don't want to consume any passed iterators\n const bindingsStream = sourceState.source.queryBindings(this.operation, this.context);\n bindingsStream.getProperty('metadata', (metadata: MetadataBindings) => {\n metadata.state = new MetadataValidationState();\n bindingsStream.destroy();\n this.accumulateMetadata(sourceState.metadata, metadata)\n .then((accumulatedMetadata) => {\n // Also merge fields that were not explicitly accumulated\n const returnMetadata = { ...sourceState.metadata, ...metadata, ...accumulatedMetadata };\n resolve(returnMetadata);\n })\n .catch(() => {\n resolve({\n ...sourceState.metadata,\n state: new MetadataValidationState(),\n });\n });\n });\n })\n .catch(reject);\n });\n }\n this.preflightMetadata\n .then(metadata => this.setProperty('metadata', metadata))\n .catch(() => {\n // Ignore errors\n });\n }\n return super.getProperty(propertyName, callback);\n }\n\n protected override _end(destroy?: boolean): void {\n // Close all running iterators\n for (const it of this.currentIterators) {\n it.destroy();\n }\n\n super._end(destroy);\n }\n\n /\n Get the internal link queue.\n * The returned instance must always be the same.\n /\n public abstract getLinkQueue(): Promise<ILinkQueue>;\n\n /\n Determine the links to be followed from the current source given its metadata.\n * @param metadata The metadata of a source.\n /\n protected abstract getSourceLinks(metadata: Record<string, any>): Promise<ILink[]>;\n\n public override _read(count: number, done: () => void): void {\n if (this.started) {\n // Read from all current iterators\n for (const iterator of this.currentIterators) {\n while (count > 0) {\n const read = iterator.read();\n if (read === null) {\n break;\n } else {\n count--;\n this._push(read);\n }\n }\n if (count <= 0) {\n break;\n }\n }\n\n // Schedule new iterators if needed\n if (count >= 0 && this.canStartNewIterator()) {\n // We can safely ignore skip catching the error, since we are guaranteed to have\n // successfully got the source for this.firstUrl before.\n // eslint-disable-next-line ts/no-floating-promises\n this.sourceStateGetter({ url: this.firstUrl }, {})\n .then((sourceState) => {\n this.startIteratorsForNextUrls(sourceState.handledDatasets, false);\n done();\n });\n } else {\n done();\n }\n } else {\n // The first time this is called, prepare the first source\n this.started = true;\n\n // Await the source to be set, and start the source iterator\n this.sourceStateGetter({ url: this.firstUrl }, {})\n .then((sourceState) => {\n this.startIterator(sourceState);\n done();\n })\n // Destroy should be async because it can be called before it is listened to\n .catch(error => setTimeout(() => this.destroy(error)));\n }\n }\n\n protected canStartNewIterator(): boolean {\n return (this.currentIterators.length + this.iteratorsPendingCreation + this.iteratorsPendingTermination) <\n this.maxIterators && (!this.canStartNewIteratorConsiderReadable() \|\| !this.readable);\n }\n\n protected canStartNewIteratorConsiderReadable(): boolean {\n return true;\n }\n\n protected areIteratorsRunning(): boolean {\n return (this.currentIterators.length + this.iteratorsPendingCreation + this.iteratorsPendingTermination) > 0;\n }\n\n /\n Append the fields from appendingMetadata into accumulatedMetadata.\n * @param accumulatedMetadata The fields to append to.\n * @param appendingMetadata The fields to append.\n * @protected\n /\n protected abstract accumulateMetadata(\n accumulatedMetadata: MetadataBindings,\n appendingMetadata: MetadataBindings,\n ): Promise<MetadataBindings>;\n\n /\n Start a new iterator for the given source.\n * Once the iterator is done, it will either determine a new source, or it will close the linked iterator.\n * @param {ISourceState} startSource The start source state.\n /\n protected startIterator(startSource: ISourceState): void {\n // Delegate the quad pattern query to the given source\n try {\n const iterator = startSource.source.queryBindings(this.operation, this.context, this.queryBindingsOptions);\n this.currentIterators.push(iterator);\n let receivedEndEvent = false;\n let receivedMetadata = false;\n\n // Attach readers to the newly created iterator\n (<any>iterator)._destination = this;\n iterator.on('error', (error: Error) => this.destroy(error));\n iterator.on('readable', () => this._fillBuffer());\n iterator.on('end', () => {\n this.currentIterators.splice(this.currentIterators.indexOf(iterator), 1);\n\n // Indicate that this iterator still needs to flush its next-links.\n // Without this, the linked iterator could sometimes be closed before next-links are obtained.\n receivedEndEvent = true;\n if (!receivedMetadata) {\n this.iteratorsPendingTermination++;\n }\n\n // If the metadata was already received, handle the next URL in the queue\n if (receivedMetadata) {\n this.startIteratorsForNextUrls(startSource.handledDatasets, true);\n }\n });\n\n // Listen for the metadata of the source\n // The metadata property is guaranteed to be set\n iterator.getProperty('metadata', (metadata: MetadataBindings) => {\n // Accumulate the metadata object\n this.accumulatedMetadata = this.accumulatedMetadata\n .then(previousMetadata => (async() => {\n if (!previousMetadata) {\n previousMetadata = startSource.metadata;\n }\n return this.accumulateMetadata(previousMetadata, metadata);\n })()\n .then((accumulatedMetadata) => {\n // Also merge fields that were not explicitly accumulated\n const returnMetadata = { ...startSource.metadata, ...metadata, ...accumulatedMetadata };\n\n // Create new metadata state\n returnMetadata.state = new MetadataValidationState();\n\n // Emit metadata, and invalidate any metadata that was set before\n this.updateMetadata(returnMetadata);\n\n // Invalidate any preflight metadata\n if (this.preflightMetadata) {\n this.preflightMetadata\n .then(metadataIn => metadataIn.state.invalidate())\n .catch(() => {\n // Ignore errors\n });\n }\n\n // Determine next urls, which will eventually become a next-next source.\n this.getSourceLinks(returnMetadata)\n .then((nextUrls: ILink[]) => Promise.all(nextUrls))\n .then(async(nextUrls: ILink[]) => {\n // Append all next URLs to our queue\n const linkQueue = await this.getLinkQueue();\n for (const nextUrl of nextUrls) {\n linkQueue.push(nextUrl, startSource.link);\n }\n\n receivedMetadata = true;\n if (receivedEndEvent) {\n this.iteratorsPendingTermination--;\n }\n\n this.startIteratorsForNextUrls(startSource.handledDatasets, true);\n }).catch(error => this.destroy(error));\n\n return returnMetadata;\n })).catch((error) => {\n this.destroy(error);\n return <MetadataBindings>{};\n });\n });\n } catch (syncError: unknown) {\n this.destroy(<Error> syncError);\n }\n }\n\n protected updateMetadata(metadataNew: MetadataBindings): void {\n const metadataToInvalidate = this.getProperty<MetadataBindings>('metadata');\n this.setProperty('metadata', metadataNew);\n metadataToInvalidate?.state.invalidate();\n }\n\n protected isRunning(): boolean {\n return !this.done;\n }\n\n /\n Check if a next URL is in the queue.\n * If yes, start a new iterator.\n * If no, close this iterator.\n * @param handledDatasets\n * @param canClose\n /\n protected startIteratorsForNextUrls(handledDatasets: Record<string, boolean>, canClose: boolean): void {\n this.getLinkQueue()\n .then((linkQueue) => {\n // Create as many new iterators as possible\n while (this.canStartNewIterator() && this.isRunning()) {\n const nextLink = linkQueue.pop();\n if (nextLink) {\n this.iteratorsPendingCreation++;\n this.sourceStateGetter(nextLink, handledDatasets)\n .then((nextSourceState) => {\n this.iteratorsPendingCreation--;\n this.startIterator(nextSourceState);\n })\n .catch(error => this.emit('error', error));\n } else {\n break;\n }\n }\n\n // Close, only if no other iterators are still running\n if (canClose && this.isCloseable(linkQueue, true)) {\n this.close();\n }\n })\n .catch(error => this.destroy(error));\n }\n\n protected isCloseable(linkQueue: ILinkQueue, _requireQueueEmpty: boolean): boolean {\n return linkQueue.isEmpty() && !this.areIteratorsRunning();\n }\n}\n\n/\n The current state of a source.\n * This is needed for following links within a source.\n /\nexport interface ISourceState {\n /\n The link to this source.\n /\n link: ILink;\n /\n A source.\n /\n source: IQuerySource;\n /\n The source's initial metadata.\n /\n metadata: MetadataBindings;\n /\n All dataset identifiers that have been passed for this source.\n */\n handledDatasets: Record<string, boolean>;\n}\n\nexport type SourceStateGetter = (link: ILink, handledDatasets: Record<string, boolean>) => Promise<ISourceState>;\n"]}
1	+ {"version":3,"file":"LinkedRdfSourcesAsyncRdfIterator.js","sourceRoot":"","sources":["LinkedRdfSourcesAsyncRdfIterator.ts"],"names":[],"mappings":";;;AACA,+DAA2D;AAC3D,iDAA6D;AAW7D,iDAAiD;AAGjD,MAAsB,gCAAiC,SAAQ,gCAA8B;IAkB3F,YACE,SAAiB,EACjB,SAA4B,EAC5B,oBAAuD,EACvD,OAAuB,EACvB,QAAgB,EAChB,YAAoB,EACpB,iBAAoC,EACpC,OAAiC;QAEjC,KAAK,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;QAlBhC,YAAO,GAAG,KAAK,CAAC;QACT,qBAAgB,GAAkC,EAAE,CAAC;QAC9D,6BAAwB,GAAG,CAAC,CAAC;QAC7B,gCAA2B,GAAG,CAAC,CAAC;QACxC,wDAAwD;QAChD,wBAAmB,GAA0C,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAc9F,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;QACjD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAE3C,IAAI,IAAI,CAAC,YAAY,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,mFAAmF,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;QAC1H,CAAC;IACH,CAAC;IAED;;OAEG;IACI,SAAS;QACd,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;IAEe,WAAW,CAAI,YAAoB,EAAE,QAA6B;QAChF,IAAI,YAAY,KAAK,UAAU,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACjD,0GAA0G;YAC1G,iDAAiD;YACjD,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBAC5B,IAAI,CAAC,iBAAiB,GAAG,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;oBACvD,IAAI,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC;yBAC/C,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE;wBACpB,6EAA6E;wBAC7E,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;wBACtF,cAAc,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,QAA0B,EAAE,EAAE;4BACpE,QAAQ,CAAC,KAAK,GAAG,IAAI,kCAAuB,EAAE,CAAC;4BAC/C,cAAc,CAAC,OAAO,EAAE,CAAC;4BACzB,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,QAAQ,EAAE,QAAQ,CAAC;iCACpD,IAAI,CAAC,CAAC,mBAAmB,EAAE,EAAE;gCAC5B,yDAAyD;gCACzD,MAAM,cAAc,GAAG,EAAE,GAAG,WAAW,CAAC,QAAQ,EAAE,GAAG,QAAQ,EAAE,GAAG,mBAAmB,EAAE,CAAC;gCACxF,OAAO,CAAC,cAAc,CAAC,CAAC;4BAC1B,CAAC,CAAC;iCACD,KAAK,CAAC,GAAG,EAAE;gCACV,OAAO,CAAC;oCACN,GAAG,WAAW,CAAC,QAAQ;oCACvB,KAAK,EAAE,IAAI,kCAAuB,EAAE;iCACrC,CAAC,CAAC;4BACL,CAAC,CAAC,CAAC;wBACP,CAAC,CAAC,CAAC;oBACL,CAAC,CAAC;yBACD,KAAK,CAAC,MAAM,CAAC,CAAC;gBACnB,CAAC,CAAC,CAAC;YACL,CAAC;YACD,IAAI,CAAC,iBAAiB;iBACnB,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;iBACxD,KAAK,CAAC,GAAG,EAAE;gBACV,gBAAgB;YAClB,CAAC,CAAC,CAAC;QACP,CAAC;QACD,OAAO,KAAK,CAAC,WAAW,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;IACnD,CAAC;IAEkB,IAAI,CAAC,OAAiB;QACvC,8BAA8B;QAC9B,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACvC,EAAE,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACtB,CAAC;IAce,KAAK,CAAC,KAAa,EAAE,IAAgB;QACnD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,kCAAkC;YAClC,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC7C,OAAO,KAAK,GAAG,CAAC,EAAE,CAAC;oBACjB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;oBAC7B,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;wBAClB,MAAM;oBACR,CAAC;yBAAM,CAAC;wBACN,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBACnB,CAAC;gBACH,CAAC;gBACD,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;oBACf,MAAM;gBACR,CAAC;YACH,CAAC;YAED,mCAAmC;YACnC,IAAI,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC,mBAAmB,EAAE,EAAE,CAAC;gBAC7C,gFAAgF;gBAChF,wDAAwD;gBACxD,mDAAmD;gBACnD,IAAI,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC;qBAC/C,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE;oBACpB,IAAI,CAAC,yBAAyB,CAAC,WAAW,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;oBACnE,IAAI,EAAE,CAAC;gBACT,CAAC,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACN,IAAI,EAAE,CAAC;YACT,CAAC;QACH,CAAC;aAAM,CAAC;YACN,0DAA0D;YAC1D,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YAEpB,4DAA4D;YAC5D,IAAI,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC;iBAC/C,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE;gBACpB,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;gBAChC,IAAI,EAAE,CAAC;YACT,CAAC,CAAC;gBACF,4EAA4E;iBAC3E,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAES,mBAAmB;QAC3B,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC,2BAA2B,CAAC;YACtG,IAAI,CAAC,YAAY,IAAI,CAAC,CAAC,IAAI,CAAC,mCAAmC,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACzF,CAAC;IAES,mCAAmC;QAC3C,OAAO,IAAI,CAAC;IACd,CAAC;IAES,mBAAmB;QAC3B,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC,2BAA2B,CAAC,GAAG,CAAC,CAAC;IAC/G,CAAC;IAaD;;;;OAIG;IACO,aAAa,CAAC,WAAyB;QAC/C,sDAAsD;QACtD,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAC3G,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACrC,IAAI,gBAAgB,GAAG,KAAK,CAAC;YAC7B,IAAI,gBAAgB,GAAG,KAAK,CAAC;YAE7B,+CAA+C;YACzC,QAAS,CAAC,YAAY,GAAG,IAAI,CAAC;YACpC,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;YAC5D,QAAQ,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YAClD,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACtB,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;gBAEzE,mEAAmE;gBACnE,8FAA8F;gBAC9F,gBAAgB,GAAG,IAAI,CAAC;gBACxB,IAAI,CAAC,gBAAgB,EAAE,CAAC;oBACtB,IAAI,CAAC,2BAA2B,EAAE,CAAC;gBACrC,CAAC;gBAED,yEAAyE;gBACzE,IAAI,gBAAgB,EAAE,CAAC;oBACrB,IAAI,CAAC,yBAAyB,CAAC,WAAW,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;gBACpE,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,wCAAwC;YACxC,gDAAgD;YAChD,QAAQ,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,QAA0B,EAAE,EAAE;gBAC9D,iCAAiC;gBACjC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB;qBAChD,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC,KAAK,IAAG,EAAE;oBACnC,IAAI,CAAC,gBAAgB,EAAE,CAAC;wBACtB,gBAAgB,GAAG,WAAW,CAAC,QAAQ,CAAC;oBAC1C,CAAC;oBACD,OAAO,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,QAAQ,CAAC,CAAC;gBAC7D,CAAC,CAAC,EAAE;qBACD,IAAI,CAAC,CAAC,mBAAmB,EAAE,EAAE;oBAC5B,yDAAyD;oBACzD,MAAM,cAAc,GAAG,EAAE,GAAG,WAAW,CAAC,QAAQ,EAAE,GAAG,QAAQ,EAAE,GAAG,mBAAmB,EAAE,CAAC;oBAExF,4BAA4B;oBAC5B,cAAc,CAAC,KAAK,GAAG,IAAI,kCAAuB,EAAE,CAAC;oBAErD,iEAAiE;oBACjE,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC,CAAC;oBAEpC,oCAAoC;oBACpC,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;wBAC3B,IAAI,CAAC,iBAAiB;6BACnB,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;6BACjD,KAAK,CAAC,GAAG,EAAE;4BACV,gBAAgB;wBAClB,CAAC,CAAC,CAAC;oBACP,CAAC;oBAED,wEAAwE;oBACxE,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,WAAW,CAAC;yBAC7C,IAAI,CAAC,CAAC,QAAiB,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;yBAClD,IAAI,CAAC,KAAK,EAAC,QAAiB,EAAE,EAAE;wBAC/B,oCAAoC;wBACpC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;wBAC5C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;4BAC/B,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC;wBAC5C,CAAC;wBAED,gBAAgB,GAAG,IAAI,CAAC;wBACxB,IAAI,gBAAgB,EAAE,CAAC;4BACrB,IAAI,CAAC,2BAA2B,EAAE,CAAC;wBACrC,CAAC;wBAED,IAAI,CAAC,yBAAyB,CAAC,WAAW,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;oBACpE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;oBAEzC,OAAO,cAAc,CAAC;gBACxB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;oBACpB,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;oBACpB,OAAyB,EAAE,CAAC;gBAC9B,CAAC,CAAC,CAAC;YACP,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,SAAkB,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO,CAAS,SAAS,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAES,cAAc,CAAC,WAA6B;QACpD,MAAM,oBAAoB,GAAG,IAAI,CAAC,WAAW,CAAmB,UAAU,CAAC,CAAC;QAC5E,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;QAC1C,oBAAoB,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC;IAC3C,CAAC;IAES,SAAS;QACjB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;IACpB,CAAC;IAED;;;;;;OAMG;IACO,yBAAyB,CAAC,eAAwC,EAAE,QAAiB;QAC7F,IAAI,CAAC,YAAY,EAAE;aAChB,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE;YAClB,2CAA2C;YAC3C,OAAO,IAAI,CAAC,mBAAmB,EAAE,IAAI,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC;gBACtD,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,EAAE,CAAC;gBACjC,IAAI,QAAQ,EAAE,CAAC;oBACb,IAAI,CAAC,wBAAwB,EAAE,CAAC;oBAChC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,eAAe,CAAC;yBAC9C,IAAI,CAAC,CAAC,eAAe,EAAE,EAAE;wBACxB,+EAA+E;wBAC/E,MAAM,yBAAyB,GAAsC,IAAI,CAAC,OAAO,CAAC,GAAG,CACnF,gCAAc,CAAC,iBAAiB,CACjC,CAAC;wBACF,IAAI,yBAAyB,EAAE,CAAC;4BAC9B,yBAAyB,CAAC,eAAe,CACvC;gCACE,GAAG,EAAE,eAAe,CAAC,IAAI,CAAC,GAAG;gCAC7B,QAAQ,EAAE,EAAE,GAAG,eAAe,CAAC,QAAQ,EAAE,GAAG,eAAe,CAAC,IAAI,CAAC,QAAQ,EAAE;6BAC5E,EACD,eAAe,CAAC,MAAM,CACvB,CAAC;wBACJ,CAAC;wBAED,IAAI,CAAC,wBAAwB,EAAE,CAAC;wBAChC,IAAI,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC;oBACtC,CAAC,CAAC;yBACD,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC/C,CAAC;qBAAM,CAAC;oBACN,MAAM;gBACR,CAAC;YACH,CAAC;YAED,sDAAsD;YACtD,IAAI,QAAQ,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,CAAC;gBAClD,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,CAAC;QACH,CAAC,CAAC;aACD,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IACzC,CAAC;IAES,WAAW,CAAC,SAAqB,EAAE,kBAA2B;QACtE,OAAO,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC;IAC5D,CAAC;CACF;AA/UD,4EA+UC","sourcesContent":["import type { ILinkQueue } from '@comunica/bus-rdf-resolve-hypermedia-links-queue';\nimport { KeysStatistics } from '@comunica/context-entries';\nimport { MetadataValidationState } from '@comunica/metadata';\nimport type {\n ILink,\n IQuerySource,\n IActionContext,\n MetadataBindings,\n IQueryBindingsOptions,\n IStatisticBase,\n} from '@comunica/types';\nimport type * as RDF from '@rdfjs/types';\nimport type { AsyncIterator, BufferedIteratorOptions } from 'asynciterator';\nimport { BufferedIterator } from 'asynciterator';\nimport type { Algebra } from 'sparqlalgebrajs';\n\nexport abstract class LinkedRdfSourcesAsyncRdfIterator extends BufferedIterator<RDF.Bindings> {\n protected readonly operation: Algebra.Operation;\n protected readonly queryBindingsOptions: IQueryBindingsOptions \| undefined;\n protected readonly context: IActionContext;\n\n private readonly cacheSize: number;\n protected readonly firstUrl: string;\n private readonly maxIterators: number;\n private readonly sourceStateGetter: SourceStateGetter;\n\n protected started = false;\n private readonly currentIterators: AsyncIterator<RDF.Bindings>[] = [];\n private iteratorsPendingCreation = 0;\n private iteratorsPendingTermination = 0;\n // eslint-disable-next-line unicorn/no-useless-undefined\n private accumulatedMetadata: Promise<MetadataBindings \| undefined> = Promise.resolve(undefined);\n private preflightMetadata: Promise<MetadataBindings> \| undefined;\n\n public constructor(\n cacheSize: number,\n operation: Algebra.Operation,\n queryBindingsOptions: IQueryBindingsOptions \| undefined,\n context: IActionContext,\n firstUrl: string,\n maxIterators: number,\n sourceStateGetter: SourceStateGetter,\n options?: BufferedIteratorOptions,\n ) {\n super({ autoStart: false, ...options });\n this._reading = false;\n this.cacheSize = cacheSize;\n this.operation = operation;\n this.queryBindingsOptions = queryBindingsOptions;\n this.context = context;\n this.firstUrl = firstUrl;\n this.maxIterators = maxIterators;\n this.sourceStateGetter = sourceStateGetter;\n\n if (this.maxIterators <= 0) {\n throw new Error(`LinkedRdfSourcesAsyncRdfIterator.maxIterators must be larger than zero, but got ${this.maxIterators}`);\n }\n }\n\n /*\n Start filling the buffer of this iterator.\n /\n public kickstart(): void {\n if (!this.started) {\n this._fillBufferAsync();\n }\n }\n\n public override getProperty<P>(propertyName: string, callback?: (value: P) => void): P \| undefined {\n if (propertyName === 'metadata' && !this.started) {\n // If the iterator has not started yet, forcefully fetch the metadata from the source without starting the\n // iterator. This way, we keep the iterator lazy.\n if (!this.preflightMetadata) {\n this.preflightMetadata = new Promise((resolve, reject) => {\n this.sourceStateGetter({ url: this.firstUrl }, {})\n .then((sourceState) => {\n // Don't pass query options, as we don't want to consume any passed iterators\n const bindingsStream = sourceState.source.queryBindings(this.operation, this.context);\n bindingsStream.getProperty('metadata', (metadata: MetadataBindings) => {\n metadata.state = new MetadataValidationState();\n bindingsStream.destroy();\n this.accumulateMetadata(sourceState.metadata, metadata)\n .then((accumulatedMetadata) => {\n // Also merge fields that were not explicitly accumulated\n const returnMetadata = { ...sourceState.metadata, ...metadata, ...accumulatedMetadata };\n resolve(returnMetadata);\n })\n .catch(() => {\n resolve({\n ...sourceState.metadata,\n state: new MetadataValidationState(),\n });\n });\n });\n })\n .catch(reject);\n });\n }\n this.preflightMetadata\n .then(metadata => this.setProperty('metadata', metadata))\n .catch(() => {\n // Ignore errors\n });\n }\n return super.getProperty(propertyName, callback);\n }\n\n protected override _end(destroy?: boolean): void {\n // Close all running iterators\n for (const it of this.currentIterators) {\n it.destroy();\n }\n\n super._end(destroy);\n }\n\n /\n Get the internal link queue.\n * The returned instance must always be the same.\n /\n public abstract getLinkQueue(): Promise<ILinkQueue>;\n\n /\n Determine the links to be followed from the current source given its metadata.\n * @param metadata The metadata of a source.\n /\n protected abstract getSourceLinks(metadata: Record<string, any>, startSource: ISourceState): Promise<ILink[]>;\n\n public override _read(count: number, done: () => void): void {\n if (this.started) {\n // Read from all current iterators\n for (const iterator of this.currentIterators) {\n while (count > 0) {\n const read = iterator.read();\n if (read === null) {\n break;\n } else {\n count--;\n this._push(read);\n }\n }\n if (count <= 0) {\n break;\n }\n }\n\n // Schedule new iterators if needed\n if (count >= 0 && this.canStartNewIterator()) {\n // We can safely ignore skip catching the error, since we are guaranteed to have\n // successfully got the source for this.firstUrl before.\n // eslint-disable-next-line ts/no-floating-promises\n this.sourceStateGetter({ url: this.firstUrl }, {})\n .then((sourceState) => {\n this.startIteratorsForNextUrls(sourceState.handledDatasets, false);\n done();\n });\n } else {\n done();\n }\n } else {\n // The first time this is called, prepare the first source\n this.started = true;\n\n // Await the source to be set, and start the source iterator\n this.sourceStateGetter({ url: this.firstUrl }, {})\n .then((sourceState) => {\n this.startIterator(sourceState);\n done();\n })\n // Destroy should be async because it can be called before it is listened to\n .catch(error => setTimeout(() => this.destroy(error)));\n }\n }\n\n protected canStartNewIterator(): boolean {\n return (this.currentIterators.length + this.iteratorsPendingCreation + this.iteratorsPendingTermination) <\n this.maxIterators && (!this.canStartNewIteratorConsiderReadable() \|\| !this.readable);\n }\n\n protected canStartNewIteratorConsiderReadable(): boolean {\n return true;\n }\n\n protected areIteratorsRunning(): boolean {\n return (this.currentIterators.length + this.iteratorsPendingCreation + this.iteratorsPendingTermination) > 0;\n }\n\n /\n Append the fields from appendingMetadata into accumulatedMetadata.\n * @param accumulatedMetadata The fields to append to.\n * @param appendingMetadata The fields to append.\n * @protected\n /\n protected abstract accumulateMetadata(\n accumulatedMetadata: MetadataBindings,\n appendingMetadata: MetadataBindings,\n ): Promise<MetadataBindings>;\n\n /\n Start a new iterator for the given source.\n * Once the iterator is done, it will either determine a new source, or it will close the linked iterator.\n * @param {ISourceState} startSource The start source state.\n /\n protected startIterator(startSource: ISourceState): void {\n // Delegate the quad pattern query to the given source\n try {\n const iterator = startSource.source.queryBindings(this.operation, this.context, this.queryBindingsOptions);\n this.currentIterators.push(iterator);\n let receivedEndEvent = false;\n let receivedMetadata = false;\n\n // Attach readers to the newly created iterator\n (<any>iterator)._destination = this;\n iterator.on('error', (error: Error) => this.destroy(error));\n iterator.on('readable', () => this._fillBuffer());\n iterator.on('end', () => {\n this.currentIterators.splice(this.currentIterators.indexOf(iterator), 1);\n\n // Indicate that this iterator still needs to flush its next-links.\n // Without this, the linked iterator could sometimes be closed before next-links are obtained.\n receivedEndEvent = true;\n if (!receivedMetadata) {\n this.iteratorsPendingTermination++;\n }\n\n // If the metadata was already received, handle the next URL in the queue\n if (receivedMetadata) {\n this.startIteratorsForNextUrls(startSource.handledDatasets, true);\n }\n });\n\n // Listen for the metadata of the source\n // The metadata property is guaranteed to be set\n iterator.getProperty('metadata', (metadata: MetadataBindings) => {\n // Accumulate the metadata object\n this.accumulatedMetadata = this.accumulatedMetadata\n .then(previousMetadata => (async() => {\n if (!previousMetadata) {\n previousMetadata = startSource.metadata;\n }\n return this.accumulateMetadata(previousMetadata, metadata);\n })()\n .then((accumulatedMetadata) => {\n // Also merge fields that were not explicitly accumulated\n const returnMetadata = { ...startSource.metadata, ...metadata, ...accumulatedMetadata };\n\n // Create new metadata state\n returnMetadata.state = new MetadataValidationState();\n\n // Emit metadata, and invalidate any metadata that was set before\n this.updateMetadata(returnMetadata);\n\n // Invalidate any preflight metadata\n if (this.preflightMetadata) {\n this.preflightMetadata\n .then(metadataIn => metadataIn.state.invalidate())\n .catch(() => {\n // Ignore errors\n });\n }\n\n // Determine next urls, which will eventually become a next-next source.\n this.getSourceLinks(returnMetadata, startSource)\n .then((nextUrls: ILink[]) => Promise.all(nextUrls))\n .then(async(nextUrls: ILink[]) => {\n // Append all next URLs to our queue\n const linkQueue = await this.getLinkQueue();\n for (const nextUrl of nextUrls) {\n linkQueue.push(nextUrl, startSource.link);\n }\n\n receivedMetadata = true;\n if (receivedEndEvent) {\n this.iteratorsPendingTermination--;\n }\n\n this.startIteratorsForNextUrls(startSource.handledDatasets, true);\n }).catch(error => this.destroy(error));\n\n return returnMetadata;\n })).catch((error) => {\n this.destroy(error);\n return <MetadataBindings>{};\n });\n });\n } catch (syncError: unknown) {\n this.destroy(<Error> syncError);\n }\n }\n\n protected updateMetadata(metadataNew: MetadataBindings): void {\n const metadataToInvalidate = this.getProperty<MetadataBindings>('metadata');\n this.setProperty('metadata', metadataNew);\n metadataToInvalidate?.state.invalidate();\n }\n\n protected isRunning(): boolean {\n return !this.done;\n }\n\n /\n Check if a next URL is in the queue.\n * If yes, start a new iterator.\n * If no, close this iterator.\n * @param handledDatasets\n * @param canClose\n /\n protected startIteratorsForNextUrls(handledDatasets: Record<string, boolean>, canClose: boolean): void {\n this.getLinkQueue()\n .then((linkQueue) => {\n // Create as many new iterators as possible\n while (this.canStartNewIterator() && this.isRunning()) {\n const nextLink = linkQueue.pop();\n if (nextLink) {\n this.iteratorsPendingCreation++;\n this.sourceStateGetter(nextLink, handledDatasets)\n .then((nextSourceState) => {\n // If we find a statistic tracking dereference events we emit the relevant data\n const statisticDereferenceLinks: IStatisticBase<ILink> \| undefined = this.context.get(\n KeysStatistics.dereferencedLinks,\n );\n if (statisticDereferenceLinks) {\n statisticDereferenceLinks.updateStatistic(\n {\n url: nextSourceState.link.url,\n metadata: { ...nextSourceState.metadata, ...nextSourceState.link.metadata },\n },\n nextSourceState.source,\n );\n }\n\n this.iteratorsPendingCreation--;\n this.startIterator(nextSourceState);\n })\n .catch(error => this.emit('error', error));\n } else {\n break;\n }\n }\n\n // Close, only if no other iterators are still running\n if (canClose && this.isCloseable(linkQueue, true)) {\n this.close();\n }\n })\n .catch(error => this.destroy(error));\n }\n\n protected isCloseable(linkQueue: ILinkQueue, _requireQueueEmpty: boolean): boolean {\n return linkQueue.isEmpty() && !this.areIteratorsRunning();\n }\n}\n\n/\n The current state of a source.\n * This is needed for following links within a source.\n /\nexport interface ISourceState {\n /\n The link to this source.\n /\n link: ILink;\n /\n A source.\n /\n source: IQuerySource;\n /\n The source's initial metadata.\n /\n metadata: MetadataBindings;\n /\n All dataset identifiers that have been passed for this source.\n */\n handledDatasets: Record<string, boolean>;\n}\n\nexport type SourceStateGetter = (link: ILink, handledDatasets: Record<string, boolean>) => Promise<ISourceState>;\n"]}

package/lib/MediatedLinkedRdfSourcesAsyncRdfIterator.d.ts CHANGED Viewed

@@ -1,7 +1,7 @@
 import type { MediatorRdfMetadataAccumulate } from '@comunica/bus-rdf-metadata-accumulate';
-import type { ILink, MediatorRdfResolveHypermediaLinks } from '@comunica/bus-rdf-resolve-hypermedia-links';
+import type { MediatorRdfResolveHypermediaLinks } from '@comunica/bus-rdf-resolve-hypermedia-links';
 import type { ILinkQueue, MediatorRdfResolveHypermediaLinksQueue } from '@comunica/bus-rdf-resolve-hypermedia-links-queue';
-import type { IActionContext, IAggregatedStore, IQueryBindingsOptions, MetadataBindings } from '@comunica/types';
+import type { IActionContext, IAggregatedStore, IQueryBindingsOptions, MetadataBindings, ILink } from '@comunica/types';
 import type { Algebra } from 'sparqlalgebrajs';
 import type { SourceStateGetter, ISourceState } from './LinkedRdfSourcesAsyncRdfIterator';
 import { LinkedRdfSourcesAsyncRdfIterator } from './LinkedRdfSourcesAsyncRdfIterator';
@@ -28,7 +28,7 @@ export declare class MediatedLinkedRdfSourcesAsyncRdfIterator extends LinkedRdfS
     protected canStartNewIteratorConsiderReadable(): boolean;
     protected isRunning(): boolean;
     getLinkQueue(): Promise<ILinkQueue>;
-    protected getSourceLinks(metadata: Record<string, any>): Promise<ILink[]>;
+    protected getSourceLinks(metadata: Record<string, any>, startSource: ISourceState): Promise<ILink[]>;
     protected startIterator(startSource: ISourceState): void;
     accumulateMetadata(accumulatedMetadata: MetadataBindings, appendingMetadata: MetadataBindings): Promise<MetadataBindings>;
     protected updateMetadata(metadataNew: MetadataBindings): void;

package/lib/MediatedLinkedRdfSourcesAsyncRdfIterator.js CHANGED Viewed

@@ -97,9 +97,16 @@ class MediatedLinkedRdfSourcesAsyncRdfIterator extends LinkedRdfSourcesAsyncRdfI
         }
         return this.linkQueue;
     }
-    async getSourceLinks(metadata) {
+    async getSourceLinks(metadata, startSource) {
         try {
             const { links } = await this.mediatorRdfResolveHypermediaLinks.mediate({ context: this.context, metadata });
+            // Update discovery event statistic if available
+            const traversalTracker = this.context.get(context_entries_1.KeysStatistics.discoveredLinks);
+            if (traversalTracker) {
+                for (const link of links) {
+                    traversalTracker.updateStatistic({ url: link.url, metadata: { ...link.metadata } }, startSource.link);
+                }
+            }
             // Filter URLs to avoid cyclic next-page loops
             return links.filter((link) => {
                 if (this.handledUrls[link.url]) {

package/lib/MediatedLinkedRdfSourcesAsyncRdfIterator.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"MediatedLinkedRdfSourcesAsyncRdfIterator.js","sourceRoot":"","sources":["MediatedLinkedRdfSourcesAsyncRdfIterator.ts"],"names":[],"mappings":";;;AAMA,+DAA+D;AAG/D,uDAA+C;AAE/C,qDAA0C;AAE1C,yFAAsF;AAEtF,MAAM,EAAE,GAAG,IAAI,8BAAW,EAAgB,CAAC;AAC3C,MAAM,EAAE,GAAG,IAAI,yBAAO,EAAE,CAAC;AAEzB;;;;;GAKG;AACH,MAAa,wCAAyC,SAAQ,mEAAgC;IAU5F,YACE,SAAiB,EACjB,SAA4B,EAC5B,oBAAuD,EACvD,OAAuB,EACvB,eAAmC,EACnC,QAAgB,EAChB,YAAoB,EACpB,iBAAoC,EACpC,eAA6C,EAC7C,0BAAyD,EACzD,iCAAoE,EACpE,sCAA8E;QAE9E,KAAK,CACH,SAAS,EACT,SAAS,EACT,oBAAoB,EACpB,OAAO,EACP,QAAQ,EACR,YAAY,EACZ,iBAAiB;QACjB,2GAA2G;QAC3G,6CAA6C;QAC7C,eAAe,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,MAAM,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,SAAS,CAC1E,CAAC;QA3BI,wBAAmB,GAAG,KAAK,CAAC;QA4BlC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,0BAA0B,GAAG,0BAA0B,CAAC;QAC7D,IAAI,CAAC,iCAAiC,GAAG,iCAAiC,CAAC;QAC3E,IAAI,CAAC,sCAAsC,GAAG,sCAAsC,CAAC;QACrF,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,CAAC;QACxC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACzC,CAAC;IAED,iGAAiG;IACjG,uEAAuE;IACvE,+FAA+F;IAE/E,KAAK;QACnB,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,KAAK,CAAC,KAAK,EAAE,CAAC;YACd,OAAO;QACT,CAAC;QAED,IAAI,CAAC,YAAY,EAAE;aAChB,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE;YAClB,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,CAAC;gBACvC,0GAA0G;gBAC1G,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;oBACzB,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,eAAgB,CAAC,GAAG,EAAE,CAAC,CAAC;gBAChD,CAAC;gBACD,KAAK,CAAC,KAAK,EAAE,CAAC;YAChB,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;YAClC,CAAC;QACH,CAAC,CAAC;aACD,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1C,CAAC;IAEe,OAAO,CAAC,KAAa;QACnC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACrB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,YAAY,EAAE;aAChB,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE;YAClB,IAAI,KAAK,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,CAAC;gBAChD,0GAA0G;gBAC1G,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;oBACzB,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,eAAgB,CAAC,GAAG,EAAE,CAAC,CAAC;gBAChD,CAAC;gBACD,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;YAClC,CAAC;QACH,CAAC,CAAC;aACD,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1C,CAAC;IAEkB,WAAW,CAAC,SAAqB,EAAE,iBAA0B;QAC9E,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,mBAAmB,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;YAChG,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC;IAChC,CAAC;IAEkB,mBAAmB;QACpC,oGAAoG;QACpG,mHAAmH;QACnH,OAAO,CAAC,CAAC,IAAI,CAAC,mBAAmB;YAC/B,wDAAwD;YACxD,CAAC,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe,CAAC,mBAAmB,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC;IACzG,CAAC;IAEkB,mCAAmC;QACpD,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC;IAC/B,CAAC;IAEkB,SAAS;QAC1B,gBAAgB;QAChB,wDAAwD;QACxD,OAAO,CAAC,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe,CAAC,mBAAmB,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;IAC5F,CAAC;IAEM,YAAY;QACjB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,sCAAsC;iBACzD,OAAO,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC;iBAC3D,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACtC,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAES,KAAK,CAAC,cAAc,CAAC,QAA6B;QAC1D,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,iCAAiC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;YAE5G,8CAA8C;YAC9C,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;gBAC3B,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC/B,OAAO,KAAK,CAAC;gBACf,CAAC;gBACD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;gBAClC,OAAO,IAAI,CAAC;YACd,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,uGAAuG;YACvG,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAEkB,aAAa,CAAC,WAAyB;QACxD,IAAI,IAAI,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,gBAAgB,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7F,6GAA6G;YAC7G,2EAA2E;YAC3E,IAAI,CAAC,eAAe,EAAE,gBAAgB,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACjE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,aAAa,CAC7C,EAAE,CAAC,aAAa,CACd,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAChB,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAChB,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAChB,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CACjB,EACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,oCAAkB,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAC7D,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CACvB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAE,EAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAE,EAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAE,EAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAClB,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,MAAM,CAAc,MAAM,CAAC;iBAC7C,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACd,KAAK,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;YACnC,CAAC,CAAC,CAAC;QACP,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,kBAAkB,CAC7B,mBAAqC,EACrC,iBAAmC;QAEnC,OAA0B,CAAC,MAAM,IAAI,CAAC,0BAA0B,CAAC,OAAO,CAAC;YACvE,IAAI,EAAE,QAAQ;YACd,mBAAmB;YACnB,iBAAiB;YACjB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC,CAAC,QAAQ,CAAC;IACf,CAAC;IAEkB,cAAc,CAAC,WAA6B;QAC7D,KAAK,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;QAClC,IAAI,CAAC,eAAe,EAAE,eAAe,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;IAC3D,CAAC;CACF;AAxLD,4FAwLC","sourcesContent":["import type { MediatorRdfMetadataAccumulate } from '@comunica/bus-rdf-metadata-accumulate';\nimport type { ILink, MediatorRdfResolveHypermediaLinks } from '@comunica/bus-rdf-resolve-hypermedia-links';\nimport type {\n ILinkQueue,\n MediatorRdfResolveHypermediaLinksQueue,\n} from '@comunica/bus-rdf-resolve-hypermedia-links-queue';\nimport { KeysQueryOperation } from '@comunica/context-entries';\nimport type { IActionContext, IAggregatedStore, IQueryBindingsOptions, MetadataBindings } from '@comunica/types';\nimport type * as RDF from '@rdfjs/types';\nimport { DataFactory } from 'rdf-data-factory';\nimport type { Algebra } from 'sparqlalgebrajs';\nimport { Factory } from 'sparqlalgebrajs';\nimport type { SourceStateGetter, ISourceState } from './LinkedRdfSourcesAsyncRdfIterator';\nimport { LinkedRdfSourcesAsyncRdfIterator } from './LinkedRdfSourcesAsyncRdfIterator';\n\nconst DF = new DataFactory<RDF.BaseQuad>();\nconst AF = new Factory();\n\n/*\n An quad iterator that can iterate over consecutive RDF sources\n * that are determined using the rdf-resolve-hypermedia-links bus.\n \n @see LinkedRdfSourcesAsyncRdfIterator\n */\nexport class MediatedLinkedRdfSourcesAsyncRdfIterator extends LinkedRdfSourcesAsyncRdfIterator {\n private readonly mediatorMetadataAccumulate: MediatorRdfMetadataAccumulate;\n private readonly mediatorRdfResolveHypermediaLinks: MediatorRdfResolveHypermediaLinks;\n private readonly mediatorRdfResolveHypermediaLinksQueue: MediatorRdfResolveHypermediaLinksQueue;\n private readonly forceSourceType?: string;\n private readonly handledUrls: Record<string, boolean>;\n private readonly aggregatedStore: IAggregatedStore \| undefined;\n private linkQueue: Promise<ILinkQueue> \| undefined;\n private wasForcefullyClosed = false;\n\n public constructor(\n cacheSize: number,\n operation: Algebra.Operation,\n queryBindingsOptions: IQueryBindingsOptions \| undefined,\n context: IActionContext,\n forceSourceType: string \| undefined,\n firstUrl: string,\n maxIterators: number,\n sourceStateGetter: SourceStateGetter,\n aggregatedStore: IAggregatedStore \| undefined,\n mediatorMetadataAccumulate: MediatorRdfMetadataAccumulate,\n mediatorRdfResolveHypermediaLinks: MediatorRdfResolveHypermediaLinks,\n mediatorRdfResolveHypermediaLinksQueue: MediatorRdfResolveHypermediaLinksQueue,\n ) {\n super(\n cacheSize,\n operation,\n queryBindingsOptions,\n context,\n firstUrl,\n maxIterators,\n sourceStateGetter,\n // Buffersize must be infinite for an aggregated store because it must keep filling until there are no more\n // derived iterators in the aggregated store.\n aggregatedStore ? { maxBufferSize: Number.POSITIVE_INFINITY } : undefined,\n );\n this.forceSourceType = forceSourceType;\n this.mediatorMetadataAccumulate = mediatorMetadataAccumulate;\n this.mediatorRdfResolveHypermediaLinks = mediatorRdfResolveHypermediaLinks;\n this.mediatorRdfResolveHypermediaLinksQueue = mediatorRdfResolveHypermediaLinksQueue;\n this.handledUrls = { [firstUrl]: true };\n this.aggregatedStore = aggregatedStore;\n }\n\n // Mark the aggregated store as ended once we trigger the closing or destroying of this iterator.\n // We don't override _end, because that would mean that we have to wait\n // until the buffer of this iterator must be fully consumed, which will not always be the case.\n\n public override close(): void {\n if (!this.aggregatedStore) {\n super.close();\n return;\n }\n\n this.getLinkQueue()\n .then((linkQueue) => {\n if (this.isCloseable(linkQueue, false)) {\n // Wait a tick before ending the aggregatedStore, to ensure that pending match() calls to it have started.\n if (this.aggregatedStore) {\n setTimeout(() => this.aggregatedStore!.end());\n }\n super.close();\n } else {\n this.wasForcefullyClosed = true;\n }\n })\n .catch(error => super.destroy(error));\n }\n\n public override destroy(cause?: Error): void {\n if (!this.aggregatedStore) {\n super.destroy(cause);\n return;\n }\n\n this.getLinkQueue()\n .then((linkQueue) => {\n if (cause ?? this.isCloseable(linkQueue, false)) {\n // Wait a tick before ending the aggregatedStore, to ensure that pending match() calls to it have started.\n if (this.aggregatedStore) {\n setTimeout(() => this.aggregatedStore!.end());\n }\n super.destroy(cause);\n } else {\n this.wasForcefullyClosed = true;\n }\n })\n .catch(error => super.destroy(error));\n }\n\n protected override isCloseable(linkQueue: ILinkQueue, requireQueueEmpty: boolean): boolean {\n return (requireQueueEmpty ? linkQueue.isEmpty() : this.wasForcefullyClosed \|\| linkQueue.isEmpty()) &&\n !this.areIteratorsRunning();\n }\n\n protected override canStartNewIterator(): boolean {\n // Also allow sub-iterators to be started if the aggregated store has at least one running iterator.\n // We need this because there are cases where these running iterators will be consumed before this linked iterator.\n return (!this.wasForcefullyClosed &&\n // eslint-disable-next-line ts/prefer-nullish-coalescing\n (this.aggregatedStore && this.aggregatedStore.hasRunningIterators())) \|\| super.canStartNewIterator();\n }\n\n protected override canStartNewIteratorConsiderReadable(): boolean {\n return !this.aggregatedStore;\n }\n\n protected override isRunning(): boolean {\n // Same as above\n // eslint-disable-next-line ts/prefer-nullish-coalescing\n return (this.aggregatedStore && this.aggregatedStore.hasRunningIterators()) \|\| !this.done;\n }\n\n public getLinkQueue(): Promise<ILinkQueue> {\n if (!this.linkQueue) {\n this.linkQueue = this.mediatorRdfResolveHypermediaLinksQueue\n .mediate({ firstUrl: this.firstUrl, context: this.context })\n .then(result => result.linkQueue);\n }\n return this.linkQueue;\n }\n\n protected async getSourceLinks(metadata: Record<string, any>): Promise<ILink[]> {\n try {\n const { links } = await this.mediatorRdfResolveHypermediaLinks.mediate({ context: this.context, metadata });\n\n // Filter URLs to avoid cyclic next-page loops\n return links.filter((link) => {\n if (this.handledUrls[link.url]) {\n return false;\n }\n this.handledUrls[link.url] = true;\n return true;\n });\n } catch {\n // No next URLs may be available, for example when we've reached the end of a Hydra next-page sequence.\n return [];\n }\n }\n\n protected override startIterator(startSource: ISourceState): void {\n if (this.aggregatedStore && !this.aggregatedStore.containedSources.has(startSource.link.url)) {\n // A source that has been cached due to earlier query executions may not be part of the aggregated store yet.\n // In that case, we add all quads from that source to the aggregated store.\n this.aggregatedStore?.containedSources.add(startSource.link.url);\n const stream = startSource.source.queryBindings(\n AF.createPattern(\n DF.variable('s'),\n DF.variable('p'),\n DF.variable('o'),\n DF.variable('g'),\n ),\n this.context.set(KeysQueryOperation.unionDefaultGraph, true),\n ).map(bindings => DF.quad(\n bindings.get('s')!,\n bindings.get('p')!,\n bindings.get('o')!,\n bindings.get('g'),\n ));\n this.aggregatedStore.import(<RDF.Stream> stream)\n .on('end', () => {\n super.startIterator(startSource);\n });\n } else {\n super.startIterator(startSource);\n }\n }\n\n public async accumulateMetadata(\n accumulatedMetadata: MetadataBindings,\n appendingMetadata: MetadataBindings,\n ): Promise<MetadataBindings> {\n return <MetadataBindings> (await this.mediatorMetadataAccumulate.mediate({\n mode: 'append',\n accumulatedMetadata,\n appendingMetadata,\n context: this.context,\n })).metadata;\n }\n\n protected override updateMetadata(metadataNew: MetadataBindings): void {\n super.updateMetadata(metadataNew);\n this.aggregatedStore?.setBaseMetadata(metadataNew, true);\n }\n}\n"]}
1	+ {"version":3,"file":"MediatedLinkedRdfSourcesAsyncRdfIterator.js","sourceRoot":"","sources":["MediatedLinkedRdfSourcesAsyncRdfIterator.ts"],"names":[],"mappings":";;;AAMA,+DAA+E;AAW/E,uDAA+C;AAE/C,qDAA0C;AAE1C,yFAAsF;AAEtF,MAAM,EAAE,GAAG,IAAI,8BAAW,EAAgB,CAAC;AAC3C,MAAM,EAAE,GAAG,IAAI,yBAAO,EAAE,CAAC;AAEzB;;;;;GAKG;AACH,MAAa,wCAAyC,SAAQ,mEAAgC;IAU5F,YACE,SAAiB,EACjB,SAA4B,EAC5B,oBAAuD,EACvD,OAAuB,EACvB,eAAmC,EACnC,QAAgB,EAChB,YAAoB,EACpB,iBAAoC,EACpC,eAA6C,EAC7C,0BAAyD,EACzD,iCAAoE,EACpE,sCAA8E;QAE9E,KAAK,CACH,SAAS,EACT,SAAS,EACT,oBAAoB,EACpB,OAAO,EACP,QAAQ,EACR,YAAY,EACZ,iBAAiB;QACjB,2GAA2G;QAC3G,6CAA6C;QAC7C,eAAe,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,MAAM,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,SAAS,CAC1E,CAAC;QA3BI,wBAAmB,GAAG,KAAK,CAAC;QA4BlC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,0BAA0B,GAAG,0BAA0B,CAAC;QAC7D,IAAI,CAAC,iCAAiC,GAAG,iCAAiC,CAAC;QAC3E,IAAI,CAAC,sCAAsC,GAAG,sCAAsC,CAAC;QACrF,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,CAAC;QACxC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACzC,CAAC;IAED,iGAAiG;IACjG,uEAAuE;IACvE,+FAA+F;IAE/E,KAAK;QACnB,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,KAAK,CAAC,KAAK,EAAE,CAAC;YACd,OAAO;QACT,CAAC;QAED,IAAI,CAAC,YAAY,EAAE;aAChB,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE;YAClB,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,CAAC;gBACvC,0GAA0G;gBAC1G,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;oBACzB,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,eAAgB,CAAC,GAAG,EAAE,CAAC,CAAC;gBAChD,CAAC;gBACD,KAAK,CAAC,KAAK,EAAE,CAAC;YAChB,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;YAClC,CAAC;QACH,CAAC,CAAC;aACD,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1C,CAAC;IAEe,OAAO,CAAC,KAAa;QACnC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACrB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,YAAY,EAAE;aAChB,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE;YAClB,IAAI,KAAK,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,CAAC;gBAChD,0GAA0G;gBAC1G,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;oBACzB,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,eAAgB,CAAC,GAAG,EAAE,CAAC,CAAC;gBAChD,CAAC;gBACD,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;YAClC,CAAC;QACH,CAAC,CAAC;aACD,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1C,CAAC;IAEkB,WAAW,CAAC,SAAqB,EAAE,iBAA0B;QAC9E,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,mBAAmB,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;YAChG,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC;IAChC,CAAC;IAEkB,mBAAmB;QACpC,oGAAoG;QACpG,mHAAmH;QACnH,OAAO,CAAC,CAAC,IAAI,CAAC,mBAAmB;YAC/B,wDAAwD;YACxD,CAAC,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe,CAAC,mBAAmB,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC;IACzG,CAAC;IAEkB,mCAAmC;QACpD,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC;IAC/B,CAAC;IAEkB,SAAS;QAC1B,gBAAgB;QAChB,wDAAwD;QACxD,OAAO,CAAC,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe,CAAC,mBAAmB,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;IAC5F,CAAC;IAEM,YAAY;QACjB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,sCAAsC;iBACzD,OAAO,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC;iBAC3D,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACtC,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAES,KAAK,CAAC,cAAc,CAAC,QAA6B,EAAE,WAAyB;QACrF,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,iCAAiC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;YAC5G,gDAAgD;YAChD,MAAM,gBAAgB,GACpB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,gCAAc,CAAC,eAAe,CAAC,CAAC;YACnD,IAAI,gBAAgB,EAAE,CAAC;gBACrB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBACzB,gBAAgB,CAAC,eAAe,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,EAAC,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC;gBACvG,CAAC;YACH,CAAC;YAED,8CAA8C;YAC9C,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;gBAC3B,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC/B,OAAO,KAAK,CAAC;gBACf,CAAC;gBACD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;gBAClC,OAAO,IAAI,CAAC;YACd,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,uGAAuG;YACvG,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAEkB,aAAa,CAAC,WAAyB;QACxD,IAAI,IAAI,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,gBAAgB,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7F,6GAA6G;YAC7G,2EAA2E;YAC3E,IAAI,CAAC,eAAe,EAAE,gBAAgB,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACjE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,aAAa,CAC7C,EAAE,CAAC,aAAa,CACd,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAChB,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAChB,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAChB,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CACjB,EACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,oCAAkB,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAC7D,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CACvB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAE,EAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAE,EAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAE,EAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAClB,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,MAAM,CAAc,MAAM,CAAC;iBAC7C,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACd,KAAK,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;YACnC,CAAC,CAAC,CAAC;QACP,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,kBAAkB,CAC7B,mBAAqC,EACrC,iBAAmC;QAEnC,OAA0B,CAAC,MAAM,IAAI,CAAC,0BAA0B,CAAC,OAAO,CAAC;YACvE,IAAI,EAAE,QAAQ;YACd,mBAAmB;YACnB,iBAAiB;YACjB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC,CAAC,QAAQ,CAAC;IACf,CAAC;IAEkB,cAAc,CAAC,WAA6B;QAC7D,KAAK,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;QAClC,IAAI,CAAC,eAAe,EAAE,eAAe,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;IAC3D,CAAC;CACF;AAhMD,4FAgMC","sourcesContent":["import type { MediatorRdfMetadataAccumulate } from '@comunica/bus-rdf-metadata-accumulate';\nimport type { MediatorRdfResolveHypermediaLinks } from '@comunica/bus-rdf-resolve-hypermedia-links';\nimport type {\n ILinkQueue,\n MediatorRdfResolveHypermediaLinksQueue,\n} from '@comunica/bus-rdf-resolve-hypermedia-links-queue';\nimport { KeysQueryOperation, KeysStatistics } from '@comunica/context-entries';\nimport type {\n IActionContext,\n IAggregatedStore,\n IQueryBindingsOptions,\n MetadataBindings,\n ILink,\n IStatisticBase,\n IDiscoverEventData,\n} from '@comunica/types';\nimport type * as RDF from '@rdfjs/types';\nimport { DataFactory } from 'rdf-data-factory';\nimport type { Algebra } from 'sparqlalgebrajs';\nimport { Factory } from 'sparqlalgebrajs';\nimport type { SourceStateGetter, ISourceState } from './LinkedRdfSourcesAsyncRdfIterator';\nimport { LinkedRdfSourcesAsyncRdfIterator } from './LinkedRdfSourcesAsyncRdfIterator';\n\nconst DF = new DataFactory<RDF.BaseQuad>();\nconst AF = new Factory();\n\n/*\n An quad iterator that can iterate over consecutive RDF sources\n * that are determined using the rdf-resolve-hypermedia-links bus.\n \n @see LinkedRdfSourcesAsyncRdfIterator\n */\nexport class MediatedLinkedRdfSourcesAsyncRdfIterator extends LinkedRdfSourcesAsyncRdfIterator {\n private readonly mediatorMetadataAccumulate: MediatorRdfMetadataAccumulate;\n private readonly mediatorRdfResolveHypermediaLinks: MediatorRdfResolveHypermediaLinks;\n private readonly mediatorRdfResolveHypermediaLinksQueue: MediatorRdfResolveHypermediaLinksQueue;\n private readonly forceSourceType?: string;\n private readonly handledUrls: Record<string, boolean>;\n private readonly aggregatedStore: IAggregatedStore \| undefined;\n private linkQueue: Promise<ILinkQueue> \| undefined;\n private wasForcefullyClosed = false;\n\n public constructor(\n cacheSize: number,\n operation: Algebra.Operation,\n queryBindingsOptions: IQueryBindingsOptions \| undefined,\n context: IActionContext,\n forceSourceType: string \| undefined,\n firstUrl: string,\n maxIterators: number,\n sourceStateGetter: SourceStateGetter,\n aggregatedStore: IAggregatedStore \| undefined,\n mediatorMetadataAccumulate: MediatorRdfMetadataAccumulate,\n mediatorRdfResolveHypermediaLinks: MediatorRdfResolveHypermediaLinks,\n mediatorRdfResolveHypermediaLinksQueue: MediatorRdfResolveHypermediaLinksQueue,\n ) {\n super(\n cacheSize,\n operation,\n queryBindingsOptions,\n context,\n firstUrl,\n maxIterators,\n sourceStateGetter,\n // Buffersize must be infinite for an aggregated store because it must keep filling until there are no more\n // derived iterators in the aggregated store.\n aggregatedStore ? { maxBufferSize: Number.POSITIVE_INFINITY } : undefined,\n );\n this.forceSourceType = forceSourceType;\n this.mediatorMetadataAccumulate = mediatorMetadataAccumulate;\n this.mediatorRdfResolveHypermediaLinks = mediatorRdfResolveHypermediaLinks;\n this.mediatorRdfResolveHypermediaLinksQueue = mediatorRdfResolveHypermediaLinksQueue;\n this.handledUrls = { [firstUrl]: true };\n this.aggregatedStore = aggregatedStore;\n }\n\n // Mark the aggregated store as ended once we trigger the closing or destroying of this iterator.\n // We don't override _end, because that would mean that we have to wait\n // until the buffer of this iterator must be fully consumed, which will not always be the case.\n\n public override close(): void {\n if (!this.aggregatedStore) {\n super.close();\n return;\n }\n\n this.getLinkQueue()\n .then((linkQueue) => {\n if (this.isCloseable(linkQueue, false)) {\n // Wait a tick before ending the aggregatedStore, to ensure that pending match() calls to it have started.\n if (this.aggregatedStore) {\n setTimeout(() => this.aggregatedStore!.end());\n }\n super.close();\n } else {\n this.wasForcefullyClosed = true;\n }\n })\n .catch(error => super.destroy(error));\n }\n\n public override destroy(cause?: Error): void {\n if (!this.aggregatedStore) {\n super.destroy(cause);\n return;\n }\n\n this.getLinkQueue()\n .then((linkQueue) => {\n if (cause ?? this.isCloseable(linkQueue, false)) {\n // Wait a tick before ending the aggregatedStore, to ensure that pending match() calls to it have started.\n if (this.aggregatedStore) {\n setTimeout(() => this.aggregatedStore!.end());\n }\n super.destroy(cause);\n } else {\n this.wasForcefullyClosed = true;\n }\n })\n .catch(error => super.destroy(error));\n }\n\n protected override isCloseable(linkQueue: ILinkQueue, requireQueueEmpty: boolean): boolean {\n return (requireQueueEmpty ? linkQueue.isEmpty() : this.wasForcefullyClosed \|\| linkQueue.isEmpty()) &&\n !this.areIteratorsRunning();\n }\n\n protected override canStartNewIterator(): boolean {\n // Also allow sub-iterators to be started if the aggregated store has at least one running iterator.\n // We need this because there are cases where these running iterators will be consumed before this linked iterator.\n return (!this.wasForcefullyClosed &&\n // eslint-disable-next-line ts/prefer-nullish-coalescing\n (this.aggregatedStore && this.aggregatedStore.hasRunningIterators())) \|\| super.canStartNewIterator();\n }\n\n protected override canStartNewIteratorConsiderReadable(): boolean {\n return !this.aggregatedStore;\n }\n\n protected override isRunning(): boolean {\n // Same as above\n // eslint-disable-next-line ts/prefer-nullish-coalescing\n return (this.aggregatedStore && this.aggregatedStore.hasRunningIterators()) \|\| !this.done;\n }\n\n public getLinkQueue(): Promise<ILinkQueue> {\n if (!this.linkQueue) {\n this.linkQueue = this.mediatorRdfResolveHypermediaLinksQueue\n .mediate({ firstUrl: this.firstUrl, context: this.context })\n .then(result => result.linkQueue);\n }\n return this.linkQueue;\n }\n\n protected async getSourceLinks(metadata: Record<string, any>, startSource: ISourceState): Promise<ILink[]> {\n try {\n const { links } = await this.mediatorRdfResolveHypermediaLinks.mediate({ context: this.context, metadata });\n // Update discovery event statistic if available\n const traversalTracker: IStatisticBase<IDiscoverEventData> \| undefined =\n this.context.get(KeysStatistics.discoveredLinks);\n if (traversalTracker) {\n for (const link of links) {\n traversalTracker.updateStatistic({ url: link.url, metadata: { ...link.metadata }}, startSource.link);\n }\n }\n\n // Filter URLs to avoid cyclic next-page loops\n return links.filter((link) => {\n if (this.handledUrls[link.url]) {\n return false;\n }\n this.handledUrls[link.url] = true;\n return true;\n });\n } catch {\n // No next URLs may be available, for example when we've reached the end of a Hydra next-page sequence.\n return [];\n }\n }\n\n protected override startIterator(startSource: ISourceState): void {\n if (this.aggregatedStore && !this.aggregatedStore.containedSources.has(startSource.link.url)) {\n // A source that has been cached due to earlier query executions may not be part of the aggregated store yet.\n // In that case, we add all quads from that source to the aggregated store.\n this.aggregatedStore?.containedSources.add(startSource.link.url);\n const stream = startSource.source.queryBindings(\n AF.createPattern(\n DF.variable('s'),\n DF.variable('p'),\n DF.variable('o'),\n DF.variable('g'),\n ),\n this.context.set(KeysQueryOperation.unionDefaultGraph, true),\n ).map(bindings => DF.quad(\n bindings.get('s')!,\n bindings.get('p')!,\n bindings.get('o')!,\n bindings.get('g'),\n ));\n this.aggregatedStore.import(<RDF.Stream> stream)\n .on('end', () => {\n super.startIterator(startSource);\n });\n } else {\n super.startIterator(startSource);\n }\n }\n\n public async accumulateMetadata(\n accumulatedMetadata: MetadataBindings,\n appendingMetadata: MetadataBindings,\n ): Promise<MetadataBindings> {\n return <MetadataBindings> (await this.mediatorMetadataAccumulate.mediate({\n mode: 'append',\n accumulatedMetadata,\n appendingMetadata,\n context: this.context,\n })).metadata;\n }\n\n protected override updateMetadata(metadataNew: MetadataBindings): void {\n super.updateMetadata(metadataNew);\n this.aggregatedStore?.setBaseMetadata(metadataNew, true);\n }\n}\n"]}

package/lib/QuerySourceHypermedia.d.ts CHANGED Viewed

@@ -4,9 +4,9 @@ import type { MediatorQuerySourceIdentifyHypermedia } from '@comunica/bus-query-
 import type { MediatorRdfMetadata } from '@comunica/bus-rdf-metadata';
 import type { MediatorRdfMetadataAccumulate } from '@comunica/bus-rdf-metadata-accumulate';
 import type { MediatorRdfMetadataExtract } from '@comunica/bus-rdf-metadata-extract';
-import type { ILink, MediatorRdfResolveHypermediaLinks } from '@comunica/bus-rdf-resolve-hypermedia-links';
+import type { MediatorRdfResolveHypermediaLinks } from '@comunica/bus-rdf-resolve-hypermedia-links';
 import type { MediatorRdfResolveHypermediaLinksQueue } from '@comunica/bus-rdf-resolve-hypermedia-links-queue';
-import type { BindingsStream, FragmentSelectorShape, IActionContext, IAggregatedStore, IQueryBindingsOptions, IQuerySource } from '@comunica/types';
+import type { BindingsStream, FragmentSelectorShape, IActionContext, IAggregatedStore, IQueryBindingsOptions, IQuerySource, ILink } from '@comunica/types';
 import type * as RDF from '@rdfjs/types';
 import type { AsyncIterator } from 'asynciterator';
 import { LRUCache } from 'lru-cache';

package/lib/QuerySourceHypermedia.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"QuerySourceHypermedia.js","sourceRoot":"","sources":["QuerySourceHypermedia.ts"],"names":[],"mappings":";;;AAAA,mGAA+E;AAS/E,+DAAoE;AAYpE,iDAAkD;AAClD,yCAAqC;AACrC,qDAA2C;AAG3C,yGAAsG;AACtG,qEAAkE;AAElE,MAAa,qBAAqB;IAiBhC,YACE,SAAiB,EACjB,QAAgB,EAChB,eAAmC,EACnC,YAAoB,EACpB,cAAuB,EACvB,SAAwB,EACxB,UAA4C,EAC5C,eAAgC;QAEhC,IAAI,CAAC,cAAc,GAAG,QAAQ,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,YAAY,GAAG,IAAI,oBAAQ,CAAgC,EAAE,GAAG,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;IAC3F,CAAC;IAEM,KAAK,CAAC,gBAAgB,CAAC,OAAuB;QACnD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;QAChH,OAAO,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;IACjD,CAAC;IAEM,aAAa,CAClB,SAA4B,EAC5B,OAAuB,EACvB,OAA+B;QAE/B,gEAAgE;QAChE,MAAM,eAAe,GAAiC,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACtF,IAAI,eAAe,IAAI,SAAS,CAAC,IAAI,KAAK,SAAS,IAAI,eAAe,CAAC,OAAO,EAAE,CAAC;YAC/E,OAAO,IAAI,oDAAgB,CAAC,eAAe,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC,aAAa,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACvG,CAAC;QAED,6CAA6C;QAC7C,IAAI,IAAI,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC,eAAe,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,eAAe,CAAC;iBACvE,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QACvC,CAAC;QAED,MAAM,EAAE,GAA6C,IAAI,mFAAwC,CAC/F,IAAI,CAAC,SAAS,EACd,SAAS,EACT,OAAO,EACP,OAAO,EACP,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,YAAY,EACjB,CAAC,IAAI,EAAE,eAAe,EAAE,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,eAAe,EAAE,OAAO,EAAE,eAAe,CAAC,EAChG,eAAe,EACf,IAAI,CAAC,SAAS,CAAC,0BAA0B,EACzC,IAAI,CAAC,SAAS,CAAC,iCAAiC,EAChD,IAAI,CAAC,SAAS,CAAC,sCAAsC,CACtD,CAAC;QACF,IAAI,eAAe,EAAE,CAAC;YACpB,eAAe,CAAC,OAAO,GAAG,IAAI,CAAC;YAE/B,uFAAuF;YACvF,uGAAuG;YACvG,MAAM,QAAQ,GAAG,GAAS,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC;YAC5C,eAAe,CAAC,0BAA0B,CAAC,QAAQ,CAAC,CAAC;YACrD,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,eAAe,CAAC,6BAA6B,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC9E,CAAC;QAED,OAAO,EAAE,CAAC;IACZ,CAAC;IAEM,UAAU,CAAC,SAA4B,EAAE,OAAuB;QACrE,OAAO,IAAI,iCAAiB,CAAC,KAAK,IAAG,EAAE;YACrC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;YAChH,OAAO,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,YAAY,CAAC,SAAsB,EAAE,OAAuB;QACvE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;QAChH,OAAO,MAAM,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC9D,CAAC;IAEM,KAAK,CAAC,SAAS,CAAC,SAAyB,EAAE,OAAuB;QACvE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;QAChH,OAAO,MAAM,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC3D,CAAC;IAED;;;;;;OAMG;IACI,KAAK,CAAC,SAAS,CACpB,IAAW,EACX,eAAwC,EACxC,OAAuB,EACvB,eAA6C;QAE7C,oCAAoC;QACpC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACxC,CAAC;QAED,mDAAmD;QACnD,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;QACnB,IAAI,KAAiB,CAAC;QACtB,IAAI,QAA6B,CAAC;QAClC,IAAI,CAAC;YACH,MAAM,oBAAoB,GAA+B,MAAM,IAAI,CAAC,SAAS,CAAC,sBAAsB;iBACjG,OAAO,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;YAC7B,GAAG,GAAG,oBAAoB,CAAC,GAAG,CAAC;YAE/B,yBAAyB;YACzB,MAAM,iBAAiB,GAA4B,MAAM,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,OAAO,CAC9F,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,oBAAoB,CAAC,IAAI,EAAE,OAAO,EAAE,oBAAoB,CAAC,QAAQ,EAAE,OAAO,EAAE,CACpG,CAAC;YAEF,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBACtC,qCAAqC;gBACrC,wDAAwD;gBACxD,kDAAkD;gBAClD,iEAAiE;YACnE,CAAC,CAAC,CAAC;YAEH,QAAQ,GAAG,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC,OAAO,CAAC;gBAC/D,OAAO;gBACP,GAAG;gBACH,2DAA2D;gBAC3D,QAAQ,EAAE,iBAAiB,CAAC,QAAQ;gBACpC,OAAO,EAAE,oBAAoB,CAAC,OAAO;gBACrC,WAAW,EAAE,oBAAoB,CAAC,WAAW;aAC9C,CAAC,CAAC,CAAC,QAAQ,CAAC;YACb,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC;YAE/B,uCAAuC;YACvC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACnB,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,+FAA+F;YAC/F,kHAAkH;YAClH,KAAK,GAAG,IAAI,0BAAQ,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,GAAG,GAAG,EAAE;gBAChB,UAAU,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC7C,OAAO,IAAI,CAAC;YACd,CAAC,CAAC;YACF,CAAC,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,0BAA0B,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC;YAE1G,kGAAkG;YAClG,wEAAwE;YACxE,IAAI,CAAC,UAAU,CAAC,2BAA2B,GAAG,YAAqB,KAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QACvF,CAAC;QAED,+CAA+C;QAC/C,eAAe,EAAE,eAAe,CAAoB,QAAQ,EAAE,KAAK,CAAC,CAAC;QACrE,eAAe,EAAE,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChD,eAAe,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QAE/B,uBAAuB;QACvB,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,qCAAqC,CAAC,OAAO,CAAC;YAC7F,OAAO;YACP,eAAe,EAAE,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS;YAC9E,eAAe;YACf,QAAQ;YACR,KAAK;YACL,GAAG;SACJ,CAAC,CAAC;QAEH,IAAI,OAAO,EAAE,CAAC;YACZ,8BAA8B;YAC9B,uFAAuF;YACvF,+CAA+C;YAC/C,eAAe,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC;QAClC,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAqB,QAAQ,EAAE,eAAe,EAAE,CAAC;IAClF,CAAC;IAED;;;;;;;OAOG;IACO,eAAe,CACvB,IAAW,EACX,eAAwC,EACxC,OAAuB,EACvB,eAA6C;QAE7C,IAAI,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC7C,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,eAAe,EAAE,OAAO,EAAE,eAAe,CAAC,CAAC;QACzE,IAAI,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,QAAQ,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;YAChE,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAC1C,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEM,iBAAiB,CAAC,OAAuB;QAC9C,IAAI,eAA6C,CAAC;QAClD,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,MAAM,gBAAgB,GAA8C,OAAO;iBACxE,GAAG,CAAC,yCAAuB,CAAC,iCAAiC,CAAC,CAAC;YAClE,IAAI,gBAAgB,EAAE,CAAC;gBACrB,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACtD,IAAI,CAAC,eAAe,EAAE,CAAC;oBACrB,eAAe,GAAG,IAAI,+CAAsB,CAC1C,SAAS,EACT,KAAK,EAAC,mBAAmB,EAAE,iBAAiB,EAAE,EAAE,CAC9C,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,0BAA0B,CAAC,OAAO,CAAC;wBACvD,IAAI,EAAE,QAAQ;wBACd,mBAAmB;wBACnB,iBAAiB;wBACjB,OAAO;qBACR,CAAC,CAAC,CAAC,QAAQ,CACf,CAAC;oBACF,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;gBACvD,CAAC;gBACD,OAAO,eAAe,CAAC;YACzB,CAAC;QACH,CAAC;IACH,CAAC;IAEM,QAAQ;QACb,OAAO,yBAAyB,IAAI,CAAC,QAAQ,GAAG,CAAC;IACnD,CAAC;CACF;AA3PD,sDA2PC","sourcesContent":["import { QuerySourceRdfJs } from '@comunica/actor-query-source-identify-rdfjs';\nimport type { BindingsFactory } from '@comunica/bindings-factory';\nimport type { IActorDereferenceRdfOutput, MediatorDereferenceRdf } from '@comunica/bus-dereference-rdf';\nimport type { MediatorQuerySourceIdentifyHypermedia } from '@comunica/bus-query-source-identify-hypermedia';\nimport type { IActorRdfMetadataOutput, MediatorRdfMetadata } from '@comunica/bus-rdf-metadata';\nimport type { MediatorRdfMetadataAccumulate } from '@comunica/bus-rdf-metadata-accumulate';\nimport type { MediatorRdfMetadataExtract } from '@comunica/bus-rdf-metadata-extract';\nimport type { ILink, MediatorRdfResolveHypermediaLinks } from '@comunica/bus-rdf-resolve-hypermedia-links';\nimport type { MediatorRdfResolveHypermediaLinksQueue } from '@comunica/bus-rdf-resolve-hypermedia-links-queue';\nimport { KeysQuerySourceIdentify } from '@comunica/context-entries';\nimport type {\n BindingsStream,\n FragmentSelectorShape,\n IActionContext,\n IAggregatedStore,\n IQueryBindingsOptions,\n IQuerySource,\n MetadataBindings,\n} from '@comunica/types';\nimport type * as RDF from '@rdfjs/types';\nimport type { AsyncIterator } from 'asynciterator';\nimport { TransformIterator } from 'asynciterator';\nimport { LRUCache } from 'lru-cache';\nimport { Readable } from 'readable-stream';\nimport type { Algebra } from 'sparqlalgebrajs';\nimport type { ISourceState } from './LinkedRdfSourcesAsyncRdfIterator';\nimport { MediatedLinkedRdfSourcesAsyncRdfIterator } from './MediatedLinkedRdfSourcesAsyncRdfIterator';\nimport { StreamingStoreMetadata } from './StreamingStoreMetadata';\n\nexport class QuerySourceHypermedia implements IQuerySource {\n public readonly referenceValue: string;\n public readonly firstUrl: string;\n public readonly forceSourceType?: string;\n public readonly aggregateStore: boolean;\n public readonly mediators: IMediatorArgs;\n public readonly logWarning: (warningMessage: string) => void;\n public readonly bindingsFactory: BindingsFactory;\n\n /*\n A cache for source URLs to source states.\n /\n public sourcesState: LRUCache<string, Promise<ISourceState>>;\n\n private readonly cacheSize: number;\n private readonly maxIterators: number;\n\n public constructor(\n cacheSize: number,\n firstUrl: string,\n forceSourceType: string \| undefined,\n maxIterators: number,\n aggregateStore: boolean,\n mediators: IMediatorArgs,\n logWarning: (warningMessage: string) => void,\n bindingsFactory: BindingsFactory,\n ) {\n this.referenceValue = firstUrl;\n this.cacheSize = cacheSize;\n this.firstUrl = firstUrl;\n this.forceSourceType = forceSourceType;\n this.maxIterators = maxIterators;\n this.mediators = mediators;\n this.aggregateStore = aggregateStore;\n this.logWarning = logWarning;\n this.bindingsFactory = bindingsFactory;\n this.sourcesState = new LRUCache<string, Promise<ISourceState>>({ max: this.cacheSize });\n }\n\n public async getSelectorShape(context: IActionContext): Promise<FragmentSelectorShape> {\n const source = await this.getSourceCached({ url: this.firstUrl }, {}, context, this.getAggregateStore(context));\n return source.source.getSelectorShape(context);\n }\n\n public queryBindings(\n operation: Algebra.Operation,\n context: IActionContext,\n options?: IQueryBindingsOptions,\n ): BindingsStream {\n // Optimized match with aggregated store if enabled and started.\n const aggregatedStore: IAggregatedStore \| undefined = this.getAggregateStore(context);\n if (aggregatedStore && operation.type === 'pattern' && aggregatedStore.started) {\n return new QuerySourceRdfJs(aggregatedStore, this.bindingsFactory).queryBindings(operation, context);\n }\n\n // Initialize the sources state on first call\n if (this.sourcesState.size === 0) {\n this.getSourceCached({ url: this.firstUrl }, {}, context, aggregatedStore)\n .catch(error => it.destroy(error));\n }\n\n const it: MediatedLinkedRdfSourcesAsyncRdfIterator = new MediatedLinkedRdfSourcesAsyncRdfIterator(\n this.cacheSize,\n operation,\n options,\n context,\n this.forceSourceType,\n this.firstUrl,\n this.maxIterators,\n (link, handledDatasets) => this.getSourceCached(link, handledDatasets, context, aggregatedStore),\n aggregatedStore,\n this.mediators.mediatorMetadataAccumulate,\n this.mediators.mediatorRdfResolveHypermediaLinks,\n this.mediators.mediatorRdfResolveHypermediaLinksQueue,\n );\n if (aggregatedStore) {\n aggregatedStore.started = true;\n\n // Kickstart this iterator when derived iterators are created from the aggregatedStore,\n // otherwise the traversal process will not start if this iterator is not the first one to be consumed.\n const listener = (): void => it.kickstart();\n aggregatedStore.addIteratorCreatedListener(listener);\n it.on('end', () => aggregatedStore.removeIteratorCreatedListener(listener));\n }\n\n return it;\n }\n\n public queryQuads(operation: Algebra.Operation, context: IActionContext): AsyncIterator<RDF.Quad> {\n return new TransformIterator(async() => {\n const source = await this.getSourceCached({ url: this.firstUrl }, {}, context, this.getAggregateStore(context));\n return source.source.queryQuads(operation, context);\n });\n }\n\n public async queryBoolean(operation: Algebra.Ask, context: IActionContext): Promise<boolean> {\n const source = await this.getSourceCached({ url: this.firstUrl }, {}, context, this.getAggregateStore(context));\n return await source.source.queryBoolean(operation, context);\n }\n\n public async queryVoid(operation: Algebra.Update, context: IActionContext): Promise<void> {\n const source = await this.getSourceCached({ url: this.firstUrl }, {}, context, this.getAggregateStore(context));\n return await source.source.queryVoid(operation, context);\n }\n\n /\n Resolve a source for the given URL.\n * @param link A source link.\n * @param handledDatasets A hash of dataset identifiers that have already been handled.\n * @param context The action context.\n * @param aggregatedStore An optional aggregated store.\n /\n public async getSource(\n link: ILink,\n handledDatasets: Record<string, boolean>,\n context: IActionContext,\n aggregatedStore: IAggregatedStore \| undefined,\n ): Promise<ISourceState> {\n // Include context entries from link\n if (link.context) {\n context = context.merge(link.context);\n }\n\n // Get the RDF representation of the given document\n let url = link.url;\n let quads: RDF.Stream;\n let metadata: Record<string, any>;\n try {\n const dereferenceRdfOutput: IActorDereferenceRdfOutput = await this.mediators.mediatorDereferenceRdf\n .mediate({ context, url });\n url = dereferenceRdfOutput.url;\n\n // Determine the metadata\n const rdfMetadataOutput: IActorRdfMetadataOutput = await this.mediators.mediatorMetadata.mediate(\n { context, url, quads: dereferenceRdfOutput.data, triples: dereferenceRdfOutput.metadata?.triples },\n );\n\n rdfMetadataOutput.data.on('error', () => {\n // Silence errors in the data stream,\n // as they will be emitted again in the metadata stream,\n // and will result in a promise rejection anyways.\n // If we don't do this, we end up with an unhandled error message\n });\n\n metadata = (await this.mediators.mediatorMetadataExtract.mediate({\n context,\n url,\n // The problem appears to be conflicting metadata keys here\n metadata: rdfMetadataOutput.metadata,\n headers: dereferenceRdfOutput.headers,\n requestTime: dereferenceRdfOutput.requestTime,\n })).metadata;\n quads = rdfMetadataOutput.data;\n\n // Optionally filter the resulting data\n if (link.transform) {\n quads = await link.transform(quads);\n }\n } catch (error: unknown) {\n // Make sure that dereference errors are only emitted once an actor really needs the read quads\n // This for example allows SPARQL endpoints that error on service description fetching to still be source-forcible\n quads = new Readable();\n quads.read = () => {\n setTimeout(() => quads.emit('error', error));\n return null;\n };\n ({ metadata } = await this.mediators.mediatorMetadataAccumulate.mediate({ context, mode: 'initialize' }));\n\n // Log as warning, because the quads above may not always be consumed (e.g. for SPARQL endpoints),\n // so the user would not be notified of something going wrong otherwise.\n this.logWarning(`Metadata extraction for ${url} failed: ${(<Error> error).message}`);\n }\n\n // Aggregate all discovered quads into a store.\n aggregatedStore?.setBaseMetadata(<MetadataBindings> metadata, false);\n aggregatedStore?.containedSources.add(link.url);\n aggregatedStore?.import(quads);\n\n // Determine the source\n const { source, dataset } = await this.mediators.mediatorQuerySourceIdentifyHypermedia.mediate({\n context,\n forceSourceType: link.url === this.firstUrl ? this.forceSourceType : undefined,\n handledDatasets,\n metadata,\n quads,\n url,\n });\n\n if (dataset) {\n // Mark the dataset as applied\n // This is needed to make sure that things like QPF search forms are only applied once,\n // and next page links are followed after that.\n handledDatasets[dataset] = true;\n }\n\n return { link, source, metadata: <MetadataBindings> metadata, handledDatasets };\n }\n\n /\n Resolve a source for the given URL.\n * This will first try to retrieve the source from cache.\n * @param link A source ILink.\n * @param handledDatasets A hash of dataset identifiers that have already been handled.\n * @param context The action context.\n * @param aggregatedStore An optional aggregated store.\n */\n protected getSourceCached(\n link: ILink,\n handledDatasets: Record<string, boolean>,\n context: IActionContext,\n aggregatedStore: IAggregatedStore \| undefined,\n ): Promise<ISourceState> {\n let source = this.sourcesState.get(link.url);\n if (source) {\n return source;\n }\n source = this.getSource(link, handledDatasets, context, aggregatedStore);\n if (link.url === this.firstUrl \|\| aggregatedStore === undefined) {\n this.sourcesState.set(link.url, source);\n }\n return source;\n }\n\n public getAggregateStore(context: IActionContext): IAggregatedStore \| undefined {\n let aggregatedStore: IAggregatedStore \| undefined;\n if (this.aggregateStore) {\n const aggregatedStores: Map<string, IAggregatedStore> \| undefined = context\n .get(KeysQuerySourceIdentify.hypermediaSourcesAggregatedStores);\n if (aggregatedStores) {\n aggregatedStore = aggregatedStores.get(this.firstUrl);\n if (!aggregatedStore) {\n aggregatedStore = new StreamingStoreMetadata(\n undefined,\n async(accumulatedMetadata, appendingMetadata) => <MetadataBindings>\n (await this.mediators.mediatorMetadataAccumulate.mediate({\n mode: 'append',\n accumulatedMetadata,\n appendingMetadata,\n context,\n })).metadata,\n );\n aggregatedStores.set(this.firstUrl, aggregatedStore);\n }\n return aggregatedStore;\n }\n }\n }\n\n public toString(): string {\n return `QuerySourceHypermedia(${this.firstUrl})`;\n }\n}\n\nexport interface IMediatorArgs {\n mediatorDereferenceRdf: MediatorDereferenceRdf;\n mediatorMetadata: MediatorRdfMetadata;\n mediatorMetadataExtract: MediatorRdfMetadataExtract;\n mediatorMetadataAccumulate: MediatorRdfMetadataAccumulate;\n mediatorQuerySourceIdentifyHypermedia: MediatorQuerySourceIdentifyHypermedia;\n mediatorRdfResolveHypermediaLinks: MediatorRdfResolveHypermediaLinks;\n mediatorRdfResolveHypermediaLinksQueue: MediatorRdfResolveHypermediaLinksQueue;\n}\n"]}
1	+ {"version":3,"file":"QuerySourceHypermedia.js","sourceRoot":"","sources":["QuerySourceHypermedia.ts"],"names":[],"mappings":";;;AAAA,mGAA+E;AAS/E,+DAAoE;AAapE,iDAAkD;AAClD,yCAAqC;AACrC,qDAA2C;AAG3C,yGAAsG;AACtG,qEAAkE;AAElE,MAAa,qBAAqB;IAiBhC,YACE,SAAiB,EACjB,QAAgB,EAChB,eAAmC,EACnC,YAAoB,EACpB,cAAuB,EACvB,SAAwB,EACxB,UAA4C,EAC5C,eAAgC;QAEhC,IAAI,CAAC,cAAc,GAAG,QAAQ,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,YAAY,GAAG,IAAI,oBAAQ,CAAgC,EAAE,GAAG,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;IAC3F,CAAC;IAEM,KAAK,CAAC,gBAAgB,CAAC,OAAuB;QACnD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;QAChH,OAAO,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;IACjD,CAAC;IAEM,aAAa,CAClB,SAA4B,EAC5B,OAAuB,EACvB,OAA+B;QAE/B,gEAAgE;QAChE,MAAM,eAAe,GAAiC,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACtF,IAAI,eAAe,IAAI,SAAS,CAAC,IAAI,KAAK,SAAS,IAAI,eAAe,CAAC,OAAO,EAAE,CAAC;YAC/E,OAAO,IAAI,oDAAgB,CAAC,eAAe,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC,aAAa,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACvG,CAAC;QAED,6CAA6C;QAC7C,IAAI,IAAI,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC,eAAe,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,eAAe,CAAC;iBACvE,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QACvC,CAAC;QAED,MAAM,EAAE,GAA6C,IAAI,mFAAwC,CAC/F,IAAI,CAAC,SAAS,EACd,SAAS,EACT,OAAO,EACP,OAAO,EACP,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,YAAY,EACjB,CAAC,IAAI,EAAE,eAAe,EAAE,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,eAAe,EAAE,OAAO,EAAE,eAAe,CAAC,EAChG,eAAe,EACf,IAAI,CAAC,SAAS,CAAC,0BAA0B,EACzC,IAAI,CAAC,SAAS,CAAC,iCAAiC,EAChD,IAAI,CAAC,SAAS,CAAC,sCAAsC,CACtD,CAAC;QACF,IAAI,eAAe,EAAE,CAAC;YACpB,eAAe,CAAC,OAAO,GAAG,IAAI,CAAC;YAE/B,uFAAuF;YACvF,uGAAuG;YACvG,MAAM,QAAQ,GAAG,GAAS,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC;YAC5C,eAAe,CAAC,0BAA0B,CAAC,QAAQ,CAAC,CAAC;YACrD,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,eAAe,CAAC,6BAA6B,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC9E,CAAC;QAED,OAAO,EAAE,CAAC;IACZ,CAAC;IAEM,UAAU,CAAC,SAA4B,EAAE,OAAuB;QACrE,OAAO,IAAI,iCAAiB,CAAC,KAAK,IAAG,EAAE;YACrC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;YAChH,OAAO,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,YAAY,CAAC,SAAsB,EAAE,OAAuB;QACvE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;QAChH,OAAO,MAAM,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC9D,CAAC;IAEM,KAAK,CAAC,SAAS,CAAC,SAAyB,EAAE,OAAuB;QACvE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;QAChH,OAAO,MAAM,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC3D,CAAC;IAED;;;;;;OAMG;IACI,KAAK,CAAC,SAAS,CACpB,IAAW,EACX,eAAwC,EACxC,OAAuB,EACvB,eAA6C;QAE7C,oCAAoC;QACpC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACxC,CAAC;QAED,mDAAmD;QACnD,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;QACnB,IAAI,KAAiB,CAAC;QACtB,IAAI,QAA6B,CAAC;QAClC,IAAI,CAAC;YACH,MAAM,oBAAoB,GAA+B,MAAM,IAAI,CAAC,SAAS,CAAC,sBAAsB;iBACjG,OAAO,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;YAC7B,GAAG,GAAG,oBAAoB,CAAC,GAAG,CAAC;YAE/B,yBAAyB;YACzB,MAAM,iBAAiB,GAA4B,MAAM,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,OAAO,CAC9F,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,oBAAoB,CAAC,IAAI,EAAE,OAAO,EAAE,oBAAoB,CAAC,QAAQ,EAAE,OAAO,EAAE,CACpG,CAAC;YAEF,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBACtC,qCAAqC;gBACrC,wDAAwD;gBACxD,kDAAkD;gBAClD,iEAAiE;YACnE,CAAC,CAAC,CAAC;YAEH,QAAQ,GAAG,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC,OAAO,CAAC;gBAC/D,OAAO;gBACP,GAAG;gBACH,2DAA2D;gBAC3D,QAAQ,EAAE,iBAAiB,CAAC,QAAQ;gBACpC,OAAO,EAAE,oBAAoB,CAAC,OAAO;gBACrC,WAAW,EAAE,oBAAoB,CAAC,WAAW;aAC9C,CAAC,CAAC,CAAC,QAAQ,CAAC;YACb,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC;YAE/B,uCAAuC;YACvC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACnB,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,+FAA+F;YAC/F,kHAAkH;YAClH,KAAK,GAAG,IAAI,0BAAQ,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,GAAG,GAAG,EAAE;gBAChB,UAAU,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC7C,OAAO,IAAI,CAAC;YACd,CAAC,CAAC;YACF,CAAC,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,0BAA0B,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC;YAE1G,kGAAkG;YAClG,wEAAwE;YACxE,IAAI,CAAC,UAAU,CAAC,2BAA2B,GAAG,YAAqB,KAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QACvF,CAAC;QAED,+CAA+C;QAC/C,eAAe,EAAE,eAAe,CAAoB,QAAQ,EAAE,KAAK,CAAC,CAAC;QACrE,eAAe,EAAE,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChD,eAAe,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QAE/B,uBAAuB;QACvB,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,qCAAqC,CAAC,OAAO,CAAC;YAC7F,OAAO;YACP,eAAe,EAAE,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS;YAC9E,eAAe;YACf,QAAQ;YACR,KAAK;YACL,GAAG;SACJ,CAAC,CAAC;QAEH,IAAI,OAAO,EAAE,CAAC;YACZ,8BAA8B;YAC9B,uFAAuF;YACvF,+CAA+C;YAC/C,eAAe,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC;QAClC,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAqB,QAAQ,EAAE,eAAe,EAAE,CAAC;IAClF,CAAC;IAED;;;;;;;OAOG;IACO,eAAe,CACvB,IAAW,EACX,eAAwC,EACxC,OAAuB,EACvB,eAA6C;QAE7C,IAAI,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC7C,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,eAAe,EAAE,OAAO,EAAE,eAAe,CAAC,CAAC;QACzE,IAAI,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,QAAQ,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;YAChE,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAC1C,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEM,iBAAiB,CAAC,OAAuB;QAC9C,IAAI,eAA6C,CAAC;QAClD,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,MAAM,gBAAgB,GAA8C,OAAO;iBACxE,GAAG,CAAC,yCAAuB,CAAC,iCAAiC,CAAC,CAAC;YAClE,IAAI,gBAAgB,EAAE,CAAC;gBACrB,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACtD,IAAI,CAAC,eAAe,EAAE,CAAC;oBACrB,eAAe,GAAG,IAAI,+CAAsB,CAC1C,SAAS,EACT,KAAK,EAAC,mBAAmB,EAAE,iBAAiB,EAAE,EAAE,CAC9C,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,0BAA0B,CAAC,OAAO,CAAC;wBACvD,IAAI,EAAE,QAAQ;wBACd,mBAAmB;wBACnB,iBAAiB;wBACjB,OAAO;qBACR,CAAC,CAAC,CAAC,QAAQ,CACf,CAAC;oBACF,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;gBACvD,CAAC;gBACD,OAAO,eAAe,CAAC;YACzB,CAAC;QACH,CAAC;IACH,CAAC;IAEM,QAAQ;QACb,OAAO,yBAAyB,IAAI,CAAC,QAAQ,GAAG,CAAC;IACnD,CAAC;CACF;AA3PD,sDA2PC","sourcesContent":["import { QuerySourceRdfJs } from '@comunica/actor-query-source-identify-rdfjs';\nimport type { BindingsFactory } from '@comunica/bindings-factory';\nimport type { IActorDereferenceRdfOutput, MediatorDereferenceRdf } from '@comunica/bus-dereference-rdf';\nimport type { MediatorQuerySourceIdentifyHypermedia } from '@comunica/bus-query-source-identify-hypermedia';\nimport type { IActorRdfMetadataOutput, MediatorRdfMetadata } from '@comunica/bus-rdf-metadata';\nimport type { MediatorRdfMetadataAccumulate } from '@comunica/bus-rdf-metadata-accumulate';\nimport type { MediatorRdfMetadataExtract } from '@comunica/bus-rdf-metadata-extract';\nimport type { MediatorRdfResolveHypermediaLinks } from '@comunica/bus-rdf-resolve-hypermedia-links';\nimport type { MediatorRdfResolveHypermediaLinksQueue } from '@comunica/bus-rdf-resolve-hypermedia-links-queue';\nimport { KeysQuerySourceIdentify } from '@comunica/context-entries';\nimport type {\n BindingsStream,\n FragmentSelectorShape,\n IActionContext,\n IAggregatedStore,\n IQueryBindingsOptions,\n IQuerySource,\n MetadataBindings,\n ILink,\n} from '@comunica/types';\nimport type * as RDF from '@rdfjs/types';\nimport type { AsyncIterator } from 'asynciterator';\nimport { TransformIterator } from 'asynciterator';\nimport { LRUCache } from 'lru-cache';\nimport { Readable } from 'readable-stream';\nimport type { Algebra } from 'sparqlalgebrajs';\nimport type { ISourceState } from './LinkedRdfSourcesAsyncRdfIterator';\nimport { MediatedLinkedRdfSourcesAsyncRdfIterator } from './MediatedLinkedRdfSourcesAsyncRdfIterator';\nimport { StreamingStoreMetadata } from './StreamingStoreMetadata';\n\nexport class QuerySourceHypermedia implements IQuerySource {\n public readonly referenceValue: string;\n public readonly firstUrl: string;\n public readonly forceSourceType?: string;\n public readonly aggregateStore: boolean;\n public readonly mediators: IMediatorArgs;\n public readonly logWarning: (warningMessage: string) => void;\n public readonly bindingsFactory: BindingsFactory;\n\n /*\n A cache for source URLs to source states.\n /\n public sourcesState: LRUCache<string, Promise<ISourceState>>;\n\n private readonly cacheSize: number;\n private readonly maxIterators: number;\n\n public constructor(\n cacheSize: number,\n firstUrl: string,\n forceSourceType: string \| undefined,\n maxIterators: number,\n aggregateStore: boolean,\n mediators: IMediatorArgs,\n logWarning: (warningMessage: string) => void,\n bindingsFactory: BindingsFactory,\n ) {\n this.referenceValue = firstUrl;\n this.cacheSize = cacheSize;\n this.firstUrl = firstUrl;\n this.forceSourceType = forceSourceType;\n this.maxIterators = maxIterators;\n this.mediators = mediators;\n this.aggregateStore = aggregateStore;\n this.logWarning = logWarning;\n this.bindingsFactory = bindingsFactory;\n this.sourcesState = new LRUCache<string, Promise<ISourceState>>({ max: this.cacheSize });\n }\n\n public async getSelectorShape(context: IActionContext): Promise<FragmentSelectorShape> {\n const source = await this.getSourceCached({ url: this.firstUrl }, {}, context, this.getAggregateStore(context));\n return source.source.getSelectorShape(context);\n }\n\n public queryBindings(\n operation: Algebra.Operation,\n context: IActionContext,\n options?: IQueryBindingsOptions,\n ): BindingsStream {\n // Optimized match with aggregated store if enabled and started.\n const aggregatedStore: IAggregatedStore \| undefined = this.getAggregateStore(context);\n if (aggregatedStore && operation.type === 'pattern' && aggregatedStore.started) {\n return new QuerySourceRdfJs(aggregatedStore, this.bindingsFactory).queryBindings(operation, context);\n }\n\n // Initialize the sources state on first call\n if (this.sourcesState.size === 0) {\n this.getSourceCached({ url: this.firstUrl }, {}, context, aggregatedStore)\n .catch(error => it.destroy(error));\n }\n\n const it: MediatedLinkedRdfSourcesAsyncRdfIterator = new MediatedLinkedRdfSourcesAsyncRdfIterator(\n this.cacheSize,\n operation,\n options,\n context,\n this.forceSourceType,\n this.firstUrl,\n this.maxIterators,\n (link, handledDatasets) => this.getSourceCached(link, handledDatasets, context, aggregatedStore),\n aggregatedStore,\n this.mediators.mediatorMetadataAccumulate,\n this.mediators.mediatorRdfResolveHypermediaLinks,\n this.mediators.mediatorRdfResolveHypermediaLinksQueue,\n );\n if (aggregatedStore) {\n aggregatedStore.started = true;\n\n // Kickstart this iterator when derived iterators are created from the aggregatedStore,\n // otherwise the traversal process will not start if this iterator is not the first one to be consumed.\n const listener = (): void => it.kickstart();\n aggregatedStore.addIteratorCreatedListener(listener);\n it.on('end', () => aggregatedStore.removeIteratorCreatedListener(listener));\n }\n\n return it;\n }\n\n public queryQuads(operation: Algebra.Operation, context: IActionContext): AsyncIterator<RDF.Quad> {\n return new TransformIterator(async() => {\n const source = await this.getSourceCached({ url: this.firstUrl }, {}, context, this.getAggregateStore(context));\n return source.source.queryQuads(operation, context);\n });\n }\n\n public async queryBoolean(operation: Algebra.Ask, context: IActionContext): Promise<boolean> {\n const source = await this.getSourceCached({ url: this.firstUrl }, {}, context, this.getAggregateStore(context));\n return await source.source.queryBoolean(operation, context);\n }\n\n public async queryVoid(operation: Algebra.Update, context: IActionContext): Promise<void> {\n const source = await this.getSourceCached({ url: this.firstUrl }, {}, context, this.getAggregateStore(context));\n return await source.source.queryVoid(operation, context);\n }\n\n /\n Resolve a source for the given URL.\n * @param link A source link.\n * @param handledDatasets A hash of dataset identifiers that have already been handled.\n * @param context The action context.\n * @param aggregatedStore An optional aggregated store.\n /\n public async getSource(\n link: ILink,\n handledDatasets: Record<string, boolean>,\n context: IActionContext,\n aggregatedStore: IAggregatedStore \| undefined,\n ): Promise<ISourceState> {\n // Include context entries from link\n if (link.context) {\n context = context.merge(link.context);\n }\n\n // Get the RDF representation of the given document\n let url = link.url;\n let quads: RDF.Stream;\n let metadata: Record<string, any>;\n try {\n const dereferenceRdfOutput: IActorDereferenceRdfOutput = await this.mediators.mediatorDereferenceRdf\n .mediate({ context, url });\n url = dereferenceRdfOutput.url;\n\n // Determine the metadata\n const rdfMetadataOutput: IActorRdfMetadataOutput = await this.mediators.mediatorMetadata.mediate(\n { context, url, quads: dereferenceRdfOutput.data, triples: dereferenceRdfOutput.metadata?.triples },\n );\n\n rdfMetadataOutput.data.on('error', () => {\n // Silence errors in the data stream,\n // as they will be emitted again in the metadata stream,\n // and will result in a promise rejection anyways.\n // If we don't do this, we end up with an unhandled error message\n });\n\n metadata = (await this.mediators.mediatorMetadataExtract.mediate({\n context,\n url,\n // The problem appears to be conflicting metadata keys here\n metadata: rdfMetadataOutput.metadata,\n headers: dereferenceRdfOutput.headers,\n requestTime: dereferenceRdfOutput.requestTime,\n })).metadata;\n quads = rdfMetadataOutput.data;\n\n // Optionally filter the resulting data\n if (link.transform) {\n quads = await link.transform(quads);\n }\n } catch (error: unknown) {\n // Make sure that dereference errors are only emitted once an actor really needs the read quads\n // This for example allows SPARQL endpoints that error on service description fetching to still be source-forcible\n quads = new Readable();\n quads.read = () => {\n setTimeout(() => quads.emit('error', error));\n return null;\n };\n ({ metadata } = await this.mediators.mediatorMetadataAccumulate.mediate({ context, mode: 'initialize' }));\n\n // Log as warning, because the quads above may not always be consumed (e.g. for SPARQL endpoints),\n // so the user would not be notified of something going wrong otherwise.\n this.logWarning(`Metadata extraction for ${url} failed: ${(<Error> error).message}`);\n }\n\n // Aggregate all discovered quads into a store.\n aggregatedStore?.setBaseMetadata(<MetadataBindings> metadata, false);\n aggregatedStore?.containedSources.add(link.url);\n aggregatedStore?.import(quads);\n\n // Determine the source\n const { source, dataset } = await this.mediators.mediatorQuerySourceIdentifyHypermedia.mediate({\n context,\n forceSourceType: link.url === this.firstUrl ? this.forceSourceType : undefined,\n handledDatasets,\n metadata,\n quads,\n url,\n });\n\n if (dataset) {\n // Mark the dataset as applied\n // This is needed to make sure that things like QPF search forms are only applied once,\n // and next page links are followed after that.\n handledDatasets[dataset] = true;\n }\n\n return { link, source, metadata: <MetadataBindings> metadata, handledDatasets };\n }\n\n /\n Resolve a source for the given URL.\n * This will first try to retrieve the source from cache.\n * @param link A source ILink.\n * @param handledDatasets A hash of dataset identifiers that have already been handled.\n * @param context The action context.\n * @param aggregatedStore An optional aggregated store.\n */\n protected getSourceCached(\n link: ILink,\n handledDatasets: Record<string, boolean>,\n context: IActionContext,\n aggregatedStore: IAggregatedStore \| undefined,\n ): Promise<ISourceState> {\n let source = this.sourcesState.get(link.url);\n if (source) {\n return source;\n }\n source = this.getSource(link, handledDatasets, context, aggregatedStore);\n if (link.url === this.firstUrl \|\| aggregatedStore === undefined) {\n this.sourcesState.set(link.url, source);\n }\n return source;\n }\n\n public getAggregateStore(context: IActionContext): IAggregatedStore \| undefined {\n let aggregatedStore: IAggregatedStore \| undefined;\n if (this.aggregateStore) {\n const aggregatedStores: Map<string, IAggregatedStore> \| undefined = context\n .get(KeysQuerySourceIdentify.hypermediaSourcesAggregatedStores);\n if (aggregatedStores) {\n aggregatedStore = aggregatedStores.get(this.firstUrl);\n if (!aggregatedStore) {\n aggregatedStore = new StreamingStoreMetadata(\n undefined,\n async(accumulatedMetadata, appendingMetadata) => <MetadataBindings>\n (await this.mediators.mediatorMetadataAccumulate.mediate({\n mode: 'append',\n accumulatedMetadata,\n appendingMetadata,\n context,\n })).metadata,\n );\n aggregatedStores.set(this.firstUrl, aggregatedStore);\n }\n return aggregatedStore;\n }\n }\n }\n\n public toString(): string {\n return `QuerySourceHypermedia(${this.firstUrl})`;\n }\n}\n\nexport interface IMediatorArgs {\n mediatorDereferenceRdf: MediatorDereferenceRdf;\n mediatorMetadata: MediatorRdfMetadata;\n mediatorMetadataExtract: MediatorRdfMetadataExtract;\n mediatorMetadataAccumulate: MediatorRdfMetadataAccumulate;\n mediatorQuerySourceIdentifyHypermedia: MediatorQuerySourceIdentifyHypermedia;\n mediatorRdfResolveHypermediaLinks: MediatorRdfResolveHypermediaLinks;\n mediatorRdfResolveHypermediaLinksQueue: MediatorRdfResolveHypermediaLinksQueue;\n}\n"]}

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@comunica/actor-query-source-identify-hypermedia",
-  "version": "3.2.2",
+  "version": "3.3.0",
   "description": "A hypermedia query-source-identify actor",
   "lsd:module": true,
   "license": "MIT",
@@ -37,22 +37,22 @@
     "build:components": "componentsjs-generator"
   },
   "dependencies": {
-    "@comunica/actor-query-source-identify-rdfjs": "^3.2.2",
-    "@comunica/bindings-factory": "^3.2.1",
-    "@comunica/bus-dereference-rdf": "^3.2.1",
-    "@comunica/bus-merge-bindings-context": "^3.2.1",
-    "@comunica/bus-query-operation": "^3.2.2",
-    "@comunica/bus-query-source-identify": "^3.2.2",
-    "@comunica/bus-query-source-identify-hypermedia": "^3.2.1",
-    "@comunica/bus-rdf-metadata": "^3.2.1",
-    "@comunica/bus-rdf-metadata-accumulate": "^3.2.1",
-    "@comunica/bus-rdf-metadata-extract": "^3.2.1",
-    "@comunica/bus-rdf-resolve-hypermedia-links": "^3.2.1",
-    "@comunica/bus-rdf-resolve-hypermedia-links-queue": "^3.2.1",
-    "@comunica/context-entries": "^3.2.1",
-    "@comunica/core": "^3.2.1",
-    "@comunica/metadata": "^3.2.1",
-    "@comunica/types": "^3.2.1",
+    "@comunica/actor-query-source-identify-rdfjs": "^3.3.0",
+    "@comunica/bindings-factory": "^3.3.0",
+    "@comunica/bus-dereference-rdf": "^3.3.0",
+    "@comunica/bus-merge-bindings-context": "^3.3.0",
+    "@comunica/bus-query-operation": "^3.3.0",
+    "@comunica/bus-query-source-identify": "^3.3.0",
+    "@comunica/bus-query-source-identify-hypermedia": "^3.3.0",
+    "@comunica/bus-rdf-metadata": "^3.3.0",
+    "@comunica/bus-rdf-metadata-accumulate": "^3.3.0",
+    "@comunica/bus-rdf-metadata-extract": "^3.3.0",
+    "@comunica/bus-rdf-resolve-hypermedia-links": "^3.3.0",
+    "@comunica/bus-rdf-resolve-hypermedia-links-queue": "^3.3.0",
+    "@comunica/context-entries": "^3.3.0",
+    "@comunica/core": "^3.3.0",
+    "@comunica/metadata": "^3.3.0",
+    "@comunica/types": "^3.3.0",
     "@rdfjs/types": "*",
     "asynciterator": "^3.9.0",
     "lru-cache": "^10.0.0",
@@ -61,5 +61,5 @@
     "readable-stream": "^4.4.2",
     "sparqlalgebrajs": "^4.3.7"
   },
-  "gitHead": "fbcc3a81f87738633ddf69ede5ca504236f7edd9"
+  "gitHead": "02bde397d206f1f5a523643a6a604c89e792e2f9"
 }