@comunica/actor-query-source-identify-hypermedia 3.1.0 → 3.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -44,8 +44,19 @@ class LinkedRdfSourcesAsyncRdfIterator extends asynciterator_1.BufferedIterator
|
|
|
44
44
|
const bindingsStream = sourceState.source.queryBindings(this.operation, this.context);
|
|
45
45
|
bindingsStream.getProperty('metadata', (metadata) => {
|
|
46
46
|
metadata.state = new metadata_1.MetadataValidationState();
|
|
47
|
-
resolve(metadata);
|
|
48
47
|
bindingsStream.destroy();
|
|
48
|
+
this.accumulateMetadata(sourceState.metadata, metadata)
|
|
49
|
+
.then((accumulatedMetadata) => {
|
|
50
|
+
// Also merge fields that were not explicitly accumulated
|
|
51
|
+
const returnMetadata = { ...sourceState.metadata, ...metadata, ...accumulatedMetadata };
|
|
52
|
+
resolve(returnMetadata);
|
|
53
|
+
})
|
|
54
|
+
.catch(() => {
|
|
55
|
+
resolve({
|
|
56
|
+
...sourceState.metadata,
|
|
57
|
+
state: new metadata_1.MetadataValidationState(),
|
|
58
|
+
});
|
|
59
|
+
});
|
|
49
60
|
});
|
|
50
61
|
})
|
|
51
62
|
.catch(reject);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LinkedRdfSourcesAsyncRdfIterator.js","sourceRoot":"","sources":["LinkedRdfSourcesAsyncRdfIterator.ts"],"names":[],"mappings":";;;AAEA,iDAA6D;AAI7D,iDAAiD;AAGjD,MAAsB,gCAAiC,SAAQ,gCAA8B;IAkB3F,YACE,SAAiB,EACjB,SAA4B,EAC5B,oBAAuD,EACvD,OAAuB,EACvB,QAAgB,EAChB,YAAoB,EACpB,iBAAoC,EACpC,OAAiC;QAEjC,KAAK,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;QAlBhC,YAAO,GAAG,KAAK,CAAC;QACT,qBAAgB,GAAkC,EAAE,CAAC;QAC9D,6BAAwB,GAAG,CAAC,CAAC;QAC7B,gCAA2B,GAAG,CAAC,CAAC;QACxC,wDAAwD;QAChD,wBAAmB,GAA0C,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAc9F,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;QACjD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAE3C,IAAI,IAAI,CAAC,YAAY,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,mFAAmF,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;QAC1H,CAAC;IACH,CAAC;IAED;;OAEG;IACI,SAAS;QACd,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;IAEe,WAAW,CAAI,YAAoB,EAAE,QAA6B;QAChF,IAAI,YAAY,KAAK,UAAU,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACjD,0GAA0G;YAC1G,iDAAiD;YACjD,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBAC5B,IAAI,CAAC,iBAAiB,GAAG,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;oBACvD,IAAI,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC;yBAC/C,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE;wBACpB,6EAA6E;wBAC7E,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;wBACtF,cAAc,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,QAA0B,EAAE,EAAE;4BACpE,QAAQ,CAAC,KAAK,GAAG,IAAI,kCAAuB,EAAE,CAAC;4BAC/C,OAAO,CAAC,QAAQ,CAAC,CAAC;4BAClB,cAAc,CAAC,OAAO,EAAE,CAAC;wBAC3B,CAAC,CAAC,CAAC;oBACL,CAAC,CAAC;yBACD,KAAK,CAAC,MAAM,CAAC,CAAC;gBACnB,CAAC,CAAC,CAAC;YACL,CAAC;YACD,IAAI,CAAC,iBAAiB;iBACnB,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;iBACxD,KAAK,CAAC,GAAG,EAAE;gBACV,gBAAgB;YAClB,CAAC,CAAC,CAAC;QACP,CAAC;QACD,OAAO,KAAK,CAAC,WAAW,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;IACnD,CAAC;IAEkB,IAAI,CAAC,OAAiB;QACvC,8BAA8B;QAC9B,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACvC,EAAE,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACtB,CAAC;IAce,KAAK,CAAC,KAAa,EAAE,IAAgB;QACnD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,kCAAkC;YAClC,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC7C,OAAO,KAAK,GAAG,CAAC,EAAE,CAAC;oBACjB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;oBAC7B,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;wBAClB,MAAM;oBACR,CAAC;yBAAM,CAAC;wBACN,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBACnB,CAAC;gBACH,CAAC;gBACD,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;oBACf,MAAM;gBACR,CAAC;YACH,CAAC;YAED,mCAAmC;YACnC,IAAI,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC,mBAAmB,EAAE,EAAE,CAAC;gBAC7C,gFAAgF;gBAChF,wDAAwD;gBACxD,mDAAmD;gBACnD,IAAI,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC;qBAC/C,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE;oBACpB,IAAI,CAAC,yBAAyB,CAAC,WAAW,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;oBACnE,IAAI,EAAE,CAAC;gBACT,CAAC,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACN,IAAI,EAAE,CAAC;YACT,CAAC;QACH,CAAC;aAAM,CAAC;YACN,0DAA0D;YAC1D,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YAEpB,4DAA4D;YAC5D,IAAI,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC;iBAC/C,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE;gBACpB,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;gBAChC,IAAI,EAAE,CAAC;YACT,CAAC,CAAC;gBACF,4EAA4E;iBAC3E,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAES,mBAAmB;QAC3B,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC,2BAA2B,CAAC;YACtG,IAAI,CAAC,YAAY,IAAI,CAAC,CAAC,IAAI,CAAC,mCAAmC,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACzF,CAAC;IAES,mCAAmC;QAC3C,OAAO,IAAI,CAAC;IACd,CAAC;IAES,mBAAmB;QAC3B,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC,2BAA2B,CAAC,GAAG,CAAC,CAAC;IAC/G,CAAC;IAaD;;;;OAIG;IACO,aAAa,CAAC,WAAyB;QAC/C,sDAAsD;QACtD,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAC3G,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACrC,IAAI,gBAAgB,GAAG,KAAK,CAAC;YAC7B,IAAI,gBAAgB,GAAG,KAAK,CAAC;YAE7B,+CAA+C;YACzC,QAAS,CAAC,YAAY,GAAG,IAAI,CAAC;YACpC,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;YAC5D,QAAQ,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YAClD,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACtB,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;gBAEzE,mEAAmE;gBACnE,8FAA8F;gBAC9F,gBAAgB,GAAG,IAAI,CAAC;gBACxB,IAAI,CAAC,gBAAgB,EAAE,CAAC;oBACtB,IAAI,CAAC,2BAA2B,EAAE,CAAC;gBACrC,CAAC;gBAED,yEAAyE;gBACzE,IAAI,gBAAgB,EAAE,CAAC;oBACrB,IAAI,CAAC,yBAAyB,CAAC,WAAW,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;gBACpE,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,wCAAwC;YACxC,gDAAgD;YAChD,QAAQ,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,QAA0B,EAAE,EAAE;gBAC9D,iCAAiC;gBACjC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB;qBAChD,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC,KAAK,IAAG,EAAE;oBACnC,IAAI,CAAC,gBAAgB,EAAE,CAAC;wBACtB,gBAAgB,GAAG,WAAW,CAAC,QAAQ,CAAC;oBAC1C,CAAC;oBACD,OAAO,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,QAAQ,CAAC,CAAC;gBAC7D,CAAC,CAAC,EAAE;qBACD,IAAI,CAAC,CAAC,mBAAmB,EAAE,EAAE;oBAC5B,yDAAyD;oBACzD,MAAM,cAAc,GAAG,EAAE,GAAG,WAAW,CAAC,QAAQ,EAAE,GAAG,QAAQ,EAAE,GAAG,mBAAmB,EAAE,CAAC;oBAExF,4BAA4B;oBAC5B,cAAc,CAAC,KAAK,GAAG,IAAI,kCAAuB,EAAE,CAAC;oBAErD,iEAAiE;oBACjE,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC,CAAC;oBAEpC,oCAAoC;oBACpC,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;wBAC3B,IAAI,CAAC,iBAAiB;6BACnB,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;6BACjD,KAAK,CAAC,GAAG,EAAE;4BACV,gBAAgB;wBAClB,CAAC,CAAC,CAAC;oBACP,CAAC;oBAED,wEAAwE;oBACxE,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC;yBAChC,IAAI,CAAC,CAAC,QAAiB,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;yBAClD,IAAI,CAAC,KAAK,EAAC,QAAiB,EAAE,EAAE;wBAC/B,oCAAoC;wBACpC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;wBAC5C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;4BAC/B,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC;wBAC5C,CAAC;wBAED,gBAAgB,GAAG,IAAI,CAAC;wBACxB,IAAI,gBAAgB,EAAE,CAAC;4BACrB,IAAI,CAAC,2BAA2B,EAAE,CAAC;wBACrC,CAAC;wBAED,IAAI,CAAC,yBAAyB,CAAC,WAAW,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;oBACpE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;oBAEzC,OAAO,cAAc,CAAC;gBACxB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;oBACpB,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;oBACpB,OAAyB,EAAE,CAAC;gBAC9B,CAAC,CAAC,CAAC;YACP,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,SAAkB,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO,CAAS,SAAS,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAES,cAAc,CAAC,WAA6B;QACpD,MAAM,oBAAoB,GAAG,IAAI,CAAC,WAAW,CAAmB,UAAU,CAAC,CAAC;QAC5E,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;QAC1C,oBAAoB,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC;IAC3C,CAAC;IAES,SAAS;QACjB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;IACpB,CAAC;IAED;;;;;;OAMG;IACO,yBAAyB,CAAC,eAAwC,EAAE,QAAiB;QAC7F,IAAI,CAAC,YAAY,EAAE;aAChB,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE;YAClB,2CAA2C;YAC3C,OAAO,IAAI,CAAC,mBAAmB,EAAE,IAAI,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC;gBACtD,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,EAAE,CAAC;gBACjC,IAAI,QAAQ,EAAE,CAAC;oBACb,IAAI,CAAC,wBAAwB,EAAE,CAAC;oBAChC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,eAAe,CAAC;yBAC9C,IAAI,CAAC,CAAC,eAAe,EAAE,EAAE;wBACxB,IAAI,CAAC,wBAAwB,EAAE,CAAC;wBAChC,IAAI,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC;oBACtC,CAAC,CAAC;yBACD,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC/C,CAAC;qBAAM,CAAC;oBACN,MAAM;gBACR,CAAC;YACH,CAAC;YAED,sDAAsD;YACtD,IAAI,QAAQ,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,CAAC;gBAClD,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,CAAC;QACH,CAAC,CAAC;aACD,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IACzC,CAAC;IAES,WAAW,CAAC,SAAqB,EAAE,kBAA2B;QACtE,OAAO,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC;IAC5D,CAAC;CACF;AAtTD,4EAsTC","sourcesContent":["import type { ILink } from '@comunica/bus-rdf-resolve-hypermedia-links';\nimport type { ILinkQueue } from '@comunica/bus-rdf-resolve-hypermedia-links-queue';\nimport { MetadataValidationState } from '@comunica/metadata';\nimport type { IQuerySource, IActionContext, MetadataBindings, IQueryBindingsOptions } from '@comunica/types';\nimport type * as RDF from '@rdfjs/types';\nimport type { AsyncIterator, BufferedIteratorOptions } from 'asynciterator';\nimport { BufferedIterator } from 'asynciterator';\nimport type { Algebra } from 'sparqlalgebrajs';\n\nexport abstract class LinkedRdfSourcesAsyncRdfIterator extends BufferedIterator<RDF.Bindings> {\n protected readonly operation: Algebra.Operation;\n protected readonly queryBindingsOptions: IQueryBindingsOptions | undefined;\n protected readonly context: IActionContext;\n\n private readonly cacheSize: number;\n protected readonly firstUrl: string;\n private readonly maxIterators: number;\n private readonly sourceStateGetter: SourceStateGetter;\n\n protected started = false;\n private readonly currentIterators: AsyncIterator<RDF.Bindings>[] = [];\n private iteratorsPendingCreation = 0;\n private iteratorsPendingTermination = 0;\n // eslint-disable-next-line unicorn/no-useless-undefined\n private accumulatedMetadata: Promise<MetadataBindings | undefined> = Promise.resolve(undefined);\n private preflightMetadata: Promise<MetadataBindings> | undefined;\n\n public constructor(\n cacheSize: number,\n operation: Algebra.Operation,\n queryBindingsOptions: IQueryBindingsOptions | undefined,\n context: IActionContext,\n firstUrl: string,\n maxIterators: number,\n sourceStateGetter: SourceStateGetter,\n options?: BufferedIteratorOptions,\n ) {\n super({ autoStart: false, ...options });\n this._reading = false;\n this.cacheSize = cacheSize;\n this.operation = operation;\n this.queryBindingsOptions = queryBindingsOptions;\n this.context = context;\n this.firstUrl = firstUrl;\n this.maxIterators = maxIterators;\n this.sourceStateGetter = sourceStateGetter;\n\n if (this.maxIterators <= 0) {\n throw new Error(`LinkedRdfSourcesAsyncRdfIterator.maxIterators must be larger than zero, but got ${this.maxIterators}`);\n }\n }\n\n /**\n * Start filling the buffer of this iterator.\n */\n public kickstart(): void {\n if (!this.started) {\n this._fillBufferAsync();\n }\n }\n\n public override getProperty<P>(propertyName: string, callback?: (value: P) => void): P | undefined {\n if (propertyName === 'metadata' && !this.started) {\n // If the iterator has not started yet, forcefully fetch the metadata from the source without starting the\n // iterator. This way, we keep the iterator lazy.\n if (!this.preflightMetadata) {\n this.preflightMetadata = new Promise((resolve, reject) => {\n this.sourceStateGetter({ url: this.firstUrl }, {})\n .then((sourceState) => {\n // Don't pass query options, as we don't want to consume any passed iterators\n const bindingsStream = sourceState.source.queryBindings(this.operation, this.context);\n bindingsStream.getProperty('metadata', (metadata: MetadataBindings) => {\n metadata.state = new MetadataValidationState();\n resolve(metadata);\n bindingsStream.destroy();\n });\n })\n .catch(reject);\n });\n }\n this.preflightMetadata\n .then(metadata => this.setProperty('metadata', metadata))\n .catch(() => {\n // Ignore errors\n });\n }\n return super.getProperty(propertyName, callback);\n }\n\n protected override _end(destroy?: boolean): void {\n // Close all running iterators\n for (const it of this.currentIterators) {\n it.destroy();\n }\n\n super._end(destroy);\n }\n\n /**\n * Get the internal link queue.\n * The returned instance must always be the same.\n */\n public abstract getLinkQueue(): Promise<ILinkQueue>;\n\n /**\n * Determine the links to be followed from the current source given its metadata.\n * @param metadata The metadata of a source.\n */\n protected abstract getSourceLinks(metadata: Record<string, any>): Promise<ILink[]>;\n\n public override _read(count: number, done: () => void): void {\n if (this.started) {\n // Read from all current iterators\n for (const iterator of this.currentIterators) {\n while (count > 0) {\n const read = iterator.read();\n if (read === null) {\n break;\n } else {\n count--;\n this._push(read);\n }\n }\n if (count <= 0) {\n break;\n }\n }\n\n // Schedule new iterators if needed\n if (count >= 0 && this.canStartNewIterator()) {\n // We can safely ignore skip catching the error, since we are guaranteed to have\n // successfully got the source for this.firstUrl before.\n // eslint-disable-next-line ts/no-floating-promises\n this.sourceStateGetter({ url: this.firstUrl }, {})\n .then((sourceState) => {\n this.startIteratorsForNextUrls(sourceState.handledDatasets, false);\n done();\n });\n } else {\n done();\n }\n } else {\n // The first time this is called, prepare the first source\n this.started = true;\n\n // Await the source to be set, and start the source iterator\n this.sourceStateGetter({ url: this.firstUrl }, {})\n .then((sourceState) => {\n this.startIterator(sourceState);\n done();\n })\n // Destroy should be async because it can be called before it is listened to\n .catch(error => setTimeout(() => this.destroy(error)));\n }\n }\n\n protected canStartNewIterator(): boolean {\n return (this.currentIterators.length + this.iteratorsPendingCreation + this.iteratorsPendingTermination) <\n this.maxIterators && (!this.canStartNewIteratorConsiderReadable() || !this.readable);\n }\n\n protected canStartNewIteratorConsiderReadable(): boolean {\n return true;\n }\n\n protected areIteratorsRunning(): boolean {\n return (this.currentIterators.length + this.iteratorsPendingCreation + this.iteratorsPendingTermination) > 0;\n }\n\n /**\n * Append the fields from appendingMetadata into accumulatedMetadata.\n * @param accumulatedMetadata The fields to append to.\n * @param appendingMetadata The fields to append.\n * @protected\n */\n protected abstract accumulateMetadata(\n accumulatedMetadata: MetadataBindings,\n appendingMetadata: MetadataBindings,\n ): Promise<MetadataBindings>;\n\n /**\n * Start a new iterator for the given source.\n * Once the iterator is done, it will either determine a new source, or it will close the linked iterator.\n * @param {ISourceState} startSource The start source state.\n */\n protected startIterator(startSource: ISourceState): void {\n // Delegate the quad pattern query to the given source\n try {\n const iterator = startSource.source.queryBindings(this.operation, this.context, this.queryBindingsOptions);\n this.currentIterators.push(iterator);\n let receivedEndEvent = false;\n let receivedMetadata = false;\n\n // Attach readers to the newly created iterator\n (<any>iterator)._destination = this;\n iterator.on('error', (error: Error) => this.destroy(error));\n iterator.on('readable', () => this._fillBuffer());\n iterator.on('end', () => {\n this.currentIterators.splice(this.currentIterators.indexOf(iterator), 1);\n\n // Indicate that this iterator still needs to flush its next-links.\n // Without this, the linked iterator could sometimes be closed before next-links are obtained.\n receivedEndEvent = true;\n if (!receivedMetadata) {\n this.iteratorsPendingTermination++;\n }\n\n // If the metadata was already received, handle the next URL in the queue\n if (receivedMetadata) {\n this.startIteratorsForNextUrls(startSource.handledDatasets, true);\n }\n });\n\n // Listen for the metadata of the source\n // The metadata property is guaranteed to be set\n iterator.getProperty('metadata', (metadata: MetadataBindings) => {\n // Accumulate the metadata object\n this.accumulatedMetadata = this.accumulatedMetadata\n .then(previousMetadata => (async() => {\n if (!previousMetadata) {\n previousMetadata = startSource.metadata;\n }\n return this.accumulateMetadata(previousMetadata, metadata);\n })()\n .then((accumulatedMetadata) => {\n // Also merge fields that were not explicitly accumulated\n const returnMetadata = { ...startSource.metadata, ...metadata, ...accumulatedMetadata };\n\n // Create new metadata state\n returnMetadata.state = new MetadataValidationState();\n\n // Emit metadata, and invalidate any metadata that was set before\n this.updateMetadata(returnMetadata);\n\n // Invalidate any preflight metadata\n if (this.preflightMetadata) {\n this.preflightMetadata\n .then(metadataIn => metadataIn.state.invalidate())\n .catch(() => {\n // Ignore errors\n });\n }\n\n // Determine next urls, which will eventually become a next-next source.\n this.getSourceLinks(returnMetadata)\n .then((nextUrls: ILink[]) => Promise.all(nextUrls))\n .then(async(nextUrls: ILink[]) => {\n // Append all next URLs to our queue\n const linkQueue = await this.getLinkQueue();\n for (const nextUrl of nextUrls) {\n linkQueue.push(nextUrl, startSource.link);\n }\n\n receivedMetadata = true;\n if (receivedEndEvent) {\n this.iteratorsPendingTermination--;\n }\n\n this.startIteratorsForNextUrls(startSource.handledDatasets, true);\n }).catch(error => this.destroy(error));\n\n return returnMetadata;\n })).catch((error) => {\n this.destroy(error);\n return <MetadataBindings>{};\n });\n });\n } catch (syncError: unknown) {\n this.destroy(<Error> syncError);\n }\n }\n\n protected updateMetadata(metadataNew: MetadataBindings): void {\n const metadataToInvalidate = this.getProperty<MetadataBindings>('metadata');\n this.setProperty('metadata', metadataNew);\n metadataToInvalidate?.state.invalidate();\n }\n\n protected isRunning(): boolean {\n return !this.done;\n }\n\n /**\n * Check if a next URL is in the queue.\n * If yes, start a new iterator.\n * If no, close this iterator.\n * @param handledDatasets\n * @param canClose\n */\n protected startIteratorsForNextUrls(handledDatasets: Record<string, boolean>, canClose: boolean): void {\n this.getLinkQueue()\n .then((linkQueue) => {\n // Create as many new iterators as possible\n while (this.canStartNewIterator() && this.isRunning()) {\n const nextLink = linkQueue.pop();\n if (nextLink) {\n this.iteratorsPendingCreation++;\n this.sourceStateGetter(nextLink, handledDatasets)\n .then((nextSourceState) => {\n this.iteratorsPendingCreation--;\n this.startIterator(nextSourceState);\n })\n .catch(error => this.emit('error', error));\n } else {\n break;\n }\n }\n\n // Close, only if no other iterators are still running\n if (canClose && this.isCloseable(linkQueue, true)) {\n this.close();\n }\n })\n .catch(error => this.destroy(error));\n }\n\n protected isCloseable(linkQueue: ILinkQueue, _requireQueueEmpty: boolean): boolean {\n return linkQueue.isEmpty() && !this.areIteratorsRunning();\n }\n}\n\n/**\n * The current state of a source.\n * This is needed for following links within a source.\n */\nexport interface ISourceState {\n /**\n * The link to this source.\n */\n link: ILink;\n /**\n * A source.\n */\n source: IQuerySource;\n /**\n * The source's initial metadata.\n */\n metadata: MetadataBindings;\n /**\n * All dataset identifiers that have been passed for this source.\n */\n handledDatasets: Record<string, boolean>;\n}\n\nexport type SourceStateGetter = (link: ILink, handledDatasets: Record<string, boolean>) => Promise<ISourceState>;\n"]}
|
|
1
|
+
{"version":3,"file":"LinkedRdfSourcesAsyncRdfIterator.js","sourceRoot":"","sources":["LinkedRdfSourcesAsyncRdfIterator.ts"],"names":[],"mappings":";;;AAEA,iDAA6D;AAI7D,iDAAiD;AAGjD,MAAsB,gCAAiC,SAAQ,gCAA8B;IAkB3F,YACE,SAAiB,EACjB,SAA4B,EAC5B,oBAAuD,EACvD,OAAuB,EACvB,QAAgB,EAChB,YAAoB,EACpB,iBAAoC,EACpC,OAAiC;QAEjC,KAAK,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;QAlBhC,YAAO,GAAG,KAAK,CAAC;QACT,qBAAgB,GAAkC,EAAE,CAAC;QAC9D,6BAAwB,GAAG,CAAC,CAAC;QAC7B,gCAA2B,GAAG,CAAC,CAAC;QACxC,wDAAwD;QAChD,wBAAmB,GAA0C,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAc9F,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;QACjD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAE3C,IAAI,IAAI,CAAC,YAAY,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,mFAAmF,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;QAC1H,CAAC;IACH,CAAC;IAED;;OAEG;IACI,SAAS;QACd,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;IAEe,WAAW,CAAI,YAAoB,EAAE,QAA6B;QAChF,IAAI,YAAY,KAAK,UAAU,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACjD,0GAA0G;YAC1G,iDAAiD;YACjD,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBAC5B,IAAI,CAAC,iBAAiB,GAAG,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;oBACvD,IAAI,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC;yBAC/C,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE;wBACpB,6EAA6E;wBAC7E,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;wBACtF,cAAc,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,QAA0B,EAAE,EAAE;4BACpE,QAAQ,CAAC,KAAK,GAAG,IAAI,kCAAuB,EAAE,CAAC;4BAC/C,cAAc,CAAC,OAAO,EAAE,CAAC;4BACzB,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,QAAQ,EAAE,QAAQ,CAAC;iCACpD,IAAI,CAAC,CAAC,mBAAmB,EAAE,EAAE;gCAC5B,yDAAyD;gCACzD,MAAM,cAAc,GAAG,EAAE,GAAG,WAAW,CAAC,QAAQ,EAAE,GAAG,QAAQ,EAAE,GAAG,mBAAmB,EAAE,CAAC;gCACxF,OAAO,CAAC,cAAc,CAAC,CAAC;4BAC1B,CAAC,CAAC;iCACD,KAAK,CAAC,GAAG,EAAE;gCACV,OAAO,CAAC;oCACN,GAAG,WAAW,CAAC,QAAQ;oCACvB,KAAK,EAAE,IAAI,kCAAuB,EAAE;iCACrC,CAAC,CAAC;4BACL,CAAC,CAAC,CAAC;wBACP,CAAC,CAAC,CAAC;oBACL,CAAC,CAAC;yBACD,KAAK,CAAC,MAAM,CAAC,CAAC;gBACnB,CAAC,CAAC,CAAC;YACL,CAAC;YACD,IAAI,CAAC,iBAAiB;iBACnB,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;iBACxD,KAAK,CAAC,GAAG,EAAE;gBACV,gBAAgB;YAClB,CAAC,CAAC,CAAC;QACP,CAAC;QACD,OAAO,KAAK,CAAC,WAAW,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;IACnD,CAAC;IAEkB,IAAI,CAAC,OAAiB;QACvC,8BAA8B;QAC9B,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACvC,EAAE,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACtB,CAAC;IAce,KAAK,CAAC,KAAa,EAAE,IAAgB;QACnD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,kCAAkC;YAClC,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC7C,OAAO,KAAK,GAAG,CAAC,EAAE,CAAC;oBACjB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;oBAC7B,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;wBAClB,MAAM;oBACR,CAAC;yBAAM,CAAC;wBACN,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBACnB,CAAC;gBACH,CAAC;gBACD,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;oBACf,MAAM;gBACR,CAAC;YACH,CAAC;YAED,mCAAmC;YACnC,IAAI,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC,mBAAmB,EAAE,EAAE,CAAC;gBAC7C,gFAAgF;gBAChF,wDAAwD;gBACxD,mDAAmD;gBACnD,IAAI,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC;qBAC/C,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE;oBACpB,IAAI,CAAC,yBAAyB,CAAC,WAAW,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;oBACnE,IAAI,EAAE,CAAC;gBACT,CAAC,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACN,IAAI,EAAE,CAAC;YACT,CAAC;QACH,CAAC;aAAM,CAAC;YACN,0DAA0D;YAC1D,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YAEpB,4DAA4D;YAC5D,IAAI,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC;iBAC/C,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE;gBACpB,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;gBAChC,IAAI,EAAE,CAAC;YACT,CAAC,CAAC;gBACF,4EAA4E;iBAC3E,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAES,mBAAmB;QAC3B,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC,2BAA2B,CAAC;YACtG,IAAI,CAAC,YAAY,IAAI,CAAC,CAAC,IAAI,CAAC,mCAAmC,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACzF,CAAC;IAES,mCAAmC;QAC3C,OAAO,IAAI,CAAC;IACd,CAAC;IAES,mBAAmB;QAC3B,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC,2BAA2B,CAAC,GAAG,CAAC,CAAC;IAC/G,CAAC;IAaD;;;;OAIG;IACO,aAAa,CAAC,WAAyB;QAC/C,sDAAsD;QACtD,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAC3G,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACrC,IAAI,gBAAgB,GAAG,KAAK,CAAC;YAC7B,IAAI,gBAAgB,GAAG,KAAK,CAAC;YAE7B,+CAA+C;YACzC,QAAS,CAAC,YAAY,GAAG,IAAI,CAAC;YACpC,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;YAC5D,QAAQ,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YAClD,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACtB,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;gBAEzE,mEAAmE;gBACnE,8FAA8F;gBAC9F,gBAAgB,GAAG,IAAI,CAAC;gBACxB,IAAI,CAAC,gBAAgB,EAAE,CAAC;oBACtB,IAAI,CAAC,2BAA2B,EAAE,CAAC;gBACrC,CAAC;gBAED,yEAAyE;gBACzE,IAAI,gBAAgB,EAAE,CAAC;oBACrB,IAAI,CAAC,yBAAyB,CAAC,WAAW,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;gBACpE,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,wCAAwC;YACxC,gDAAgD;YAChD,QAAQ,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,QAA0B,EAAE,EAAE;gBAC9D,iCAAiC;gBACjC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB;qBAChD,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC,KAAK,IAAG,EAAE;oBACnC,IAAI,CAAC,gBAAgB,EAAE,CAAC;wBACtB,gBAAgB,GAAG,WAAW,CAAC,QAAQ,CAAC;oBAC1C,CAAC;oBACD,OAAO,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,QAAQ,CAAC,CAAC;gBAC7D,CAAC,CAAC,EAAE;qBACD,IAAI,CAAC,CAAC,mBAAmB,EAAE,EAAE;oBAC5B,yDAAyD;oBACzD,MAAM,cAAc,GAAG,EAAE,GAAG,WAAW,CAAC,QAAQ,EAAE,GAAG,QAAQ,EAAE,GAAG,mBAAmB,EAAE,CAAC;oBAExF,4BAA4B;oBAC5B,cAAc,CAAC,KAAK,GAAG,IAAI,kCAAuB,EAAE,CAAC;oBAErD,iEAAiE;oBACjE,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC,CAAC;oBAEpC,oCAAoC;oBACpC,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;wBAC3B,IAAI,CAAC,iBAAiB;6BACnB,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;6BACjD,KAAK,CAAC,GAAG,EAAE;4BACV,gBAAgB;wBAClB,CAAC,CAAC,CAAC;oBACP,CAAC;oBAED,wEAAwE;oBACxE,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC;yBAChC,IAAI,CAAC,CAAC,QAAiB,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;yBAClD,IAAI,CAAC,KAAK,EAAC,QAAiB,EAAE,EAAE;wBAC/B,oCAAoC;wBACpC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;wBAC5C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;4BAC/B,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC;wBAC5C,CAAC;wBAED,gBAAgB,GAAG,IAAI,CAAC;wBACxB,IAAI,gBAAgB,EAAE,CAAC;4BACrB,IAAI,CAAC,2BAA2B,EAAE,CAAC;wBACrC,CAAC;wBAED,IAAI,CAAC,yBAAyB,CAAC,WAAW,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;oBACpE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;oBAEzC,OAAO,cAAc,CAAC;gBACxB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;oBACpB,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;oBACpB,OAAyB,EAAE,CAAC;gBAC9B,CAAC,CAAC,CAAC;YACP,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,SAAkB,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO,CAAS,SAAS,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAES,cAAc,CAAC,WAA6B;QACpD,MAAM,oBAAoB,GAAG,IAAI,CAAC,WAAW,CAAmB,UAAU,CAAC,CAAC;QAC5E,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;QAC1C,oBAAoB,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC;IAC3C,CAAC;IAES,SAAS;QACjB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;IACpB,CAAC;IAED;;;;;;OAMG;IACO,yBAAyB,CAAC,eAAwC,EAAE,QAAiB;QAC7F,IAAI,CAAC,YAAY,EAAE;aAChB,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE;YAClB,2CAA2C;YAC3C,OAAO,IAAI,CAAC,mBAAmB,EAAE,IAAI,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC;gBACtD,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,EAAE,CAAC;gBACjC,IAAI,QAAQ,EAAE,CAAC;oBACb,IAAI,CAAC,wBAAwB,EAAE,CAAC;oBAChC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,eAAe,CAAC;yBAC9C,IAAI,CAAC,CAAC,eAAe,EAAE,EAAE;wBACxB,IAAI,CAAC,wBAAwB,EAAE,CAAC;wBAChC,IAAI,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC;oBACtC,CAAC,CAAC;yBACD,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC/C,CAAC;qBAAM,CAAC;oBACN,MAAM;gBACR,CAAC;YACH,CAAC;YAED,sDAAsD;YACtD,IAAI,QAAQ,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,CAAC;gBAClD,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,CAAC;QACH,CAAC,CAAC;aACD,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IACzC,CAAC;IAES,WAAW,CAAC,SAAqB,EAAE,kBAA2B;QACtE,OAAO,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC;IAC5D,CAAC;CACF;AAjUD,4EAiUC","sourcesContent":["import type { ILink } from '@comunica/bus-rdf-resolve-hypermedia-links';\nimport type { ILinkQueue } from '@comunica/bus-rdf-resolve-hypermedia-links-queue';\nimport { MetadataValidationState } from '@comunica/metadata';\nimport type { IQuerySource, IActionContext, MetadataBindings, IQueryBindingsOptions } from '@comunica/types';\nimport type * as RDF from '@rdfjs/types';\nimport type { AsyncIterator, BufferedIteratorOptions } from 'asynciterator';\nimport { BufferedIterator } from 'asynciterator';\nimport type { Algebra } from 'sparqlalgebrajs';\n\nexport abstract class LinkedRdfSourcesAsyncRdfIterator extends BufferedIterator<RDF.Bindings> {\n protected readonly operation: Algebra.Operation;\n protected readonly queryBindingsOptions: IQueryBindingsOptions | undefined;\n protected readonly context: IActionContext;\n\n private readonly cacheSize: number;\n protected readonly firstUrl: string;\n private readonly maxIterators: number;\n private readonly sourceStateGetter: SourceStateGetter;\n\n protected started = false;\n private readonly currentIterators: AsyncIterator<RDF.Bindings>[] = [];\n private iteratorsPendingCreation = 0;\n private iteratorsPendingTermination = 0;\n // eslint-disable-next-line unicorn/no-useless-undefined\n private accumulatedMetadata: Promise<MetadataBindings | undefined> = Promise.resolve(undefined);\n private preflightMetadata: Promise<MetadataBindings> | undefined;\n\n public constructor(\n cacheSize: number,\n operation: Algebra.Operation,\n queryBindingsOptions: IQueryBindingsOptions | undefined,\n context: IActionContext,\n firstUrl: string,\n maxIterators: number,\n sourceStateGetter: SourceStateGetter,\n options?: BufferedIteratorOptions,\n ) {\n super({ autoStart: false, ...options });\n this._reading = false;\n this.cacheSize = cacheSize;\n this.operation = operation;\n this.queryBindingsOptions = queryBindingsOptions;\n this.context = context;\n this.firstUrl = firstUrl;\n this.maxIterators = maxIterators;\n this.sourceStateGetter = sourceStateGetter;\n\n if (this.maxIterators <= 0) {\n throw new Error(`LinkedRdfSourcesAsyncRdfIterator.maxIterators must be larger than zero, but got ${this.maxIterators}`);\n }\n }\n\n /**\n * Start filling the buffer of this iterator.\n */\n public kickstart(): void {\n if (!this.started) {\n this._fillBufferAsync();\n }\n }\n\n public override getProperty<P>(propertyName: string, callback?: (value: P) => void): P | undefined {\n if (propertyName === 'metadata' && !this.started) {\n // If the iterator has not started yet, forcefully fetch the metadata from the source without starting the\n // iterator. This way, we keep the iterator lazy.\n if (!this.preflightMetadata) {\n this.preflightMetadata = new Promise((resolve, reject) => {\n this.sourceStateGetter({ url: this.firstUrl }, {})\n .then((sourceState) => {\n // Don't pass query options, as we don't want to consume any passed iterators\n const bindingsStream = sourceState.source.queryBindings(this.operation, this.context);\n bindingsStream.getProperty('metadata', (metadata: MetadataBindings) => {\n metadata.state = new MetadataValidationState();\n bindingsStream.destroy();\n this.accumulateMetadata(sourceState.metadata, metadata)\n .then((accumulatedMetadata) => {\n // Also merge fields that were not explicitly accumulated\n const returnMetadata = { ...sourceState.metadata, ...metadata, ...accumulatedMetadata };\n resolve(returnMetadata);\n })\n .catch(() => {\n resolve({\n ...sourceState.metadata,\n state: new MetadataValidationState(),\n });\n });\n });\n })\n .catch(reject);\n });\n }\n this.preflightMetadata\n .then(metadata => this.setProperty('metadata', metadata))\n .catch(() => {\n // Ignore errors\n });\n }\n return super.getProperty(propertyName, callback);\n }\n\n protected override _end(destroy?: boolean): void {\n // Close all running iterators\n for (const it of this.currentIterators) {\n it.destroy();\n }\n\n super._end(destroy);\n }\n\n /**\n * Get the internal link queue.\n * The returned instance must always be the same.\n */\n public abstract getLinkQueue(): Promise<ILinkQueue>;\n\n /**\n * Determine the links to be followed from the current source given its metadata.\n * @param metadata The metadata of a source.\n */\n protected abstract getSourceLinks(metadata: Record<string, any>): Promise<ILink[]>;\n\n public override _read(count: number, done: () => void): void {\n if (this.started) {\n // Read from all current iterators\n for (const iterator of this.currentIterators) {\n while (count > 0) {\n const read = iterator.read();\n if (read === null) {\n break;\n } else {\n count--;\n this._push(read);\n }\n }\n if (count <= 0) {\n break;\n }\n }\n\n // Schedule new iterators if needed\n if (count >= 0 && this.canStartNewIterator()) {\n // We can safely ignore skip catching the error, since we are guaranteed to have\n // successfully got the source for this.firstUrl before.\n // eslint-disable-next-line ts/no-floating-promises\n this.sourceStateGetter({ url: this.firstUrl }, {})\n .then((sourceState) => {\n this.startIteratorsForNextUrls(sourceState.handledDatasets, false);\n done();\n });\n } else {\n done();\n }\n } else {\n // The first time this is called, prepare the first source\n this.started = true;\n\n // Await the source to be set, and start the source iterator\n this.sourceStateGetter({ url: this.firstUrl }, {})\n .then((sourceState) => {\n this.startIterator(sourceState);\n done();\n })\n // Destroy should be async because it can be called before it is listened to\n .catch(error => setTimeout(() => this.destroy(error)));\n }\n }\n\n protected canStartNewIterator(): boolean {\n return (this.currentIterators.length + this.iteratorsPendingCreation + this.iteratorsPendingTermination) <\n this.maxIterators && (!this.canStartNewIteratorConsiderReadable() || !this.readable);\n }\n\n protected canStartNewIteratorConsiderReadable(): boolean {\n return true;\n }\n\n protected areIteratorsRunning(): boolean {\n return (this.currentIterators.length + this.iteratorsPendingCreation + this.iteratorsPendingTermination) > 0;\n }\n\n /**\n * Append the fields from appendingMetadata into accumulatedMetadata.\n * @param accumulatedMetadata The fields to append to.\n * @param appendingMetadata The fields to append.\n * @protected\n */\n protected abstract accumulateMetadata(\n accumulatedMetadata: MetadataBindings,\n appendingMetadata: MetadataBindings,\n ): Promise<MetadataBindings>;\n\n /**\n * Start a new iterator for the given source.\n * Once the iterator is done, it will either determine a new source, or it will close the linked iterator.\n * @param {ISourceState} startSource The start source state.\n */\n protected startIterator(startSource: ISourceState): void {\n // Delegate the quad pattern query to the given source\n try {\n const iterator = startSource.source.queryBindings(this.operation, this.context, this.queryBindingsOptions);\n this.currentIterators.push(iterator);\n let receivedEndEvent = false;\n let receivedMetadata = false;\n\n // Attach readers to the newly created iterator\n (<any>iterator)._destination = this;\n iterator.on('error', (error: Error) => this.destroy(error));\n iterator.on('readable', () => this._fillBuffer());\n iterator.on('end', () => {\n this.currentIterators.splice(this.currentIterators.indexOf(iterator), 1);\n\n // Indicate that this iterator still needs to flush its next-links.\n // Without this, the linked iterator could sometimes be closed before next-links are obtained.\n receivedEndEvent = true;\n if (!receivedMetadata) {\n this.iteratorsPendingTermination++;\n }\n\n // If the metadata was already received, handle the next URL in the queue\n if (receivedMetadata) {\n this.startIteratorsForNextUrls(startSource.handledDatasets, true);\n }\n });\n\n // Listen for the metadata of the source\n // The metadata property is guaranteed to be set\n iterator.getProperty('metadata', (metadata: MetadataBindings) => {\n // Accumulate the metadata object\n this.accumulatedMetadata = this.accumulatedMetadata\n .then(previousMetadata => (async() => {\n if (!previousMetadata) {\n previousMetadata = startSource.metadata;\n }\n return this.accumulateMetadata(previousMetadata, metadata);\n })()\n .then((accumulatedMetadata) => {\n // Also merge fields that were not explicitly accumulated\n const returnMetadata = { ...startSource.metadata, ...metadata, ...accumulatedMetadata };\n\n // Create new metadata state\n returnMetadata.state = new MetadataValidationState();\n\n // Emit metadata, and invalidate any metadata that was set before\n this.updateMetadata(returnMetadata);\n\n // Invalidate any preflight metadata\n if (this.preflightMetadata) {\n this.preflightMetadata\n .then(metadataIn => metadataIn.state.invalidate())\n .catch(() => {\n // Ignore errors\n });\n }\n\n // Determine next urls, which will eventually become a next-next source.\n this.getSourceLinks(returnMetadata)\n .then((nextUrls: ILink[]) => Promise.all(nextUrls))\n .then(async(nextUrls: ILink[]) => {\n // Append all next URLs to our queue\n const linkQueue = await this.getLinkQueue();\n for (const nextUrl of nextUrls) {\n linkQueue.push(nextUrl, startSource.link);\n }\n\n receivedMetadata = true;\n if (receivedEndEvent) {\n this.iteratorsPendingTermination--;\n }\n\n this.startIteratorsForNextUrls(startSource.handledDatasets, true);\n }).catch(error => this.destroy(error));\n\n return returnMetadata;\n })).catch((error) => {\n this.destroy(error);\n return <MetadataBindings>{};\n });\n });\n } catch (syncError: unknown) {\n this.destroy(<Error> syncError);\n }\n }\n\n protected updateMetadata(metadataNew: MetadataBindings): void {\n const metadataToInvalidate = this.getProperty<MetadataBindings>('metadata');\n this.setProperty('metadata', metadataNew);\n metadataToInvalidate?.state.invalidate();\n }\n\n protected isRunning(): boolean {\n return !this.done;\n }\n\n /**\n * Check if a next URL is in the queue.\n * If yes, start a new iterator.\n * If no, close this iterator.\n * @param handledDatasets\n * @param canClose\n */\n protected startIteratorsForNextUrls(handledDatasets: Record<string, boolean>, canClose: boolean): void {\n this.getLinkQueue()\n .then((linkQueue) => {\n // Create as many new iterators as possible\n while (this.canStartNewIterator() && this.isRunning()) {\n const nextLink = linkQueue.pop();\n if (nextLink) {\n this.iteratorsPendingCreation++;\n this.sourceStateGetter(nextLink, handledDatasets)\n .then((nextSourceState) => {\n this.iteratorsPendingCreation--;\n this.startIterator(nextSourceState);\n })\n .catch(error => this.emit('error', error));\n } else {\n break;\n }\n }\n\n // Close, only if no other iterators are still running\n if (canClose && this.isCloseable(linkQueue, true)) {\n this.close();\n }\n })\n .catch(error => this.destroy(error));\n }\n\n protected isCloseable(linkQueue: ILinkQueue, _requireQueueEmpty: boolean): boolean {\n return linkQueue.isEmpty() && !this.areIteratorsRunning();\n }\n}\n\n/**\n * The current state of a source.\n * This is needed for following links within a source.\n */\nexport interface ISourceState {\n /**\n * The link to this source.\n */\n link: ILink;\n /**\n * A source.\n */\n source: IQuerySource;\n /**\n * The source's initial metadata.\n */\n metadata: MetadataBindings;\n /**\n * All dataset identifiers that have been passed for this source.\n */\n handledDatasets: Record<string, boolean>;\n}\n\nexport type SourceStateGetter = (link: ILink, handledDatasets: Record<string, boolean>) => Promise<ISourceState>;\n"]}
|
|
@@ -119,10 +119,7 @@ class MediatedLinkedRdfSourcesAsyncRdfIterator extends LinkedRdfSourcesAsyncRdfI
|
|
|
119
119
|
// A source that has been cached due to earlier query executions may not be part of the aggregated store yet.
|
|
120
120
|
// In that case, we add all quads from that source to the aggregated store.
|
|
121
121
|
this.aggregatedStore?.containedSources.add(startSource.link.url);
|
|
122
|
-
const stream = startSource.source.queryBindings(AF.createPattern(DF.variable('s'), DF.variable('p'), DF.variable('o'), DF.variable('g')), this.context.set(context_entries_1.KeysQueryOperation.unionDefaultGraph, true)).
|
|
123
|
-
map: bindings => DF.quad(bindings.get('s'), bindings.get('p'), bindings.get('o'), bindings.get('g')),
|
|
124
|
-
autoStart: false,
|
|
125
|
-
});
|
|
122
|
+
const stream = startSource.source.queryBindings(AF.createPattern(DF.variable('s'), DF.variable('p'), DF.variable('o'), DF.variable('g')), this.context.set(context_entries_1.KeysQueryOperation.unionDefaultGraph, true)).map(bindings => DF.quad(bindings.get('s'), bindings.get('p'), bindings.get('o'), bindings.get('g')));
|
|
126
123
|
this.aggregatedStore.import(stream)
|
|
127
124
|
.on('end', () => {
|
|
128
125
|
super.startIterator(startSource);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MediatedLinkedRdfSourcesAsyncRdfIterator.js","sourceRoot":"","sources":["MediatedLinkedRdfSourcesAsyncRdfIterator.ts"],"names":[],"mappings":";;;AAMA,+DAA+D;AAG/D,uDAA+C;AAE/C,qDAA0C;AAE1C,yFAAsF;AAEtF,MAAM,EAAE,GAAG,IAAI,8BAAW,EAAgB,CAAC;AAC3C,MAAM,EAAE,GAAG,IAAI,yBAAO,EAAE,CAAC;AAEzB;;;;;GAKG;AACH,MAAa,wCAAyC,SAAQ,mEAAgC;IAU5F,YACE,SAAiB,EACjB,SAA4B,EAC5B,oBAAuD,EACvD,OAAuB,EACvB,eAAmC,EACnC,QAAgB,EAChB,YAAoB,EACpB,iBAAoC,EACpC,eAA6C,EAC7C,0BAAyD,EACzD,iCAAoE,EACpE,sCAA8E;QAE9E,KAAK,CACH,SAAS,EACT,SAAS,EACT,oBAAoB,EACpB,OAAO,EACP,QAAQ,EACR,YAAY,EACZ,iBAAiB;QACjB,2GAA2G;QAC3G,6CAA6C;QAC7C,eAAe,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,MAAM,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,SAAS,CAC1E,CAAC;QA3BI,wBAAmB,GAAG,KAAK,CAAC;QA4BlC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,0BAA0B,GAAG,0BAA0B,CAAC;QAC7D,IAAI,CAAC,iCAAiC,GAAG,iCAAiC,CAAC;QAC3E,IAAI,CAAC,sCAAsC,GAAG,sCAAsC,CAAC;QACrF,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,CAAC;QACxC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACzC,CAAC;IAED,iGAAiG;IACjG,uEAAuE;IACvE,+FAA+F;IAE/E,KAAK;QACnB,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,KAAK,CAAC,KAAK,EAAE,CAAC;YACd,OAAO;QACT,CAAC;QAED,IAAI,CAAC,YAAY,EAAE;aAChB,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE;YAClB,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,CAAC;gBACvC,0GAA0G;gBAC1G,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;oBACzB,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,eAAgB,CAAC,GAAG,EAAE,CAAC,CAAC;gBAChD,CAAC;gBACD,KAAK,CAAC,KAAK,EAAE,CAAC;YAChB,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;YAClC,CAAC;QACH,CAAC,CAAC;aACD,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1C,CAAC;IAEe,OAAO,CAAC,KAAa;QACnC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACrB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,YAAY,EAAE;aAChB,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE;YAClB,IAAI,KAAK,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,CAAC;gBAChD,0GAA0G;gBAC1G,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;oBACzB,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,eAAgB,CAAC,GAAG,EAAE,CAAC,CAAC;gBAChD,CAAC;gBACD,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;YAClC,CAAC;QACH,CAAC,CAAC;aACD,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1C,CAAC;IAEkB,WAAW,CAAC,SAAqB,EAAE,iBAA0B;QAC9E,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,mBAAmB,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;YAChG,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC;IAChC,CAAC;IAEkB,mBAAmB;QACpC,oGAAoG;QACpG,mHAAmH;QACnH,OAAO,CAAC,CAAC,IAAI,CAAC,mBAAmB;YAC/B,wDAAwD;YACxD,CAAC,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe,CAAC,mBAAmB,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC;IACzG,CAAC;IAEkB,mCAAmC;QACpD,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC;IAC/B,CAAC;IAEkB,SAAS;QAC1B,gBAAgB;QAChB,wDAAwD;QACxD,OAAO,CAAC,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe,CAAC,mBAAmB,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;IAC5F,CAAC;IAEM,YAAY;QACjB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,sCAAsC;iBACzD,OAAO,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC;iBAC3D,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACtC,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAES,KAAK,CAAC,cAAc,CAAC,QAA6B;QAC1D,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,iCAAiC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;YAE5G,8CAA8C;YAC9C,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;gBAC3B,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC/B,OAAO,KAAK,CAAC;gBACf,CAAC;gBACD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;gBAClC,OAAO,IAAI,CAAC;YACd,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,uGAAuG;YACvG,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAEkB,aAAa,CAAC,WAAyB;QACxD,IAAI,IAAI,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,gBAAgB,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7F,6GAA6G;YAC7G,2EAA2E;YAC3E,IAAI,CAAC,eAAe,EAAE,gBAAgB,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACjE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,aAAa,CAC7C,EAAE,CAAC,aAAa,CACd,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAChB,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAChB,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAChB,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CACjB,EACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,oCAAkB,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAC7D,CAAC,SAAS,CAAC;gBACV,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CACtB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAE,EAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAE,EAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAE,EAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAClB;gBACD,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,MAAM,CAAc,MAAM,CAAC;iBAC7C,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACd,KAAK,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;YACnC,CAAC,CAAC,CAAC;QACP,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,kBAAkB,CAC7B,mBAAqC,EACrC,iBAAmC;QAEnC,OAA0B,CAAC,MAAM,IAAI,CAAC,0BAA0B,CAAC,OAAO,CAAC;YACvE,IAAI,EAAE,QAAQ;YACd,mBAAmB;YACnB,iBAAiB;YACjB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC,CAAC,QAAQ,CAAC;IACf,CAAC;IAEkB,cAAc,CAAC,WAA6B;QAC7D,KAAK,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;QAClC,IAAI,CAAC,eAAe,EAAE,eAAe,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;IAC3D,CAAC;CACF;AA3LD,4FA2LC","sourcesContent":["import type { MediatorRdfMetadataAccumulate } from '@comunica/bus-rdf-metadata-accumulate';\nimport type { ILink, MediatorRdfResolveHypermediaLinks } from '@comunica/bus-rdf-resolve-hypermedia-links';\nimport type {\n ILinkQueue,\n MediatorRdfResolveHypermediaLinksQueue,\n} from '@comunica/bus-rdf-resolve-hypermedia-links-queue';\nimport { KeysQueryOperation } from '@comunica/context-entries';\nimport type { IActionContext, IAggregatedStore, IQueryBindingsOptions, MetadataBindings } from '@comunica/types';\nimport type * as RDF from '@rdfjs/types';\nimport { DataFactory } from 'rdf-data-factory';\nimport type { Algebra } from 'sparqlalgebrajs';\nimport { Factory } from 'sparqlalgebrajs';\nimport type { SourceStateGetter, ISourceState } from './LinkedRdfSourcesAsyncRdfIterator';\nimport { LinkedRdfSourcesAsyncRdfIterator } from './LinkedRdfSourcesAsyncRdfIterator';\n\nconst DF = new DataFactory<RDF.BaseQuad>();\nconst AF = new Factory();\n\n/**\n * An quad iterator that can iterate over consecutive RDF sources\n * that are determined using the rdf-resolve-hypermedia-links bus.\n *\n * @see LinkedRdfSourcesAsyncRdfIterator\n */\nexport class MediatedLinkedRdfSourcesAsyncRdfIterator extends LinkedRdfSourcesAsyncRdfIterator {\n private readonly mediatorMetadataAccumulate: MediatorRdfMetadataAccumulate;\n private readonly mediatorRdfResolveHypermediaLinks: MediatorRdfResolveHypermediaLinks;\n private readonly mediatorRdfResolveHypermediaLinksQueue: MediatorRdfResolveHypermediaLinksQueue;\n private readonly forceSourceType?: string;\n private readonly handledUrls: Record<string, boolean>;\n private readonly aggregatedStore: IAggregatedStore | undefined;\n private linkQueue: Promise<ILinkQueue> | undefined;\n private wasForcefullyClosed = false;\n\n public constructor(\n cacheSize: number,\n operation: Algebra.Operation,\n queryBindingsOptions: IQueryBindingsOptions | undefined,\n context: IActionContext,\n forceSourceType: string | undefined,\n firstUrl: string,\n maxIterators: number,\n sourceStateGetter: SourceStateGetter,\n aggregatedStore: IAggregatedStore | undefined,\n mediatorMetadataAccumulate: MediatorRdfMetadataAccumulate,\n mediatorRdfResolveHypermediaLinks: MediatorRdfResolveHypermediaLinks,\n mediatorRdfResolveHypermediaLinksQueue: MediatorRdfResolveHypermediaLinksQueue,\n ) {\n super(\n cacheSize,\n operation,\n queryBindingsOptions,\n context,\n firstUrl,\n maxIterators,\n sourceStateGetter,\n // Buffersize must be infinite for an aggregated store because it must keep filling until there are no more\n // derived iterators in the aggregated store.\n aggregatedStore ? { maxBufferSize: Number.POSITIVE_INFINITY } : undefined,\n );\n this.forceSourceType = forceSourceType;\n this.mediatorMetadataAccumulate = mediatorMetadataAccumulate;\n this.mediatorRdfResolveHypermediaLinks = mediatorRdfResolveHypermediaLinks;\n this.mediatorRdfResolveHypermediaLinksQueue = mediatorRdfResolveHypermediaLinksQueue;\n this.handledUrls = { [firstUrl]: true };\n this.aggregatedStore = aggregatedStore;\n }\n\n // Mark the aggregated store as ended once we trigger the closing or destroying of this iterator.\n // We don't override _end, because that would mean that we have to wait\n // until the buffer of this iterator must be fully consumed, which will not always be the case.\n\n public override close(): void {\n if (!this.aggregatedStore) {\n super.close();\n return;\n }\n\n this.getLinkQueue()\n .then((linkQueue) => {\n if (this.isCloseable(linkQueue, false)) {\n // Wait a tick before ending the aggregatedStore, to ensure that pending match() calls to it have started.\n if (this.aggregatedStore) {\n setTimeout(() => this.aggregatedStore!.end());\n }\n super.close();\n } else {\n this.wasForcefullyClosed = true;\n }\n })\n .catch(error => super.destroy(error));\n }\n\n public override destroy(cause?: Error): void {\n if (!this.aggregatedStore) {\n super.destroy(cause);\n return;\n }\n\n this.getLinkQueue()\n .then((linkQueue) => {\n if (cause ?? this.isCloseable(linkQueue, false)) {\n // Wait a tick before ending the aggregatedStore, to ensure that pending match() calls to it have started.\n if (this.aggregatedStore) {\n setTimeout(() => this.aggregatedStore!.end());\n }\n super.destroy(cause);\n } else {\n this.wasForcefullyClosed = true;\n }\n })\n .catch(error => super.destroy(error));\n }\n\n protected override isCloseable(linkQueue: ILinkQueue, requireQueueEmpty: boolean): boolean {\n return (requireQueueEmpty ? linkQueue.isEmpty() : this.wasForcefullyClosed || linkQueue.isEmpty()) &&\n !this.areIteratorsRunning();\n }\n\n protected override canStartNewIterator(): boolean {\n // Also allow sub-iterators to be started if the aggregated store has at least one running iterator.\n // We need this because there are cases where these running iterators will be consumed before this linked iterator.\n return (!this.wasForcefullyClosed &&\n // eslint-disable-next-line ts/prefer-nullish-coalescing\n (this.aggregatedStore && this.aggregatedStore.hasRunningIterators())) || super.canStartNewIterator();\n }\n\n protected override canStartNewIteratorConsiderReadable(): boolean {\n return !this.aggregatedStore;\n }\n\n protected override isRunning(): boolean {\n // Same as above\n // eslint-disable-next-line ts/prefer-nullish-coalescing\n return (this.aggregatedStore && this.aggregatedStore.hasRunningIterators()) || !this.done;\n }\n\n public getLinkQueue(): Promise<ILinkQueue> {\n if (!this.linkQueue) {\n this.linkQueue = this.mediatorRdfResolveHypermediaLinksQueue\n .mediate({ firstUrl: this.firstUrl, context: this.context })\n .then(result => result.linkQueue);\n }\n return this.linkQueue;\n }\n\n protected async getSourceLinks(metadata: Record<string, any>): Promise<ILink[]> {\n try {\n const { links } = await this.mediatorRdfResolveHypermediaLinks.mediate({ context: this.context, metadata });\n\n // Filter URLs to avoid cyclic next-page loops\n return links.filter((link) => {\n if (this.handledUrls[link.url]) {\n return false;\n }\n this.handledUrls[link.url] = true;\n return true;\n });\n } catch {\n // No next URLs may be available, for example when we've reached the end of a Hydra next-page sequence.\n return [];\n }\n }\n\n protected override startIterator(startSource: ISourceState): void {\n if (this.aggregatedStore && !this.aggregatedStore.containedSources.has(startSource.link.url)) {\n // A source that has been cached due to earlier query executions may not be part of the aggregated store yet.\n // In that case, we add all quads from that source to the aggregated store.\n this.aggregatedStore?.containedSources.add(startSource.link.url);\n const stream = startSource.source.queryBindings(\n AF.createPattern(\n DF.variable('s'),\n DF.variable('p'),\n DF.variable('o'),\n DF.variable('g'),\n ),\n this.context.set(KeysQueryOperation.unionDefaultGraph, true),\n ).transform({\n map: bindings => DF.quad(\n bindings.get('s')!,\n bindings.get('p')!,\n bindings.get('o')!,\n bindings.get('g'),\n ),\n autoStart: false,\n });\n this.aggregatedStore.import(<RDF.Stream> stream)\n .on('end', () => {\n super.startIterator(startSource);\n });\n } else {\n super.startIterator(startSource);\n }\n }\n\n public async accumulateMetadata(\n accumulatedMetadata: MetadataBindings,\n appendingMetadata: MetadataBindings,\n ): Promise<MetadataBindings> {\n return <MetadataBindings> (await this.mediatorMetadataAccumulate.mediate({\n mode: 'append',\n accumulatedMetadata,\n appendingMetadata,\n context: this.context,\n })).metadata;\n }\n\n protected override updateMetadata(metadataNew: MetadataBindings): void {\n super.updateMetadata(metadataNew);\n this.aggregatedStore?.setBaseMetadata(metadataNew, true);\n }\n}\n"]}
|
|
1
|
+
{"version":3,"file":"MediatedLinkedRdfSourcesAsyncRdfIterator.js","sourceRoot":"","sources":["MediatedLinkedRdfSourcesAsyncRdfIterator.ts"],"names":[],"mappings":";;;AAMA,+DAA+D;AAG/D,uDAA+C;AAE/C,qDAA0C;AAE1C,yFAAsF;AAEtF,MAAM,EAAE,GAAG,IAAI,8BAAW,EAAgB,CAAC;AAC3C,MAAM,EAAE,GAAG,IAAI,yBAAO,EAAE,CAAC;AAEzB;;;;;GAKG;AACH,MAAa,wCAAyC,SAAQ,mEAAgC;IAU5F,YACE,SAAiB,EACjB,SAA4B,EAC5B,oBAAuD,EACvD,OAAuB,EACvB,eAAmC,EACnC,QAAgB,EAChB,YAAoB,EACpB,iBAAoC,EACpC,eAA6C,EAC7C,0BAAyD,EACzD,iCAAoE,EACpE,sCAA8E;QAE9E,KAAK,CACH,SAAS,EACT,SAAS,EACT,oBAAoB,EACpB,OAAO,EACP,QAAQ,EACR,YAAY,EACZ,iBAAiB;QACjB,2GAA2G;QAC3G,6CAA6C;QAC7C,eAAe,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,MAAM,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,SAAS,CAC1E,CAAC;QA3BI,wBAAmB,GAAG,KAAK,CAAC;QA4BlC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,0BAA0B,GAAG,0BAA0B,CAAC;QAC7D,IAAI,CAAC,iCAAiC,GAAG,iCAAiC,CAAC;QAC3E,IAAI,CAAC,sCAAsC,GAAG,sCAAsC,CAAC;QACrF,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,CAAC;QACxC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACzC,CAAC;IAED,iGAAiG;IACjG,uEAAuE;IACvE,+FAA+F;IAE/E,KAAK;QACnB,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,KAAK,CAAC,KAAK,EAAE,CAAC;YACd,OAAO;QACT,CAAC;QAED,IAAI,CAAC,YAAY,EAAE;aAChB,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE;YAClB,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,CAAC;gBACvC,0GAA0G;gBAC1G,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;oBACzB,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,eAAgB,CAAC,GAAG,EAAE,CAAC,CAAC;gBAChD,CAAC;gBACD,KAAK,CAAC,KAAK,EAAE,CAAC;YAChB,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;YAClC,CAAC;QACH,CAAC,CAAC;aACD,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1C,CAAC;IAEe,OAAO,CAAC,KAAa;QACnC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACrB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,YAAY,EAAE;aAChB,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE;YAClB,IAAI,KAAK,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,CAAC;gBAChD,0GAA0G;gBAC1G,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;oBACzB,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,eAAgB,CAAC,GAAG,EAAE,CAAC,CAAC;gBAChD,CAAC;gBACD,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;YAClC,CAAC;QACH,CAAC,CAAC;aACD,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1C,CAAC;IAEkB,WAAW,CAAC,SAAqB,EAAE,iBAA0B;QAC9E,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,mBAAmB,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;YAChG,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC;IAChC,CAAC;IAEkB,mBAAmB;QACpC,oGAAoG;QACpG,mHAAmH;QACnH,OAAO,CAAC,CAAC,IAAI,CAAC,mBAAmB;YAC/B,wDAAwD;YACxD,CAAC,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe,CAAC,mBAAmB,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC;IACzG,CAAC;IAEkB,mCAAmC;QACpD,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC;IAC/B,CAAC;IAEkB,SAAS;QAC1B,gBAAgB;QAChB,wDAAwD;QACxD,OAAO,CAAC,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe,CAAC,mBAAmB,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;IAC5F,CAAC;IAEM,YAAY;QACjB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,sCAAsC;iBACzD,OAAO,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC;iBAC3D,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACtC,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAES,KAAK,CAAC,cAAc,CAAC,QAA6B;QAC1D,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,iCAAiC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;YAE5G,8CAA8C;YAC9C,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;gBAC3B,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC/B,OAAO,KAAK,CAAC;gBACf,CAAC;gBACD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;gBAClC,OAAO,IAAI,CAAC;YACd,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,uGAAuG;YACvG,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAEkB,aAAa,CAAC,WAAyB;QACxD,IAAI,IAAI,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,gBAAgB,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7F,6GAA6G;YAC7G,2EAA2E;YAC3E,IAAI,CAAC,eAAe,EAAE,gBAAgB,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACjE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,aAAa,CAC7C,EAAE,CAAC,aAAa,CACd,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAChB,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAChB,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAChB,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CACjB,EACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,oCAAkB,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAC7D,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CACvB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAE,EAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAE,EAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAE,EAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAClB,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,MAAM,CAAc,MAAM,CAAC;iBAC7C,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACd,KAAK,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;YACnC,CAAC,CAAC,CAAC;QACP,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,kBAAkB,CAC7B,mBAAqC,EACrC,iBAAmC;QAEnC,OAA0B,CAAC,MAAM,IAAI,CAAC,0BAA0B,CAAC,OAAO,CAAC;YACvE,IAAI,EAAE,QAAQ;YACd,mBAAmB;YACnB,iBAAiB;YACjB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC,CAAC,QAAQ,CAAC;IACf,CAAC;IAEkB,cAAc,CAAC,WAA6B;QAC7D,KAAK,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;QAClC,IAAI,CAAC,eAAe,EAAE,eAAe,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;IAC3D,CAAC;CACF;AAxLD,4FAwLC","sourcesContent":["import type { MediatorRdfMetadataAccumulate } from '@comunica/bus-rdf-metadata-accumulate';\nimport type { ILink, MediatorRdfResolveHypermediaLinks } from '@comunica/bus-rdf-resolve-hypermedia-links';\nimport type {\n ILinkQueue,\n MediatorRdfResolveHypermediaLinksQueue,\n} from '@comunica/bus-rdf-resolve-hypermedia-links-queue';\nimport { KeysQueryOperation } from '@comunica/context-entries';\nimport type { IActionContext, IAggregatedStore, IQueryBindingsOptions, MetadataBindings } from '@comunica/types';\nimport type * as RDF from '@rdfjs/types';\nimport { DataFactory } from 'rdf-data-factory';\nimport type { Algebra } from 'sparqlalgebrajs';\nimport { Factory } from 'sparqlalgebrajs';\nimport type { SourceStateGetter, ISourceState } from './LinkedRdfSourcesAsyncRdfIterator';\nimport { LinkedRdfSourcesAsyncRdfIterator } from './LinkedRdfSourcesAsyncRdfIterator';\n\nconst DF = new DataFactory<RDF.BaseQuad>();\nconst AF = new Factory();\n\n/**\n * An quad iterator that can iterate over consecutive RDF sources\n * that are determined using the rdf-resolve-hypermedia-links bus.\n *\n * @see LinkedRdfSourcesAsyncRdfIterator\n */\nexport class MediatedLinkedRdfSourcesAsyncRdfIterator extends LinkedRdfSourcesAsyncRdfIterator {\n private readonly mediatorMetadataAccumulate: MediatorRdfMetadataAccumulate;\n private readonly mediatorRdfResolveHypermediaLinks: MediatorRdfResolveHypermediaLinks;\n private readonly mediatorRdfResolveHypermediaLinksQueue: MediatorRdfResolveHypermediaLinksQueue;\n private readonly forceSourceType?: string;\n private readonly handledUrls: Record<string, boolean>;\n private readonly aggregatedStore: IAggregatedStore | undefined;\n private linkQueue: Promise<ILinkQueue> | undefined;\n private wasForcefullyClosed = false;\n\n public constructor(\n cacheSize: number,\n operation: Algebra.Operation,\n queryBindingsOptions: IQueryBindingsOptions | undefined,\n context: IActionContext,\n forceSourceType: string | undefined,\n firstUrl: string,\n maxIterators: number,\n sourceStateGetter: SourceStateGetter,\n aggregatedStore: IAggregatedStore | undefined,\n mediatorMetadataAccumulate: MediatorRdfMetadataAccumulate,\n mediatorRdfResolveHypermediaLinks: MediatorRdfResolveHypermediaLinks,\n mediatorRdfResolveHypermediaLinksQueue: MediatorRdfResolveHypermediaLinksQueue,\n ) {\n super(\n cacheSize,\n operation,\n queryBindingsOptions,\n context,\n firstUrl,\n maxIterators,\n sourceStateGetter,\n // Buffersize must be infinite for an aggregated store because it must keep filling until there are no more\n // derived iterators in the aggregated store.\n aggregatedStore ? { maxBufferSize: Number.POSITIVE_INFINITY } : undefined,\n );\n this.forceSourceType = forceSourceType;\n this.mediatorMetadataAccumulate = mediatorMetadataAccumulate;\n this.mediatorRdfResolveHypermediaLinks = mediatorRdfResolveHypermediaLinks;\n this.mediatorRdfResolveHypermediaLinksQueue = mediatorRdfResolveHypermediaLinksQueue;\n this.handledUrls = { [firstUrl]: true };\n this.aggregatedStore = aggregatedStore;\n }\n\n // Mark the aggregated store as ended once we trigger the closing or destroying of this iterator.\n // We don't override _end, because that would mean that we have to wait\n // until the buffer of this iterator must be fully consumed, which will not always be the case.\n\n public override close(): void {\n if (!this.aggregatedStore) {\n super.close();\n return;\n }\n\n this.getLinkQueue()\n .then((linkQueue) => {\n if (this.isCloseable(linkQueue, false)) {\n // Wait a tick before ending the aggregatedStore, to ensure that pending match() calls to it have started.\n if (this.aggregatedStore) {\n setTimeout(() => this.aggregatedStore!.end());\n }\n super.close();\n } else {\n this.wasForcefullyClosed = true;\n }\n })\n .catch(error => super.destroy(error));\n }\n\n public override destroy(cause?: Error): void {\n if (!this.aggregatedStore) {\n super.destroy(cause);\n return;\n }\n\n this.getLinkQueue()\n .then((linkQueue) => {\n if (cause ?? this.isCloseable(linkQueue, false)) {\n // Wait a tick before ending the aggregatedStore, to ensure that pending match() calls to it have started.\n if (this.aggregatedStore) {\n setTimeout(() => this.aggregatedStore!.end());\n }\n super.destroy(cause);\n } else {\n this.wasForcefullyClosed = true;\n }\n })\n .catch(error => super.destroy(error));\n }\n\n protected override isCloseable(linkQueue: ILinkQueue, requireQueueEmpty: boolean): boolean {\n return (requireQueueEmpty ? linkQueue.isEmpty() : this.wasForcefullyClosed || linkQueue.isEmpty()) &&\n !this.areIteratorsRunning();\n }\n\n protected override canStartNewIterator(): boolean {\n // Also allow sub-iterators to be started if the aggregated store has at least one running iterator.\n // We need this because there are cases where these running iterators will be consumed before this linked iterator.\n return (!this.wasForcefullyClosed &&\n // eslint-disable-next-line ts/prefer-nullish-coalescing\n (this.aggregatedStore && this.aggregatedStore.hasRunningIterators())) || super.canStartNewIterator();\n }\n\n protected override canStartNewIteratorConsiderReadable(): boolean {\n return !this.aggregatedStore;\n }\n\n protected override isRunning(): boolean {\n // Same as above\n // eslint-disable-next-line ts/prefer-nullish-coalescing\n return (this.aggregatedStore && this.aggregatedStore.hasRunningIterators()) || !this.done;\n }\n\n public getLinkQueue(): Promise<ILinkQueue> {\n if (!this.linkQueue) {\n this.linkQueue = this.mediatorRdfResolveHypermediaLinksQueue\n .mediate({ firstUrl: this.firstUrl, context: this.context })\n .then(result => result.linkQueue);\n }\n return this.linkQueue;\n }\n\n protected async getSourceLinks(metadata: Record<string, any>): Promise<ILink[]> {\n try {\n const { links } = await this.mediatorRdfResolveHypermediaLinks.mediate({ context: this.context, metadata });\n\n // Filter URLs to avoid cyclic next-page loops\n return links.filter((link) => {\n if (this.handledUrls[link.url]) {\n return false;\n }\n this.handledUrls[link.url] = true;\n return true;\n });\n } catch {\n // No next URLs may be available, for example when we've reached the end of a Hydra next-page sequence.\n return [];\n }\n }\n\n protected override startIterator(startSource: ISourceState): void {\n if (this.aggregatedStore && !this.aggregatedStore.containedSources.has(startSource.link.url)) {\n // A source that has been cached due to earlier query executions may not be part of the aggregated store yet.\n // In that case, we add all quads from that source to the aggregated store.\n this.aggregatedStore?.containedSources.add(startSource.link.url);\n const stream = startSource.source.queryBindings(\n AF.createPattern(\n DF.variable('s'),\n DF.variable('p'),\n DF.variable('o'),\n DF.variable('g'),\n ),\n this.context.set(KeysQueryOperation.unionDefaultGraph, true),\n ).map(bindings => DF.quad(\n bindings.get('s')!,\n bindings.get('p')!,\n bindings.get('o')!,\n bindings.get('g'),\n ));\n this.aggregatedStore.import(<RDF.Stream> stream)\n .on('end', () => {\n super.startIterator(startSource);\n });\n } else {\n super.startIterator(startSource);\n }\n }\n\n public async accumulateMetadata(\n accumulatedMetadata: MetadataBindings,\n appendingMetadata: MetadataBindings,\n ): Promise<MetadataBindings> {\n return <MetadataBindings> (await this.mediatorMetadataAccumulate.mediate({\n mode: 'append',\n accumulatedMetadata,\n appendingMetadata,\n context: this.context,\n })).metadata;\n }\n\n protected override updateMetadata(metadataNew: MetadataBindings): void {\n super.updateMetadata(metadataNew);\n this.aggregatedStore?.setBaseMetadata(metadataNew, true);\n }\n}\n"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@comunica/actor-query-source-identify-hypermedia",
|
|
3
|
-
"version": "3.1.
|
|
3
|
+
"version": "3.1.2",
|
|
4
4
|
"description": "A hypermedia query-source-identify actor",
|
|
5
5
|
"lsd:module": true,
|
|
6
6
|
"license": "MIT",
|
|
@@ -37,12 +37,12 @@
|
|
|
37
37
|
"build:components": "componentsjs-generator"
|
|
38
38
|
},
|
|
39
39
|
"dependencies": {
|
|
40
|
-
"@comunica/actor-query-source-identify-rdfjs": "^3.1.
|
|
40
|
+
"@comunica/actor-query-source-identify-rdfjs": "^3.1.2",
|
|
41
41
|
"@comunica/bindings-factory": "^3.1.0",
|
|
42
42
|
"@comunica/bus-dereference-rdf": "^3.1.0",
|
|
43
43
|
"@comunica/bus-merge-bindings-context": "^3.1.0",
|
|
44
|
-
"@comunica/bus-query-operation": "^3.1.
|
|
45
|
-
"@comunica/bus-query-source-identify": "^3.1.
|
|
44
|
+
"@comunica/bus-query-operation": "^3.1.2",
|
|
45
|
+
"@comunica/bus-query-source-identify": "^3.1.2",
|
|
46
46
|
"@comunica/bus-query-source-identify-hypermedia": "^3.1.0",
|
|
47
47
|
"@comunica/bus-rdf-metadata": "^3.1.0",
|
|
48
48
|
"@comunica/bus-rdf-metadata-accumulate": "^3.1.0",
|
|
@@ -61,5 +61,5 @@
|
|
|
61
61
|
"readable-stream": "^4.4.2",
|
|
62
62
|
"sparqlalgebrajs": "^4.3.3"
|
|
63
63
|
},
|
|
64
|
-
"gitHead": "
|
|
64
|
+
"gitHead": "db31677bdef5400194783048840c147e04b793e5"
|
|
65
65
|
}
|