@comunica/actor-query-source-identify-hypermedia 3.0.1-alpha.43.0 → 3.0.1-alpha.45.0

package/README.md

@@ -45,7 +45,7 @@ After installing, this package can be added to your engine's configuration as fo
 
 * `cacheSize`: The maximum number of entries in the LRU cache, set to 0 to disable, defaults to 100.
 * `maxIterators`: The maximum number of links that can be followed in parallel, defaults to 64.
-* `aggregateStore`: If all discovered quads across all links from a seed source should be indexed in an aggregated store, to speed up later calls. This should only be used for sources without filter factor. Defaults to false.
+* `aggregateTraversalStore`: If all discovered quads across all links from a traversal source should be indexed in an aggregated store, to speed up later calls. This only applies to sources annotated with KeysQuerySourceIdentify.traverse. Defaults to true.
 * `httpInvalidator`: An optional actor that listens to HTTP invalidation events.
 * `mediatorDereferenceRdf`: A mediator over the [RDF Dereference bus](https://github.com/comunica/comunica/tree/master/packages/bus-dereference-rdf).
 * `mediatorMetadata`: A mediator over the [RDF Metadata bus](https://github.com/comunica/comunica/tree/master/packages/bus-rdf-metadata).

package/components/ActorQuerySourceIdentifyHypermedia.jsonld

@@ -28,10 +28,10 @@
           "comment": "The maximum number of links that can be followed in parallel."
         },
         {
-          "@id": "caqsih:components/ActorQuerySourceIdentifyHypermedia.jsonld#ActorQuerySourceIdentifyHypermedia_args_aggregateStore",
+          "@id": "caqsih:components/ActorQuerySourceIdentifyHypermedia.jsonld#ActorQuerySourceIdentifyHypermedia_args_aggregateTraversalStore",
           "range": "xsd:boolean",
-          "default": "false",
-          "comment": "If all discovered quads across all links from a seed source should be indexed in an aggregated store, to speed up later calls. This should only be used for sources without filter factor."
+          "default": "true",
+          "comment": "If all discovered quads across all links from a traversal source should be indexed in an aggregated store, to speed up later calls. This only applies to sources annotated with KeysQuerySourceIdentify.traverse."
         },
         {
           "@id": "caqsih:components/ActorQuerySourceIdentifyHypermedia.jsonld#ActorQuerySourceIdentifyHypermedia_args_mediatorDereferenceRdf",
@@ -219,8 +219,8 @@
           "range": "xsd:number"
         },
         {
-          "@id": "caqsih:components/ActorQuerySourceIdentifyHypermedia.jsonld#ActorQuerySourceIdentifyHypermedia__member_aggregateStore",
-          "memberFieldName": "aggregateStore",
+          "@id": "caqsih:components/ActorQuerySourceIdentifyHypermedia.jsonld#ActorQuerySourceIdentifyHypermedia__member_aggregateTraversalStore",
+          "memberFieldName": "aggregateTraversalStore",
           "range": "xsd:boolean"
         },
         {
@@ -253,9 +253,9 @@
               }
             },
             {
-              "keyRaw": "aggregateStore",
+              "keyRaw": "aggregateTraversalStore",
               "value": {
-                "@id": "caqsih:components/ActorQuerySourceIdentifyHypermedia.jsonld#ActorQuerySourceIdentifyHypermedia_args_aggregateStore"
+                "@id": "caqsih:components/ActorQuerySourceIdentifyHypermedia.jsonld#ActorQuerySourceIdentifyHypermedia_args_aggregateTraversalStore"
               }
             },
             {
@@ -343,8 +343,8 @@
           "memberFieldName": "maxIterators"
         },
         {
-          "@id": "caqsih:components/ActorQuerySourceIdentifyHypermedia.jsonld#IActorQuerySourceIdentifyHypermediaArgs__member_aggregateStore",
-          "memberFieldName": "aggregateStore"
+          "@id": "caqsih:components/ActorQuerySourceIdentifyHypermedia.jsonld#IActorQuerySourceIdentifyHypermediaArgs__member_aggregateTraversalStore",
+          "memberFieldName": "aggregateTraversalStore"
         },
         {
           "@id": "caqsih:components/ActorQuerySourceIdentifyHypermedia.jsonld#IActorQuerySourceIdentifyHypermediaArgs__member_mediatorDereferenceRdf",

package/components/context.jsonld

@@ -14,8 +14,8 @@
           "args_maxIterators": {
             "@id": "caqsih:components/ActorQuerySourceIdentifyHypermedia.jsonld#ActorQuerySourceIdentifyHypermedia_args_maxIterators"
           },
-          "args_aggregateStore": {
-            "@id": "caqsih:components/ActorQuerySourceIdentifyHypermedia.jsonld#ActorQuerySourceIdentifyHypermedia_args_aggregateStore"
+          "args_aggregateTraversalStore": {
+            "@id": "caqsih:components/ActorQuerySourceIdentifyHypermedia.jsonld#ActorQuerySourceIdentifyHypermedia_args_aggregateTraversalStore"
           },
           "args_mediatorDereferenceRdf": {
             "@id": "caqsih:components/ActorQuerySourceIdentifyHypermedia.jsonld#ActorQuerySourceIdentifyHypermedia_args_mediatorDereferenceRdf"
@@ -57,8 +57,8 @@
           "maxIterators": {
             "@id": "caqsih:components/ActorQuerySourceIdentifyHypermedia.jsonld#ActorQuerySourceIdentifyHypermedia_args_maxIterators"
           },
-          "aggregateStore": {
-            "@id": "caqsih:components/ActorQuerySourceIdentifyHypermedia.jsonld#ActorQuerySourceIdentifyHypermedia_args_aggregateStore"
+          "aggregateTraversalStore": {
+            "@id": "caqsih:components/ActorQuerySourceIdentifyHypermedia.jsonld#ActorQuerySourceIdentifyHypermedia_args_aggregateTraversalStore"
           },
           "mediatorDereferenceRdf": {
             "@id": "caqsih:components/ActorQuerySourceIdentifyHypermedia.jsonld#ActorQuerySourceIdentifyHypermedia_args_mediatorDereferenceRdf"

package/lib/ActorQuerySourceIdentifyHypermedia.d.ts

@@ -23,7 +23,7 @@ export declare class ActorQuerySourceIdentifyHypermedia extends ActorQuerySource
     readonly mediatorMergeBindingsContext: MediatorMergeBindingsContext;
     readonly cacheSize: number;
     readonly maxIterators: number;
-    readonly aggregateStore: boolean;
+    readonly aggregateTraversalStore: boolean;
     constructor(args: IActorQuerySourceIdentifyHypermediaArgs);
     test(action: IActionQuerySourceIdentify): Promise<IActorTest>;
     run(action: IActionQuerySourceIdentify): Promise<IActorQuerySourceIdentifyOutput>;
@@ -41,12 +41,12 @@ export interface IActorQuerySourceIdentifyHypermediaArgs extends IActorQuerySour
      */
     maxIterators: number;
     /**
-     * If all discovered quads across all links from a seed source should be indexed in an aggregated store,
+     * If all discovered quads across all links from a traversal source should be indexed in an aggregated store,
      * to speed up later calls.
-     * This should only be used for sources without filter factor.
-     * @default {false}
+     * This only applies to sources annotated with KeysQuerySourceIdentify.traverse.
+     * @default {true}
      */
-    aggregateStore: boolean;
+    aggregateTraversalStore: boolean;
     /**
      * The RDF dereference mediator
      */

package/lib/ActorQuerySourceIdentifyHypermedia.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.ActorQuerySourceIdentifyHypermedia = void 0;
 const bindings_factory_1 = require("@comunica/bindings-factory");
 const bus_query_source_identify_1 = require("@comunica/bus-query-source-identify");
+const context_entries_1 = require("@comunica/context-entries");
 const core_1 = require("@comunica/core");
 const QuerySourceHypermedia_1 = require("./QuerySourceHypermedia");
 /**
@@ -21,7 +22,8 @@ class ActorQuerySourceIdentifyHypermedia extends bus_query_source_identify_1.Act
     async run(action) {
         return {
             querySource: {
-                source: new QuerySourceHypermedia_1.QuerySourceHypermedia(this.cacheSize, action.querySourceUnidentified.value, action.querySourceUnidentified.type, this.maxIterators, this.aggregateStore, {
+                source: new QuerySourceHypermedia_1.QuerySourceHypermedia(this.cacheSize, action.querySourceUnidentified.value, action.querySourceUnidentified.type, this.maxIterators, this.aggregateTraversalStore &&
+                    Boolean(action.querySourceUnidentified.context?.get(context_entries_1.KeysQuerySourceIdentify.traverse)), {
                     mediatorMetadata: this.mediatorMetadata,
                     mediatorMetadataExtract: this.mediatorMetadataExtract,
                     mediatorMetadataAccumulate: this.mediatorMetadataAccumulate,

package/lib/ActorQuerySourceIdentifyHypermedia.js.map

@@ -1 +1 @@
-{"version":3,"file":"ActorQuerySourceIdentifyHypermedia.js","sourceRoot":"","sources":["ActorQuerySourceIdentifyHypermedia.ts"],"names":[],"mappings":";;;AAAA,iEAA6D;AAG7D,mFAA+E;AAS/E,yCAA+C;AAE/C,mEAAgE;AAEhE;;GAEG;AACH,MAAa,kCAAmC,SAAQ,oDAAwB;IAa9E,YAAmB,IAA6C;QAC9D,KAAK,CAAC,IAAI,CAAC,CAAC;IACd,CAAC;IAEM,KAAK,CAAC,IAAI,CAAC,MAAkC;QAClD,IAAI,OAAO,MAAM,CAAC,uBAAuB,CAAC,KAAK,KAAK,QAAQ,EAAE;YAC5D,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,gFAAgF,CAAC,CAAC;SAC/G;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,KAAK,CAAC,GAAG,CAAC,MAAkC;QACjD,OAAO;YACL,WAAW,EAAE;gBACX,MAAM,EAAE,IAAI,6CAAqB,CAC/B,IAAI,CAAC,SAAS,EACL,MAAM,CAAC,uBAAuB,CAAC,KAAK,EAC7C,MAAM,CAAC,uBAAuB,CAAC,IAAI,EACnC,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,cAAc,EACnB;oBACE,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;oBACvC,uBAAuB,EAAE,IAAI,CAAC,uBAAuB;oBACrD,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;oBAC3D,sBAAsB,EAAE,IAAI,CAAC,sBAAsB;oBACnD,qCAAqC,EAAE,IAAI,CAAC,qCAAqC;oBACjF,iCAAiC,EAAE,IAAI,CAAC,iCAAiC;oBACzE,sCAAsC,EAAE,IAAI,CAAC,sCAAsC;iBACpF,EACD,cAAc,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,cAAc,CAAC,EAC9D,MAAM,kCAAe,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4B,EAAE,MAAM,CAAC,OAAO,CAAC,CAChF;gBACD,OAAO,EAAE,MAAM,CAAC,uBAAuB,CAAC,OAAO,IAAI,IAAI,oBAAa,EAAE;aACvE;SACF,CAAC;IACJ,CAAC;CACF;AAjDD,gFAiDC","sourcesContent":["import { BindingsFactory } from '@comunica/bindings-factory';\nimport type { MediatorDereferenceRdf } from '@comunica/bus-dereference-rdf';\nimport type { MediatorMergeBindingsContext } from '@comunica/bus-merge-bindings-context';\nimport { ActorQuerySourceIdentify } from '@comunica/bus-query-source-identify';\nimport type { IActionQuerySourceIdentify, IActorQuerySourceIdentifyOutput,\n  IActorQuerySourceIdentifyArgs } from '@comunica/bus-query-source-identify';\nimport type { MediatorQuerySourceIdentifyHypermedia } from '@comunica/bus-query-source-identify-hypermedia';\nimport type { MediatorRdfMetadata } from '@comunica/bus-rdf-metadata';\nimport type { MediatorRdfMetadataAccumulate } from '@comunica/bus-rdf-metadata-accumulate';\nimport type { MediatorRdfMetadataExtract } from '@comunica/bus-rdf-metadata-extract';\nimport type { MediatorRdfResolveHypermediaLinks } from '@comunica/bus-rdf-resolve-hypermedia-links';\nimport type { MediatorRdfResolveHypermediaLinksQueue } from '@comunica/bus-rdf-resolve-hypermedia-links-queue';\nimport { ActionContext } from '@comunica/core';\nimport type { IActorTest } from '@comunica/core';\nimport { QuerySourceHypermedia } from './QuerySourceHypermedia';\n\n/**\n * A comunica Hypermedia Query Source Identify Actor.\n */\nexport class ActorQuerySourceIdentifyHypermedia extends ActorQuerySourceIdentify {\n  public readonly mediatorDereferenceRdf: MediatorDereferenceRdf;\n  public readonly mediatorMetadata: MediatorRdfMetadata;\n  public readonly mediatorMetadataExtract: MediatorRdfMetadataExtract;\n  public readonly mediatorMetadataAccumulate: MediatorRdfMetadataAccumulate;\n  public readonly mediatorQuerySourceIdentifyHypermedia: MediatorQuerySourceIdentifyHypermedia;\n  public readonly mediatorRdfResolveHypermediaLinks: MediatorRdfResolveHypermediaLinks;\n  public readonly mediatorRdfResolveHypermediaLinksQueue: MediatorRdfResolveHypermediaLinksQueue;\n  public readonly mediatorMergeBindingsContext: MediatorMergeBindingsContext;\n  public readonly cacheSize: number;\n  public readonly maxIterators: number;\n  public readonly aggregateStore: boolean;\n\n  public constructor(args: IActorQuerySourceIdentifyHypermediaArgs) {\n    super(args);\n  }\n\n  public async test(action: IActionQuerySourceIdentify): Promise<IActorTest> {\n    if (typeof action.querySourceUnidentified.value !== 'string') {\n      throw new Error(`${this.name} requires a single query source with a URL value to be present in the context.`);\n    }\n    return true;\n  }\n\n  public async run(action: IActionQuerySourceIdentify): Promise<IActorQuerySourceIdentifyOutput> {\n    return {\n      querySource: {\n        source: new QuerySourceHypermedia(\n          this.cacheSize,\n          <string> action.querySourceUnidentified.value,\n          action.querySourceUnidentified.type,\n          this.maxIterators,\n          this.aggregateStore,\n          {\n            mediatorMetadata: this.mediatorMetadata,\n            mediatorMetadataExtract: this.mediatorMetadataExtract,\n            mediatorMetadataAccumulate: this.mediatorMetadataAccumulate,\n            mediatorDereferenceRdf: this.mediatorDereferenceRdf,\n            mediatorQuerySourceIdentifyHypermedia: this.mediatorQuerySourceIdentifyHypermedia,\n            mediatorRdfResolveHypermediaLinks: this.mediatorRdfResolveHypermediaLinks,\n            mediatorRdfResolveHypermediaLinksQueue: this.mediatorRdfResolveHypermediaLinksQueue,\n          },\n          warningMessage => this.logWarn(action.context, warningMessage),\n          await BindingsFactory.create(this.mediatorMergeBindingsContext, action.context),\n        ),\n        context: action.querySourceUnidentified.context || new ActionContext(),\n      },\n    };\n  }\n}\n\nexport interface IActorQuerySourceIdentifyHypermediaArgs extends IActorQuerySourceIdentifyArgs {\n  /**\n   * The maximum number of entries in the LRU cache, set to 0 to disable.\n   * @range {integer}\n   * @default {100}\n   */\n  cacheSize: number;\n  /**\n   * The maximum number of links that can be followed in parallel.\n   * @default {64}\n   */\n  maxIterators: number;\n  /**\n   * If all discovered quads across all links from a seed source should be indexed in an aggregated store,\n   * to speed up later calls.\n   * This should only be used for sources without filter factor.\n   * @default {false}\n   */\n  aggregateStore: boolean;\n  /**\n   * The RDF dereference mediator\n   */\n  mediatorDereferenceRdf: MediatorDereferenceRdf;\n  /**\n   * The metadata mediator\n   */\n  mediatorMetadata: MediatorRdfMetadata;\n  /**\n   * The metadata extract mediator\n   */\n  mediatorMetadataExtract: MediatorRdfMetadataExtract;\n  /**\n   * The metadata accumulate mediator\n   */\n  mediatorMetadataAccumulate?: MediatorRdfMetadataAccumulate;\n  /**\n   * The hypermedia resolve mediator\n   */\n  mediatorQuerySourceIdentifyHypermedia: MediatorQuerySourceIdentifyHypermedia;\n  /**\n   * The hypermedia links resolve mediator\n   */\n  mediatorRdfResolveHypermediaLinks: MediatorRdfResolveHypermediaLinks;\n  /**\n   * The hypermedia links queue resolve mediator\n   */\n  mediatorRdfResolveHypermediaLinksQueue: MediatorRdfResolveHypermediaLinksQueue;\n  /**\n   * A mediator for creating binding context merge handlers\n   */\n  mediatorMergeBindingsContext: MediatorMergeBindingsContext;\n}\n"]}
+{"version":3,"file":"ActorQuerySourceIdentifyHypermedia.js","sourceRoot":"","sources":["ActorQuerySourceIdentifyHypermedia.ts"],"names":[],"mappings":";;;AAAA,iEAA6D;AAG7D,mFAA+E;AAS/E,+DAAoE;AACpE,yCAA+C;AAE/C,mEAAgE;AAEhE;;GAEG;AACH,MAAa,kCAAmC,SAAQ,oDAAwB;IAa9E,YAAmB,IAA6C;QAC9D,KAAK,CAAC,IAAI,CAAC,CAAC;IACd,CAAC;IAEM,KAAK,CAAC,IAAI,CAAC,MAAkC;QAClD,IAAI,OAAO,MAAM,CAAC,uBAAuB,CAAC,KAAK,KAAK,QAAQ,EAAE;YAC5D,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,gFAAgF,CAAC,CAAC;SAC/G;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,KAAK,CAAC,GAAG,CAAC,MAAkC;QACjD,OAAO;YACL,WAAW,EAAE;gBACX,MAAM,EAAE,IAAI,6CAAqB,CAC/B,IAAI,CAAC,SAAS,EACL,MAAM,CAAC,uBAAuB,CAAC,KAAK,EAC7C,MAAM,CAAC,uBAAuB,CAAC,IAAI,EACnC,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,uBAAuB;oBAC5B,OAAO,CAAC,MAAM,CAAC,uBAAuB,CAAC,OAAO,EAAE,GAAG,CAAC,yCAAuB,CAAC,QAAQ,CAAC,CAAC,EACtF;oBACE,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;oBACvC,uBAAuB,EAAE,IAAI,CAAC,uBAAuB;oBACrD,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;oBAC3D,sBAAsB,EAAE,IAAI,CAAC,sBAAsB;oBACnD,qCAAqC,EAAE,IAAI,CAAC,qCAAqC;oBACjF,iCAAiC,EAAE,IAAI,CAAC,iCAAiC;oBACzE,sCAAsC,EAAE,IAAI,CAAC,sCAAsC;iBACpF,EACD,cAAc,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,cAAc,CAAC,EAC9D,MAAM,kCAAe,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4B,EAAE,MAAM,CAAC,OAAO,CAAC,CAChF;gBACD,OAAO,EAAE,MAAM,CAAC,uBAAuB,CAAC,OAAO,IAAI,IAAI,oBAAa,EAAE;aACvE;SACF,CAAC;IACJ,CAAC;CACF;AAlDD,gFAkDC","sourcesContent":["import { BindingsFactory } from '@comunica/bindings-factory';\nimport type { MediatorDereferenceRdf } from '@comunica/bus-dereference-rdf';\nimport type { MediatorMergeBindingsContext } from '@comunica/bus-merge-bindings-context';\nimport { ActorQuerySourceIdentify } from '@comunica/bus-query-source-identify';\nimport type { IActionQuerySourceIdentify, IActorQuerySourceIdentifyOutput,\n  IActorQuerySourceIdentifyArgs } from '@comunica/bus-query-source-identify';\nimport type { MediatorQuerySourceIdentifyHypermedia } from '@comunica/bus-query-source-identify-hypermedia';\nimport type { MediatorRdfMetadata } from '@comunica/bus-rdf-metadata';\nimport type { MediatorRdfMetadataAccumulate } from '@comunica/bus-rdf-metadata-accumulate';\nimport type { MediatorRdfMetadataExtract } from '@comunica/bus-rdf-metadata-extract';\nimport type { MediatorRdfResolveHypermediaLinks } from '@comunica/bus-rdf-resolve-hypermedia-links';\nimport type { MediatorRdfResolveHypermediaLinksQueue } from '@comunica/bus-rdf-resolve-hypermedia-links-queue';\nimport { KeysQuerySourceIdentify } from '@comunica/context-entries';\nimport { ActionContext } from '@comunica/core';\nimport type { IActorTest } from '@comunica/core';\nimport { QuerySourceHypermedia } from './QuerySourceHypermedia';\n\n/**\n * A comunica Hypermedia Query Source Identify Actor.\n */\nexport class ActorQuerySourceIdentifyHypermedia extends ActorQuerySourceIdentify {\n  public readonly mediatorDereferenceRdf: MediatorDereferenceRdf;\n  public readonly mediatorMetadata: MediatorRdfMetadata;\n  public readonly mediatorMetadataExtract: MediatorRdfMetadataExtract;\n  public readonly mediatorMetadataAccumulate: MediatorRdfMetadataAccumulate;\n  public readonly mediatorQuerySourceIdentifyHypermedia: MediatorQuerySourceIdentifyHypermedia;\n  public readonly mediatorRdfResolveHypermediaLinks: MediatorRdfResolveHypermediaLinks;\n  public readonly mediatorRdfResolveHypermediaLinksQueue: MediatorRdfResolveHypermediaLinksQueue;\n  public readonly mediatorMergeBindingsContext: MediatorMergeBindingsContext;\n  public readonly cacheSize: number;\n  public readonly maxIterators: number;\n  public readonly aggregateTraversalStore: boolean;\n\n  public constructor(args: IActorQuerySourceIdentifyHypermediaArgs) {\n    super(args);\n  }\n\n  public async test(action: IActionQuerySourceIdentify): Promise<IActorTest> {\n    if (typeof action.querySourceUnidentified.value !== 'string') {\n      throw new Error(`${this.name} requires a single query source with a URL value to be present in the context.`);\n    }\n    return true;\n  }\n\n  public async run(action: IActionQuerySourceIdentify): Promise<IActorQuerySourceIdentifyOutput> {\n    return {\n      querySource: {\n        source: new QuerySourceHypermedia(\n          this.cacheSize,\n          <string> action.querySourceUnidentified.value,\n          action.querySourceUnidentified.type,\n          this.maxIterators,\n          this.aggregateTraversalStore &&\n          Boolean(action.querySourceUnidentified.context?.get(KeysQuerySourceIdentify.traverse)),\n          {\n            mediatorMetadata: this.mediatorMetadata,\n            mediatorMetadataExtract: this.mediatorMetadataExtract,\n            mediatorMetadataAccumulate: this.mediatorMetadataAccumulate,\n            mediatorDereferenceRdf: this.mediatorDereferenceRdf,\n            mediatorQuerySourceIdentifyHypermedia: this.mediatorQuerySourceIdentifyHypermedia,\n            mediatorRdfResolveHypermediaLinks: this.mediatorRdfResolveHypermediaLinks,\n            mediatorRdfResolveHypermediaLinksQueue: this.mediatorRdfResolveHypermediaLinksQueue,\n          },\n          warningMessage => this.logWarn(action.context, warningMessage),\n          await BindingsFactory.create(this.mediatorMergeBindingsContext, action.context),\n        ),\n        context: action.querySourceUnidentified.context || new ActionContext(),\n      },\n    };\n  }\n}\n\nexport interface IActorQuerySourceIdentifyHypermediaArgs extends IActorQuerySourceIdentifyArgs {\n  /**\n   * The maximum number of entries in the LRU cache, set to 0 to disable.\n   * @range {integer}\n   * @default {100}\n   */\n  cacheSize: number;\n  /**\n   * The maximum number of links that can be followed in parallel.\n   * @default {64}\n   */\n  maxIterators: number;\n  /**\n   * If all discovered quads across all links from a traversal source should be indexed in an aggregated store,\n   * to speed up later calls.\n   * This only applies to sources annotated with KeysQuerySourceIdentify.traverse.\n   * @default {true}\n   */\n  aggregateTraversalStore: boolean;\n  /**\n   * The RDF dereference mediator\n   */\n  mediatorDereferenceRdf: MediatorDereferenceRdf;\n  /**\n   * The metadata mediator\n   */\n  mediatorMetadata: MediatorRdfMetadata;\n  /**\n   * The metadata extract mediator\n   */\n  mediatorMetadataExtract: MediatorRdfMetadataExtract;\n  /**\n   * The metadata accumulate mediator\n   */\n  mediatorMetadataAccumulate?: MediatorRdfMetadataAccumulate;\n  /**\n   * The hypermedia resolve mediator\n   */\n  mediatorQuerySourceIdentifyHypermedia: MediatorQuerySourceIdentifyHypermedia;\n  /**\n   * The hypermedia links resolve mediator\n   */\n  mediatorRdfResolveHypermediaLinks: MediatorRdfResolveHypermediaLinks;\n  /**\n   * The hypermedia links queue resolve mediator\n   */\n  mediatorRdfResolveHypermediaLinksQueue: MediatorRdfResolveHypermediaLinksQueue;\n  /**\n   * A mediator for creating binding context merge handlers\n   */\n  mediatorMergeBindingsContext: MediatorMergeBindingsContext;\n}\n"]}

package/lib/LinkedRdfSourcesAsyncRdfIterator.d.ts

@@ -13,12 +13,17 @@ export declare abstract class LinkedRdfSourcesAsyncRdfIterator extends BufferedI
     protected readonly firstUrl: string;
     private readonly maxIterators;
     private readonly sourceStateGetter;
-    private started;
+    protected started: boolean;
     private readonly currentIterators;
     private iteratorsPendingCreation;
+    private iteratorsPendingTermination;
     private accumulatedMetadata;
     private preflightMetadata;
     constructor(cacheSize: number, operation: Algebra.Operation, queryBindingsOptions: IQueryBindingsOptions | undefined, context: IActionContext, firstUrl: string, maxIterators: number, sourceStateGetter: SourceStateGetter, options?: BufferedIteratorOptions);
+    /**
+     * Start filling the buffer of this iterator.
+     */
+    kickstart(): void;
     getProperty<P>(propertyName: string, callback?: (value: P) => void): P | undefined;
     protected _end(destroy?: boolean): void;
     /**
@@ -33,6 +38,7 @@ export declare abstract class LinkedRdfSourcesAsyncRdfIterator extends BufferedI
     protected abstract getSourceLinks(metadata: Record<string, any>): Promise<ILink[]>;
     _read(count: number, done: () => void): void;
     protected canStartNewIterator(): boolean;
+    protected canStartNewIteratorConsiderReadable(): boolean;
     protected areIteratorsRunning(): boolean;
     /**
      * Append the fields from appendingMetadata into accumulatedMetadata.
@@ -57,7 +63,7 @@ export declare abstract class LinkedRdfSourcesAsyncRdfIterator extends BufferedI
      * @param canClose
      */
     protected startIteratorsForNextUrls(handledDatasets: Record<string, boolean>, canClose: boolean): void;
-    protected isCloseable(linkQueue: ILinkQueue): boolean;
+    protected isCloseable(linkQueue: ILinkQueue, requireQueueEmpty: boolean): boolean;
 }
 /**
  * The current state of a source.

package/lib/LinkedRdfSourcesAsyncRdfIterator.js

@@ -9,6 +9,7 @@ class LinkedRdfSourcesAsyncRdfIterator extends asynciterator_1.BufferedIterator
         this.started = false;
         this.currentIterators = [];
         this.iteratorsPendingCreation = 0;
+        this.iteratorsPendingTermination = 0;
         // eslint-disable-next-line unicorn/no-useless-undefined
         this.accumulatedMetadata = Promise.resolve(undefined);
         this._reading = false;
@@ -23,6 +24,14 @@ class LinkedRdfSourcesAsyncRdfIterator extends asynciterator_1.BufferedIterator
             throw new Error(`LinkedRdfSourcesAsyncRdfIterator.maxIterators must be larger than zero, but got ${this.maxIterators}`);
         }
     }
+    /**
+     * Start filling the buffer of this iterator.
+     */
+    kickstart() {
+        if (!this.started) {
+            this._fillBufferAsync();
+        }
+    }
     getProperty(propertyName, callback) {
         if (propertyName === 'metadata' && !this.started) {
             // If the iterator has not started yet, forcefully fetch the metadata from the source without starting the
@@ -104,10 +113,14 @@ class LinkedRdfSourcesAsyncRdfIterator extends asynciterator_1.BufferedIterator
         }
     }
     canStartNewIterator() {
-        return (this.currentIterators.length + this.iteratorsPendingCreation) < this.maxIterators && !this.readable;
+        return (this.currentIterators.length + this.iteratorsPendingCreation + this.iteratorsPendingTermination) <
+            this.maxIterators && (!this.canStartNewIteratorConsiderReadable() || !this.readable);
+    }
+    canStartNewIteratorConsiderReadable() {
+        return true;
     }
     areIteratorsRunning() {
-        return (this.currentIterators.length + this.iteratorsPendingCreation) > 0;
+        return (this.currentIterators.length + this.iteratorsPendingCreation + this.iteratorsPendingTermination) > 0;
     }
     /**
      * Start a new iterator for the given source.
@@ -119,6 +132,7 @@ class LinkedRdfSourcesAsyncRdfIterator extends asynciterator_1.BufferedIterator
         try {
             const iterator = startSource.source.queryBindings(this.operation, this.context, this.queryBindingsOptions);
             this.currentIterators.push(iterator);
+            let receivedEndEvent = false;
             let receivedMetadata = false;
             // Attach readers to the newly created iterator
             // eslint-disable-next-line @typescript-eslint/no-this-alias
@@ -127,6 +141,12 @@ class LinkedRdfSourcesAsyncRdfIterator extends asynciterator_1.BufferedIterator
             iterator.on('readable', () => this._fillBuffer());
             iterator.on('end', () => {
                 this.currentIterators.splice(this.currentIterators.indexOf(iterator), 1);
+                // Indicate that this iterator still needs to flush its next-links.
+                // Without this, the linked iterator could sometimes be closed before next-links are obtained.
+                receivedEndEvent = true;
+                if (!receivedMetadata) {
+                    this.iteratorsPendingTermination++;
+                }
                 // If the metadata was already received, handle the next URL in the queue
                 if (receivedMetadata) {
                     this.startIteratorsForNextUrls(startSource.handledDatasets, true);
@@ -168,6 +188,9 @@ class LinkedRdfSourcesAsyncRdfIterator extends asynciterator_1.BufferedIterator
                             linkQueue.push(nextUrl, startSource.link);
                         }
                         receivedMetadata = true;
+                        if (receivedEndEvent) {
+                            this.iteratorsPendingTermination--;
+                        }
                         this.startIteratorsForNextUrls(startSource.handledDatasets, true);
                     }).catch(error => this.destroy(error));
                     return returnMetadata;
@@ -216,13 +239,13 @@ class LinkedRdfSourcesAsyncRdfIterator extends asynciterator_1.BufferedIterator
                 }
             }
             // Close, only if no other iterators are still running
-            if (canClose && this.isCloseable(linkQueue)) {
+            if (canClose && this.isCloseable(linkQueue, true)) {
                 this.close();
             }
         })
             .catch(error => this.destroy(error));
     }
-    isCloseable(linkQueue) {
+    isCloseable(linkQueue, requireQueueEmpty) {
         return linkQueue.isEmpty() && !this.areIteratorsRunning();
     }
 }

package/lib/LinkedRdfSourcesAsyncRdfIterator.js.map

@@ -1 +1 @@
-{"version":3,"file":"LinkedRdfSourcesAsyncRdfIterator.js","sourceRoot":"","sources":["LinkedRdfSourcesAsyncRdfIterator.ts"],"names":[],"mappings":";;;AAEA,iDAA6D;AAI7D,iDAAiD;AAGjD,MAAsB,gCAAiC,SAAQ,gCAA8B;IAiB3F,YACE,SAAiB,EACjB,SAA4B,EAC5B,oBAAuD,EACvD,OAAuB,EACvB,QAAgB,EAChB,YAAoB,EACpB,iBAAoC,EACpC,OAAiC;QAEjC,KAAK,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;QAjBlC,YAAO,GAAG,KAAK,CAAC;QACP,qBAAgB,GAAkC,EAAE,CAAC;QAC9D,6BAAwB,GAAG,CAAC,CAAC;QACrC,wDAAwD;QAChD,wBAAmB,GAA0C,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAc9F,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;QACjD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAE3C,IAAI,IAAI,CAAC,YAAY,IAAI,CAAC,EAAE;YAC1B,MAAM,IAAI,KAAK,CAAC,mFAAmF,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;SACzH;IACH,CAAC;IAEM,WAAW,CAAI,YAAoB,EAAE,QAA6B;QACvE,IAAI,YAAY,KAAK,UAAU,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YAChD,0GAA0G;YAC1G,iDAAiD;YACjD,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE;gBAC3B,IAAI,CAAC,iBAAiB,GAAG,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;oBACvD,IAAI,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC;yBAC/C,IAAI,CAAC,WAAW,CAAC,EAAE;wBAClB,6EAA6E;wBAC7E,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;wBACtF,cAAc,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,QAA0B,EAAE,EAAE;4BACpE,QAAQ,CAAC,KAAK,GAAG,IAAI,kCAAuB,EAAE,CAAC;4BAC/C,OAAO,CAAC,QAAQ,CAAC,CAAC;4BAClB,cAAc,CAAC,OAAO,EAAE,CAAC;wBAC3B,CAAC,CAAC,CAAC;oBACL,CAAC,CAAC;yBACD,KAAK,CAAC,MAAM,CAAC,CAAC;gBACnB,CAAC,CAAC,CAAC;aACJ;YACD,IAAI,CAAC,iBAAiB;iBACnB,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;iBACxD,KAAK,CAAC,GAAG,EAAE;gBACV,gBAAgB;YAClB,CAAC,CAAC,CAAC;SACN;QACD,OAAO,KAAK,CAAC,WAAW,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;IACnD,CAAC;IAES,IAAI,CAAC,OAAiB;QAC9B,8BAA8B;QAC9B,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,gBAAgB,EAAE;YACtC,EAAE,CAAC,OAAO,EAAE,CAAC;SACd;QAED,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACtB,CAAC;IAcM,KAAK,CAAC,KAAa,EAAE,IAAgB;QAC1C,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACjB,0DAA0D;YAC1D,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YAEpB,4DAA4D;YAC5D,IAAI,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC;iBAC/C,IAAI,CAAC,WAAW,CAAC,EAAE;gBAClB,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;gBAChC,IAAI,EAAE,CAAC;YACT,CAAC,CAAC;gBACF,4EAA4E;iBAC3E,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;SAC1D;aAAM;YACL,kCAAkC;YAClC,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,gBAAgB,EAAE;gBAC5C,OAAO,KAAK,GAAG,CAAC,EAAE;oBAChB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;oBAC7B,IAAI,IAAI,KAAK,IAAI,EAAE;wBACjB,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;qBAClB;yBAAM;wBACL,MAAM;qBACP;iBACF;gBACD,IAAI,KAAK,IAAI,CAAC,EAAE;oBACd,MAAM;iBACP;aACF;YAED,mCAAmC;YACnC,IAAI,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC,mBAAmB,EAAE,EAAE;gBAC5C,gFAAgF;gBAChF,wDAAwD;gBACxD,mEAAmE;gBACnE,IAAI,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC;qBAC/C,IAAI,CAAC,WAAW,CAAC,EAAE;oBAClB,IAAI,CAAC,yBAAyB,CAAC,WAAW,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;oBACnE,IAAI,EAAE,CAAC;gBACT,CAAC,CAAC,CAAC;aACN;iBAAM;gBACL,IAAI,EAAE,CAAC;aACR;SACF;IACH,CAAC;IAES,mBAAmB;QAC3B,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,IAAI,CAAC,wBAAwB,CAAC,GAAG,IAAI,CAAC,YAAY,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC;IAC9G,CAAC;IAES,mBAAmB;QAC3B,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,CAAC;IAC5E,CAAC;IAaD;;;;OAIG;IACO,aAAa,CAAC,WAAyB;QAC/C,sDAAsD;QACtD,IAAI;YACF,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAC3G,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACrC,IAAI,gBAAgB,GAAG,KAAK,CAAC;YAE7B,+CAA+C;YAC/C,4DAA4D;YACtD,QAAS,CAAC,YAAY,GAAG,IAAI,CAAC;YACpC,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;YAC5D,QAAQ,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YAClD,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACtB,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;gBAEzE,yEAAyE;gBACzE,IAAI,gBAAgB,EAAE;oBACpB,IAAI,CAAC,yBAAyB,CAAC,WAAW,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;iBACnE;YACH,CAAC,CAAC,CAAC;YAEH,wCAAwC;YACxC,gDAAgD;YAChD,QAAQ,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,QAA0B,EAAE,EAAE;gBAC9D,iCAAiC;gBACjC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB;qBAChD,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC,KAAK,IAAG,EAAE;oBACnC,IAAI,CAAC,gBAAgB,EAAE;wBACrB,gBAAgB,GAAG,WAAW,CAAC,QAAQ,CAAC;qBACzC;oBACD,OAAO,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,QAAQ,CAAC,CAAC;gBAC7D,CAAC,CAAC,EAAE;qBACD,IAAI,CAAC,mBAAmB,CAAC,EAAE;oBAC1B,yDAAyD;oBACzD,MAAM,cAAc,GAAG,EAAE,GAAG,WAAW,CAAC,QAAQ,EAAE,GAAG,QAAQ,EAAE,GAAG,mBAAmB,EAAE,CAAC;oBAExF,4BAA4B;oBAC5B,cAAc,CAAC,KAAK,GAAG,IAAI,kCAAuB,EAAE,CAAC;oBAErD,iEAAiE;oBACjE,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC,CAAC;oBAEpC,oCAAoC;oBACpC,IAAI,IAAI,CAAC,iBAAiB,EAAE;wBAC1B,IAAI,CAAC,iBAAiB;6BACnB,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;6BACjD,KAAK,CAAC,GAAG,EAAE;4BACV,gBAAgB;wBAClB,CAAC,CAAC,CAAC;qBACN;oBAED,wEAAwE;oBACxE,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC;yBAChC,IAAI,CAAC,CAAC,QAAiB,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;yBAClD,IAAI,CAAC,KAAK,EAAC,QAAiB,EAAE,EAAE;wBAC/B,oCAAoC;wBACpC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;wBAC5C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;4BAC9B,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC;yBAC3C;wBAED,gBAAgB,GAAG,IAAI,CAAC;wBACxB,IAAI,CAAC,yBAAyB,CAAC,WAAW,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;oBACpE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;oBAEzC,OAAO,cAAc,CAAC;gBACxB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;oBAClB,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;oBACpB,OAAyB,EAAE,CAAC;gBAC9B,CAAC,CAAC,CAAC;YACP,CAAC,CAAC,CAAC;SACJ;QAAC,OAAO,SAAkB,EAAE;YAC3B,IAAI,CAAC,OAAO,CAAS,SAAS,CAAC,CAAC;SACjC;IACH,CAAC;IAES,cAAc,CAAC,WAA6B;QACpD,MAAM,oBAAoB,GAAG,IAAI,CAAC,WAAW,CAAmB,UAAU,CAAC,CAAC;QAC5E,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;QAC1C,oBAAoB,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC;IAC3C,CAAC;IAES,SAAS;QACjB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;IACpB,CAAC;IAED;;;;;;OAMG;IACO,yBAAyB,CAAC,eAAwC,EAAE,QAAiB;QAC7F,IAAI,CAAC,YAAY,EAAE;aAChB,IAAI,CAAC,SAAS,CAAC,EAAE;YAChB,2CAA2C;YAC3C,OAAO,IAAI,CAAC,mBAAmB,EAAE,IAAI,IAAI,CAAC,SAAS,EAAE,EAAE;gBACrD,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,EAAE,CAAC;gBACjC,IAAI,QAAQ,EAAE;oBACZ,IAAI,CAAC,wBAAwB,EAAE,CAAC;oBAChC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,eAAe,CAAC;yBAC9C,IAAI,CAAC,eAAe,CAAC,EAAE;wBACtB,IAAI,CAAC,wBAAwB,EAAE,CAAC;wBAChC,IAAI,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC;oBACtC,CAAC,CAAC;yBACD,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;iBAC9C;qBAAM;oBACL,MAAM;iBACP;aACF;YAED,sDAAsD;YACtD,IAAI,QAAQ,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,EAAE;gBAC3C,IAAI,CAAC,KAAK,EAAE,CAAC;aACd;QACH,CAAC,CAAC;aACD,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IACzC,CAAC;IAES,WAAW,CAAC,SAAqB;QACzC,OAAO,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC;IAC5D,CAAC;CACF;AA5RD,4EA4RC","sourcesContent":["import type { ILink } from '@comunica/bus-rdf-resolve-hypermedia-links';\nimport type { ILinkQueue } from '@comunica/bus-rdf-resolve-hypermedia-links-queue';\nimport { MetadataValidationState } from '@comunica/metadata';\nimport type { IQuerySource, IActionContext, MetadataBindings, IQueryBindingsOptions } from '@comunica/types';\nimport type * as RDF from '@rdfjs/types';\nimport type { AsyncIterator, BufferedIteratorOptions } from 'asynciterator';\nimport { BufferedIterator } from 'asynciterator';\nimport type { Algebra } from 'sparqlalgebrajs';\n\nexport abstract class LinkedRdfSourcesAsyncRdfIterator extends BufferedIterator<RDF.Bindings> {\n  protected readonly operation: Algebra.Operation;\n  protected readonly queryBindingsOptions: IQueryBindingsOptions | undefined;\n  protected readonly context: IActionContext;\n\n  private readonly cacheSize: number;\n  protected readonly firstUrl: string;\n  private readonly maxIterators: number;\n  private readonly sourceStateGetter: SourceStateGetter;\n\n  private started = false;\n  private readonly currentIterators: AsyncIterator<RDF.Bindings>[] = [];\n  private iteratorsPendingCreation = 0;\n  // eslint-disable-next-line unicorn/no-useless-undefined\n  private accumulatedMetadata: Promise<MetadataBindings | undefined> = Promise.resolve(undefined);\n  private preflightMetadata: Promise<MetadataBindings> | undefined;\n\n  public constructor(\n    cacheSize: number,\n    operation: Algebra.Operation,\n    queryBindingsOptions: IQueryBindingsOptions | undefined,\n    context: IActionContext,\n    firstUrl: string,\n    maxIterators: number,\n    sourceStateGetter: SourceStateGetter,\n    options?: BufferedIteratorOptions,\n  ) {\n    super({ autoStart: false, ...options });\n    this._reading = false;\n    this.cacheSize = cacheSize;\n    this.operation = operation;\n    this.queryBindingsOptions = queryBindingsOptions;\n    this.context = context;\n    this.firstUrl = firstUrl;\n    this.maxIterators = maxIterators;\n    this.sourceStateGetter = sourceStateGetter;\n\n    if (this.maxIterators <= 0) {\n      throw new Error(`LinkedRdfSourcesAsyncRdfIterator.maxIterators must be larger than zero, but got ${this.maxIterators}`);\n    }\n  }\n\n  public getProperty<P>(propertyName: string, callback?: (value: P) => void): P | undefined {\n    if (propertyName === 'metadata' && !this.started) {\n      // If the iterator has not started yet, forcefully fetch the metadata from the source without starting the\n      // iterator. This way, we keep the iterator lazy.\n      if (!this.preflightMetadata) {\n        this.preflightMetadata = new Promise((resolve, reject) => {\n          this.sourceStateGetter({ url: this.firstUrl }, {})\n            .then(sourceState => {\n              // Don't pass query options, as we don't want to consume any passed iterators\n              const bindingsStream = sourceState.source.queryBindings(this.operation, this.context);\n              bindingsStream.getProperty('metadata', (metadata: MetadataBindings) => {\n                metadata.state = new MetadataValidationState();\n                resolve(metadata);\n                bindingsStream.destroy();\n              });\n            })\n            .catch(reject);\n        });\n      }\n      this.preflightMetadata\n        .then(metadata => this.setProperty('metadata', metadata))\n        .catch(() => {\n          // Ignore errors\n        });\n    }\n    return super.getProperty(propertyName, callback);\n  }\n\n  protected _end(destroy?: boolean): void {\n    // Close all running iterators\n    for (const it of this.currentIterators) {\n      it.destroy();\n    }\n\n    super._end(destroy);\n  }\n\n  /**\n   * Get the internal link queue.\n   * The returned instance must always be the same.\n   */\n  public abstract getLinkQueue(): Promise<ILinkQueue>;\n\n  /**\n   * Determine the links to be followed from the current source given its metadata.\n   * @param metadata The metadata of a source.\n   */\n  protected abstract getSourceLinks(metadata: Record<string, any>): Promise<ILink[]>;\n\n  public _read(count: number, done: () => void): void {\n    if (!this.started) {\n      // The first time this is called, prepare the first source\n      this.started = true;\n\n      // Await the source to be set, and start the source iterator\n      this.sourceStateGetter({ url: this.firstUrl }, {})\n        .then(sourceState => {\n          this.startIterator(sourceState);\n          done();\n        })\n        // Destroy should be async because it can be called before it is listened to\n        .catch(error => setTimeout(() => this.destroy(error)));\n    } else {\n      // Read from all current iterators\n      for (const iterator of this.currentIterators) {\n        while (count > 0) {\n          const read = iterator.read();\n          if (read !== null) {\n            count--;\n            this._push(read);\n          } else {\n            break;\n          }\n        }\n        if (count <= 0) {\n          break;\n        }\n      }\n\n      // Schedule new iterators if needed\n      if (count >= 0 && this.canStartNewIterator()) {\n        // We can safely ignore skip catching the error, since we are guaranteed to have\n        // successfully got the source for this.firstUrl before.\n        // eslint-disable-next-line @typescript-eslint/no-floating-promises\n        this.sourceStateGetter({ url: this.firstUrl }, {})\n          .then(sourceState => {\n            this.startIteratorsForNextUrls(sourceState.handledDatasets, false);\n            done();\n          });\n      } else {\n        done();\n      }\n    }\n  }\n\n  protected canStartNewIterator(): boolean {\n    return (this.currentIterators.length + this.iteratorsPendingCreation) < this.maxIterators && !this.readable;\n  }\n\n  protected areIteratorsRunning(): boolean {\n    return (this.currentIterators.length + this.iteratorsPendingCreation) > 0;\n  }\n\n  /**\n   * Append the fields from appendingMetadata into accumulatedMetadata.\n   * @param accumulatedMetadata The fields to append to.\n   * @param appendingMetadata The fields to append.\n   * @protected\n   */\n  protected abstract accumulateMetadata(\n    accumulatedMetadata: MetadataBindings,\n    appendingMetadata: MetadataBindings,\n  ): Promise<MetadataBindings>;\n\n  /**\n   * Start a new iterator for the given source.\n   * Once the iterator is done, it will either determine a new source, or it will close the linked iterator.\n   * @param {ISourceState} startSource The start source state.\n   */\n  protected startIterator(startSource: ISourceState): void {\n    // Delegate the quad pattern query to the given source\n    try {\n      const iterator = startSource.source.queryBindings(this.operation, this.context, this.queryBindingsOptions);\n      this.currentIterators.push(iterator);\n      let receivedMetadata = false;\n\n      // Attach readers to the newly created iterator\n      // eslint-disable-next-line @typescript-eslint/no-this-alias\n      (<any>iterator)._destination = this;\n      iterator.on('error', (error: Error) => this.destroy(error));\n      iterator.on('readable', () => this._fillBuffer());\n      iterator.on('end', () => {\n        this.currentIterators.splice(this.currentIterators.indexOf(iterator), 1);\n\n        // If the metadata was already received, handle the next URL in the queue\n        if (receivedMetadata) {\n          this.startIteratorsForNextUrls(startSource.handledDatasets, true);\n        }\n      });\n\n      // Listen for the metadata of the source\n      // The metadata property is guaranteed to be set\n      iterator.getProperty('metadata', (metadata: MetadataBindings) => {\n        // Accumulate the metadata object\n        this.accumulatedMetadata = this.accumulatedMetadata\n          .then(previousMetadata => (async() => {\n            if (!previousMetadata) {\n              previousMetadata = startSource.metadata;\n            }\n            return this.accumulateMetadata(previousMetadata, metadata);\n          })()\n            .then(accumulatedMetadata => {\n              // Also merge fields that were not explicitly accumulated\n              const returnMetadata = { ...startSource.metadata, ...metadata, ...accumulatedMetadata };\n\n              // Create new metadata state\n              returnMetadata.state = new MetadataValidationState();\n\n              // Emit metadata, and invalidate any metadata that was set before\n              this.updateMetadata(returnMetadata);\n\n              // Invalidate any preflight metadata\n              if (this.preflightMetadata) {\n                this.preflightMetadata\n                  .then(metadataIn => metadataIn.state.invalidate())\n                  .catch(() => {\n                    // Ignore errors\n                  });\n              }\n\n              // Determine next urls, which will eventually become a next-next source.\n              this.getSourceLinks(returnMetadata)\n                .then((nextUrls: ILink[]) => Promise.all(nextUrls))\n                .then(async(nextUrls: ILink[]) => {\n                  // Append all next URLs to our queue\n                  const linkQueue = await this.getLinkQueue();\n                  for (const nextUrl of nextUrls) {\n                    linkQueue.push(nextUrl, startSource.link);\n                  }\n\n                  receivedMetadata = true;\n                  this.startIteratorsForNextUrls(startSource.handledDatasets, true);\n                }).catch(error => this.destroy(error));\n\n              return returnMetadata;\n            })).catch(error => {\n            this.destroy(error);\n            return <MetadataBindings>{};\n          });\n      });\n    } catch (syncError: unknown) {\n      this.destroy(<Error> syncError);\n    }\n  }\n\n  protected updateMetadata(metadataNew: MetadataBindings): void {\n    const metadataToInvalidate = this.getProperty<MetadataBindings>('metadata');\n    this.setProperty('metadata', metadataNew);\n    metadataToInvalidate?.state.invalidate();\n  }\n\n  protected isRunning(): boolean {\n    return !this.done;\n  }\n\n  /**\n   * Check if a next URL is in the queue.\n   * If yes, start a new iterator.\n   * If no, close this iterator.\n   * @param handledDatasets\n   * @param canClose\n   */\n  protected startIteratorsForNextUrls(handledDatasets: Record<string, boolean>, canClose: boolean): void {\n    this.getLinkQueue()\n      .then(linkQueue => {\n        // Create as many new iterators as possible\n        while (this.canStartNewIterator() && this.isRunning()) {\n          const nextLink = linkQueue.pop();\n          if (nextLink) {\n            this.iteratorsPendingCreation++;\n            this.sourceStateGetter(nextLink, handledDatasets)\n              .then(nextSourceState => {\n                this.iteratorsPendingCreation--;\n                this.startIterator(nextSourceState);\n              })\n              .catch(error => this.emit('error', error));\n          } else {\n            break;\n          }\n        }\n\n        // Close, only if no other iterators are still running\n        if (canClose && this.isCloseable(linkQueue)) {\n          this.close();\n        }\n      })\n      .catch(error => this.destroy(error));\n  }\n\n  protected isCloseable(linkQueue: ILinkQueue): boolean {\n    return linkQueue.isEmpty() && !this.areIteratorsRunning();\n  }\n}\n\n/**\n * The current state of a source.\n * This is needed for following links within a source.\n */\nexport interface ISourceState {\n  /**\n   * The link to this source.\n   */\n  link: ILink;\n  /**\n   * A source.\n   */\n  source: IQuerySource;\n  /**\n   * The source's initial metadata.\n   */\n  metadata: MetadataBindings;\n  /**\n   * All dataset identifiers that have been passed for this source.\n   */\n  handledDatasets: Record<string, boolean>;\n}\n\nexport type SourceStateGetter = (link: ILink, handledDatasets: Record<string, boolean>) => Promise<ISourceState>;\n"]}
+{"version":3,"file":"LinkedRdfSourcesAsyncRdfIterator.js","sourceRoot":"","sources":["LinkedRdfSourcesAsyncRdfIterator.ts"],"names":[],"mappings":";;;AAEA,iDAA6D;AAI7D,iDAAiD;AAGjD,MAAsB,gCAAiC,SAAQ,gCAA8B;IAkB3F,YACE,SAAiB,EACjB,SAA4B,EAC5B,oBAAuD,EACvD,OAAuB,EACvB,QAAgB,EAChB,YAAoB,EACpB,iBAAoC,EACpC,OAAiC;QAEjC,KAAK,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;QAlBhC,YAAO,GAAG,KAAK,CAAC;QACT,qBAAgB,GAAkC,EAAE,CAAC;QAC9D,6BAAwB,GAAG,CAAC,CAAC;QAC7B,gCAA2B,GAAG,CAAC,CAAC;QACxC,wDAAwD;QAChD,wBAAmB,GAA0C,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAc9F,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;QACjD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAE3C,IAAI,IAAI,CAAC,YAAY,IAAI,CAAC,EAAE;YAC1B,MAAM,IAAI,KAAK,CAAC,mFAAmF,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;SACzH;IACH,CAAC;IAED;;OAEG;IACI,SAAS;QACd,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACjB,IAAI,CAAC,gBAAgB,EAAE,CAAC;SACzB;IACH,CAAC;IAEM,WAAW,CAAI,YAAoB,EAAE,QAA6B;QACvE,IAAI,YAAY,KAAK,UAAU,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YAChD,0GAA0G;YAC1G,iDAAiD;YACjD,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE;gBAC3B,IAAI,CAAC,iBAAiB,GAAG,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;oBACvD,IAAI,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC;yBAC/C,IAAI,CAAC,WAAW,CAAC,EAAE;wBAClB,6EAA6E;wBAC7E,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;wBACtF,cAAc,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,QAA0B,EAAE,EAAE;4BACpE,QAAQ,CAAC,KAAK,GAAG,IAAI,kCAAuB,EAAE,CAAC;4BAC/C,OAAO,CAAC,QAAQ,CAAC,CAAC;4BAClB,cAAc,CAAC,OAAO,EAAE,CAAC;wBAC3B,CAAC,CAAC,CAAC;oBACL,CAAC,CAAC;yBACD,KAAK,CAAC,MAAM,CAAC,CAAC;gBACnB,CAAC,CAAC,CAAC;aACJ;YACD,IAAI,CAAC,iBAAiB;iBACnB,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;iBACxD,KAAK,CAAC,GAAG,EAAE;gBACV,gBAAgB;YAClB,CAAC,CAAC,CAAC;SACN;QACD,OAAO,KAAK,CAAC,WAAW,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;IACnD,CAAC;IAES,IAAI,CAAC,OAAiB;QAC9B,8BAA8B;QAC9B,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,gBAAgB,EAAE;YACtC,EAAE,CAAC,OAAO,EAAE,CAAC;SACd;QAED,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACtB,CAAC;IAcM,KAAK,CAAC,KAAa,EAAE,IAAgB;QAC1C,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACjB,0DAA0D;YAC1D,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YAEpB,4DAA4D;YAC5D,IAAI,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC;iBAC/C,IAAI,CAAC,WAAW,CAAC,EAAE;gBAClB,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;gBAChC,IAAI,EAAE,CAAC;YACT,CAAC,CAAC;gBACF,4EAA4E;iBAC3E,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;SAC1D;aAAM;YACL,kCAAkC;YAClC,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,gBAAgB,EAAE;gBAC5C,OAAO,KAAK,GAAG,CAAC,EAAE;oBAChB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;oBAC7B,IAAI,IAAI,KAAK,IAAI,EAAE;wBACjB,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;qBAClB;yBAAM;wBACL,MAAM;qBACP;iBACF;gBACD,IAAI,KAAK,IAAI,CAAC,EAAE;oBACd,MAAM;iBACP;aACF;YAED,mCAAmC;YACnC,IAAI,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC,mBAAmB,EAAE,EAAE;gBAC5C,gFAAgF;gBAChF,wDAAwD;gBACxD,mEAAmE;gBACnE,IAAI,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC;qBAC/C,IAAI,CAAC,WAAW,CAAC,EAAE;oBAClB,IAAI,CAAC,yBAAyB,CAAC,WAAW,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;oBACnE,IAAI,EAAE,CAAC;gBACT,CAAC,CAAC,CAAC;aACN;iBAAM;gBACL,IAAI,EAAE,CAAC;aACR;SACF;IACH,CAAC;IAES,mBAAmB;QAC3B,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC,2BAA2B,CAAC;YACtG,IAAI,CAAC,YAAY,IAAI,CAAC,CAAC,IAAI,CAAC,mCAAmC,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACzF,CAAC;IAES,mCAAmC;QAC3C,OAAO,IAAI,CAAC;IACd,CAAC;IAES,mBAAmB;QAC3B,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC,2BAA2B,CAAC,GAAG,CAAC,CAAC;IAC/G,CAAC;IAaD;;;;OAIG;IACO,aAAa,CAAC,WAAyB;QAC/C,sDAAsD;QACtD,IAAI;YACF,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAC3G,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACrC,IAAI,gBAAgB,GAAG,KAAK,CAAC;YAC7B,IAAI,gBAAgB,GAAG,KAAK,CAAC;YAE7B,+CAA+C;YAC/C,4DAA4D;YACtD,QAAS,CAAC,YAAY,GAAG,IAAI,CAAC;YACpC,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;YAC5D,QAAQ,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YAClD,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACtB,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;gBAEzE,mEAAmE;gBACnE,8FAA8F;gBAC9F,gBAAgB,GAAG,IAAI,CAAC;gBACxB,IAAI,CAAC,gBAAgB,EAAE;oBACrB,IAAI,CAAC,2BAA2B,EAAE,CAAC;iBACpC;gBAED,yEAAyE;gBACzE,IAAI,gBAAgB,EAAE;oBACpB,IAAI,CAAC,yBAAyB,CAAC,WAAW,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;iBACnE;YACH,CAAC,CAAC,CAAC;YAEH,wCAAwC;YACxC,gDAAgD;YAChD,QAAQ,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,QAA0B,EAAE,EAAE;gBAC9D,iCAAiC;gBACjC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB;qBAChD,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC,KAAK,IAAG,EAAE;oBACnC,IAAI,CAAC,gBAAgB,EAAE;wBACrB,gBAAgB,GAAG,WAAW,CAAC,QAAQ,CAAC;qBACzC;oBACD,OAAO,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,QAAQ,CAAC,CAAC;gBAC7D,CAAC,CAAC,EAAE;qBACD,IAAI,CAAC,mBAAmB,CAAC,EAAE;oBAC1B,yDAAyD;oBACzD,MAAM,cAAc,GAAG,EAAE,GAAG,WAAW,CAAC,QAAQ,EAAE,GAAG,QAAQ,EAAE,GAAG,mBAAmB,EAAE,CAAC;oBAExF,4BAA4B;oBAC5B,cAAc,CAAC,KAAK,GAAG,IAAI,kCAAuB,EAAE,CAAC;oBAErD,iEAAiE;oBACjE,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC,CAAC;oBAEpC,oCAAoC;oBACpC,IAAI,IAAI,CAAC,iBAAiB,EAAE;wBAC1B,IAAI,CAAC,iBAAiB;6BACnB,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;6BACjD,KAAK,CAAC,GAAG,EAAE;4BACV,gBAAgB;wBAClB,CAAC,CAAC,CAAC;qBACN;oBAED,wEAAwE;oBACxE,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC;yBAChC,IAAI,CAAC,CAAC,QAAiB,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;yBAClD,IAAI,CAAC,KAAK,EAAC,QAAiB,EAAE,EAAE;wBAC/B,oCAAoC;wBACpC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;wBAC5C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;4BAC9B,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC;yBAC3C;wBAED,gBAAgB,GAAG,IAAI,CAAC;wBACxB,IAAI,gBAAgB,EAAE;4BACpB,IAAI,CAAC,2BAA2B,EAAE,CAAC;yBACpC;wBAED,IAAI,CAAC,yBAAyB,CAAC,WAAW,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;oBACpE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;oBAEzC,OAAO,cAAc,CAAC;gBACxB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;oBAClB,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;oBACpB,OAAyB,EAAE,CAAC;gBAC9B,CAAC,CAAC,CAAC;YACP,CAAC,CAAC,CAAC;SACJ;QAAC,OAAO,SAAkB,EAAE;YAC3B,IAAI,CAAC,OAAO,CAAS,SAAS,CAAC,CAAC;SACjC;IACH,CAAC;IAES,cAAc,CAAC,WAA6B;QACpD,MAAM,oBAAoB,GAAG,IAAI,CAAC,WAAW,CAAmB,UAAU,CAAC,CAAC;QAC5E,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;QAC1C,oBAAoB,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC;IAC3C,CAAC;IAES,SAAS;QACjB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;IACpB,CAAC;IAED;;;;;;OAMG;IACO,yBAAyB,CAAC,eAAwC,EAAE,QAAiB;QAC7F,IAAI,CAAC,YAAY,EAAE;aAChB,IAAI,CAAC,SAAS,CAAC,EAAE;YAChB,2CAA2C;YAC3C,OAAO,IAAI,CAAC,mBAAmB,EAAE,IAAI,IAAI,CAAC,SAAS,EAAE,EAAE;gBACrD,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,EAAE,CAAC;gBACjC,IAAI,QAAQ,EAAE;oBACZ,IAAI,CAAC,wBAAwB,EAAE,CAAC;oBAChC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,eAAe,CAAC;yBAC9C,IAAI,CAAC,eAAe,CAAC,EAAE;wBACtB,IAAI,CAAC,wBAAwB,EAAE,CAAC;wBAChC,IAAI,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC;oBACtC,CAAC,CAAC;yBACD,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;iBAC9C;qBAAM;oBACL,MAAM;iBACP;aACF;YAED,sDAAsD;YACtD,IAAI,QAAQ,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE;gBACjD,IAAI,CAAC,KAAK,EAAE,CAAC;aACd;QACH,CAAC,CAAC;aACD,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IACzC,CAAC;IAES,WAAW,CAAC,SAAqB,EAAE,iBAA0B;QACrE,OAAO,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC;IAC5D,CAAC;CACF;AAvTD,4EAuTC","sourcesContent":["import type { ILink } from '@comunica/bus-rdf-resolve-hypermedia-links';\nimport type { ILinkQueue } from '@comunica/bus-rdf-resolve-hypermedia-links-queue';\nimport { MetadataValidationState } from '@comunica/metadata';\nimport type { IQuerySource, IActionContext, MetadataBindings, IQueryBindingsOptions } from '@comunica/types';\nimport type * as RDF from '@rdfjs/types';\nimport type { AsyncIterator, BufferedIteratorOptions } from 'asynciterator';\nimport { BufferedIterator } from 'asynciterator';\nimport type { Algebra } from 'sparqlalgebrajs';\n\nexport abstract class LinkedRdfSourcesAsyncRdfIterator extends BufferedIterator<RDF.Bindings> {\n  protected readonly operation: Algebra.Operation;\n  protected readonly queryBindingsOptions: IQueryBindingsOptions | undefined;\n  protected readonly context: IActionContext;\n\n  private readonly cacheSize: number;\n  protected readonly firstUrl: string;\n  private readonly maxIterators: number;\n  private readonly sourceStateGetter: SourceStateGetter;\n\n  protected started = false;\n  private readonly currentIterators: AsyncIterator<RDF.Bindings>[] = [];\n  private iteratorsPendingCreation = 0;\n  private iteratorsPendingTermination = 0;\n  // eslint-disable-next-line unicorn/no-useless-undefined\n  private accumulatedMetadata: Promise<MetadataBindings | undefined> = Promise.resolve(undefined);\n  private preflightMetadata: Promise<MetadataBindings> | undefined;\n\n  public constructor(\n    cacheSize: number,\n    operation: Algebra.Operation,\n    queryBindingsOptions: IQueryBindingsOptions | undefined,\n    context: IActionContext,\n    firstUrl: string,\n    maxIterators: number,\n    sourceStateGetter: SourceStateGetter,\n    options?: BufferedIteratorOptions,\n  ) {\n    super({ autoStart: false, ...options });\n    this._reading = false;\n    this.cacheSize = cacheSize;\n    this.operation = operation;\n    this.queryBindingsOptions = queryBindingsOptions;\n    this.context = context;\n    this.firstUrl = firstUrl;\n    this.maxIterators = maxIterators;\n    this.sourceStateGetter = sourceStateGetter;\n\n    if (this.maxIterators <= 0) {\n      throw new Error(`LinkedRdfSourcesAsyncRdfIterator.maxIterators must be larger than zero, but got ${this.maxIterators}`);\n    }\n  }\n\n  /**\n   * Start filling the buffer of this iterator.\n   */\n  public kickstart(): void {\n    if (!this.started) {\n      this._fillBufferAsync();\n    }\n  }\n\n  public getProperty<P>(propertyName: string, callback?: (value: P) => void): P | undefined {\n    if (propertyName === 'metadata' && !this.started) {\n      // If the iterator has not started yet, forcefully fetch the metadata from the source without starting the\n      // iterator. This way, we keep the iterator lazy.\n      if (!this.preflightMetadata) {\n        this.preflightMetadata = new Promise((resolve, reject) => {\n          this.sourceStateGetter({ url: this.firstUrl }, {})\n            .then(sourceState => {\n              // Don't pass query options, as we don't want to consume any passed iterators\n              const bindingsStream = sourceState.source.queryBindings(this.operation, this.context);\n              bindingsStream.getProperty('metadata', (metadata: MetadataBindings) => {\n                metadata.state = new MetadataValidationState();\n                resolve(metadata);\n                bindingsStream.destroy();\n              });\n            })\n            .catch(reject);\n        });\n      }\n      this.preflightMetadata\n        .then(metadata => this.setProperty('metadata', metadata))\n        .catch(() => {\n          // Ignore errors\n        });\n    }\n    return super.getProperty(propertyName, callback);\n  }\n\n  protected _end(destroy?: boolean): void {\n    // Close all running iterators\n    for (const it of this.currentIterators) {\n      it.destroy();\n    }\n\n    super._end(destroy);\n  }\n\n  /**\n   * Get the internal link queue.\n   * The returned instance must always be the same.\n   */\n  public abstract getLinkQueue(): Promise<ILinkQueue>;\n\n  /**\n   * Determine the links to be followed from the current source given its metadata.\n   * @param metadata The metadata of a source.\n   */\n  protected abstract getSourceLinks(metadata: Record<string, any>): Promise<ILink[]>;\n\n  public _read(count: number, done: () => void): void {\n    if (!this.started) {\n      // The first time this is called, prepare the first source\n      this.started = true;\n\n      // Await the source to be set, and start the source iterator\n      this.sourceStateGetter({ url: this.firstUrl }, {})\n        .then(sourceState => {\n          this.startIterator(sourceState);\n          done();\n        })\n        // Destroy should be async because it can be called before it is listened to\n        .catch(error => setTimeout(() => this.destroy(error)));\n    } else {\n      // Read from all current iterators\n      for (const iterator of this.currentIterators) {\n        while (count > 0) {\n          const read = iterator.read();\n          if (read !== null) {\n            count--;\n            this._push(read);\n          } else {\n            break;\n          }\n        }\n        if (count <= 0) {\n          break;\n        }\n      }\n\n      // Schedule new iterators if needed\n      if (count >= 0 && this.canStartNewIterator()) {\n        // We can safely ignore skip catching the error, since we are guaranteed to have\n        // successfully got the source for this.firstUrl before.\n        // eslint-disable-next-line @typescript-eslint/no-floating-promises\n        this.sourceStateGetter({ url: this.firstUrl }, {})\n          .then(sourceState => {\n            this.startIteratorsForNextUrls(sourceState.handledDatasets, false);\n            done();\n          });\n      } else {\n        done();\n      }\n    }\n  }\n\n  protected canStartNewIterator(): boolean {\n    return (this.currentIterators.length + this.iteratorsPendingCreation + this.iteratorsPendingTermination) <\n      this.maxIterators && (!this.canStartNewIteratorConsiderReadable() || !this.readable);\n  }\n\n  protected canStartNewIteratorConsiderReadable(): boolean {\n    return true;\n  }\n\n  protected areIteratorsRunning(): boolean {\n    return (this.currentIterators.length + this.iteratorsPendingCreation + this.iteratorsPendingTermination) > 0;\n  }\n\n  /**\n   * Append the fields from appendingMetadata into accumulatedMetadata.\n   * @param accumulatedMetadata The fields to append to.\n   * @param appendingMetadata The fields to append.\n   * @protected\n   */\n  protected abstract accumulateMetadata(\n    accumulatedMetadata: MetadataBindings,\n    appendingMetadata: MetadataBindings,\n  ): Promise<MetadataBindings>;\n\n  /**\n   * Start a new iterator for the given source.\n   * Once the iterator is done, it will either determine a new source, or it will close the linked iterator.\n   * @param {ISourceState} startSource The start source state.\n   */\n  protected startIterator(startSource: ISourceState): void {\n    // Delegate the quad pattern query to the given source\n    try {\n      const iterator = startSource.source.queryBindings(this.operation, this.context, this.queryBindingsOptions);\n      this.currentIterators.push(iterator);\n      let receivedEndEvent = false;\n      let receivedMetadata = false;\n\n      // Attach readers to the newly created iterator\n      // eslint-disable-next-line @typescript-eslint/no-this-alias\n      (<any>iterator)._destination = this;\n      iterator.on('error', (error: Error) => this.destroy(error));\n      iterator.on('readable', () => this._fillBuffer());\n      iterator.on('end', () => {\n        this.currentIterators.splice(this.currentIterators.indexOf(iterator), 1);\n\n        // Indicate that this iterator still needs to flush its next-links.\n        // Without this, the linked iterator could sometimes be closed before next-links are obtained.\n        receivedEndEvent = true;\n        if (!receivedMetadata) {\n          this.iteratorsPendingTermination++;\n        }\n\n        // If the metadata was already received, handle the next URL in the queue\n        if (receivedMetadata) {\n          this.startIteratorsForNextUrls(startSource.handledDatasets, true);\n        }\n      });\n\n      // Listen for the metadata of the source\n      // The metadata property is guaranteed to be set\n      iterator.getProperty('metadata', (metadata: MetadataBindings) => {\n        // Accumulate the metadata object\n        this.accumulatedMetadata = this.accumulatedMetadata\n          .then(previousMetadata => (async() => {\n            if (!previousMetadata) {\n              previousMetadata = startSource.metadata;\n            }\n            return this.accumulateMetadata(previousMetadata, metadata);\n          })()\n            .then(accumulatedMetadata => {\n              // Also merge fields that were not explicitly accumulated\n              const returnMetadata = { ...startSource.metadata, ...metadata, ...accumulatedMetadata };\n\n              // Create new metadata state\n              returnMetadata.state = new MetadataValidationState();\n\n              // Emit metadata, and invalidate any metadata that was set before\n              this.updateMetadata(returnMetadata);\n\n              // Invalidate any preflight metadata\n              if (this.preflightMetadata) {\n                this.preflightMetadata\n                  .then(metadataIn => metadataIn.state.invalidate())\n                  .catch(() => {\n                    // Ignore errors\n                  });\n              }\n\n              // Determine next urls, which will eventually become a next-next source.\n              this.getSourceLinks(returnMetadata)\n                .then((nextUrls: ILink[]) => Promise.all(nextUrls))\n                .then(async(nextUrls: ILink[]) => {\n                  // Append all next URLs to our queue\n                  const linkQueue = await this.getLinkQueue();\n                  for (const nextUrl of nextUrls) {\n                    linkQueue.push(nextUrl, startSource.link);\n                  }\n\n                  receivedMetadata = true;\n                  if (receivedEndEvent) {\n                    this.iteratorsPendingTermination--;\n                  }\n\n                  this.startIteratorsForNextUrls(startSource.handledDatasets, true);\n                }).catch(error => this.destroy(error));\n\n              return returnMetadata;\n            })).catch(error => {\n            this.destroy(error);\n            return <MetadataBindings>{};\n          });\n      });\n    } catch (syncError: unknown) {\n      this.destroy(<Error> syncError);\n    }\n  }\n\n  protected updateMetadata(metadataNew: MetadataBindings): void {\n    const metadataToInvalidate = this.getProperty<MetadataBindings>('metadata');\n    this.setProperty('metadata', metadataNew);\n    metadataToInvalidate?.state.invalidate();\n  }\n\n  protected isRunning(): boolean {\n    return !this.done;\n  }\n\n  /**\n   * Check if a next URL is in the queue.\n   * If yes, start a new iterator.\n   * If no, close this iterator.\n   * @param handledDatasets\n   * @param canClose\n   */\n  protected startIteratorsForNextUrls(handledDatasets: Record<string, boolean>, canClose: boolean): void {\n    this.getLinkQueue()\n      .then(linkQueue => {\n        // Create as many new iterators as possible\n        while (this.canStartNewIterator() && this.isRunning()) {\n          const nextLink = linkQueue.pop();\n          if (nextLink) {\n            this.iteratorsPendingCreation++;\n            this.sourceStateGetter(nextLink, handledDatasets)\n              .then(nextSourceState => {\n                this.iteratorsPendingCreation--;\n                this.startIterator(nextSourceState);\n              })\n              .catch(error => this.emit('error', error));\n          } else {\n            break;\n          }\n        }\n\n        // Close, only if no other iterators are still running\n        if (canClose && this.isCloseable(linkQueue, true)) {\n          this.close();\n        }\n      })\n      .catch(error => this.destroy(error));\n  }\n\n  protected isCloseable(linkQueue: ILinkQueue, requireQueueEmpty: boolean): boolean {\n    return linkQueue.isEmpty() && !this.areIteratorsRunning();\n  }\n}\n\n/**\n * The current state of a source.\n * This is needed for following links within a source.\n */\nexport interface ISourceState {\n  /**\n   * The link to this source.\n   */\n  link: ILink;\n  /**\n   * A source.\n   */\n  source: IQuerySource;\n  /**\n   * The source's initial metadata.\n   */\n  metadata: MetadataBindings;\n  /**\n   * All dataset identifiers that have been passed for this source.\n   */\n  handledDatasets: Record<string, boolean>;\n}\n\nexport type SourceStateGetter = (link: ILink, handledDatasets: Record<string, boolean>) => Promise<ISourceState>;\n"]}

package/lib/MediatedLinkedRdfSourcesAsyncRdfIterator.d.ts

@@ -23,8 +23,9 @@ export declare class MediatedLinkedRdfSourcesAsyncRdfIterator extends LinkedRdfS
     constructor(cacheSize: number, operation: Algebra.Operation, queryBindingsOptions: IQueryBindingsOptions | undefined, context: IActionContext, forceSourceType: string | undefined, firstUrl: string, maxIterators: number, sourceStateGetter: SourceStateGetter, aggregatedStore: IAggregatedStore | undefined, mediatorMetadataAccumulate: MediatorRdfMetadataAccumulate, mediatorRdfResolveHypermediaLinks: MediatorRdfResolveHypermediaLinks, mediatorRdfResolveHypermediaLinksQueue: MediatorRdfResolveHypermediaLinksQueue);
     close(): void;
     destroy(cause?: Error): void;
-    protected isCloseable(linkQueue: ILinkQueue): boolean;
+    protected isCloseable(linkQueue: ILinkQueue, requireQueueEmpty: boolean): boolean;
     protected canStartNewIterator(): boolean;
+    protected canStartNewIteratorConsiderReadable(): boolean;
     protected isRunning(): boolean;
     getLinkQueue(): Promise<ILinkQueue>;
     protected getSourceLinks(metadata: Record<string, any>): Promise<ILink[]>;

package/lib/MediatedLinkedRdfSourcesAsyncRdfIterator.js

@@ -37,8 +37,11 @@ class MediatedLinkedRdfSourcesAsyncRdfIterator extends LinkedRdfSourcesAsyncRdfI
         }
         this.getLinkQueue()
             .then(linkQueue => {
-            if (this.isCloseable(linkQueue)) {
-                this.aggregatedStore?.end();
+            if (this.isCloseable(linkQueue, false)) {
+                // Wait a tick before ending the aggregatedStore, to ensure that pending match() calls to it have started.
+                if (this.aggregatedStore) {
+                    setTimeout(() => this.aggregatedStore.end());
+                }
                 super.close();
             }
             else {
@@ -54,8 +57,11 @@ class MediatedLinkedRdfSourcesAsyncRdfIterator extends LinkedRdfSourcesAsyncRdfI
         }
         this.getLinkQueue()
             .then(linkQueue => {
-            if (cause || this.isCloseable(linkQueue)) {
-                this.aggregatedStore?.end();
+            if (cause || this.isCloseable(linkQueue, false)) {
+                // Wait a tick before ending the aggregatedStore, to ensure that pending match() calls to it have started.
+                if (this.aggregatedStore) {
+                    setTimeout(() => this.aggregatedStore.end());
+                }
                 super.destroy(cause);
             }
             else {
@@ -64,8 +70,9 @@ class MediatedLinkedRdfSourcesAsyncRdfIterator extends LinkedRdfSourcesAsyncRdfI
         })
             .catch(error => super.destroy(error));
     }
-    isCloseable(linkQueue) {
-        return (this.wasForcefullyClosed || linkQueue.isEmpty()) && !this.areIteratorsRunning();
+    isCloseable(linkQueue, requireQueueEmpty) {
+        return (requireQueueEmpty ? linkQueue.isEmpty() : this.wasForcefullyClosed || linkQueue.isEmpty()) &&
+            !this.areIteratorsRunning();
     }
     canStartNewIterator() {
         // Also allow sub-iterators to be started if the aggregated store has at least one running iterator.
@@ -73,6 +80,9 @@ class MediatedLinkedRdfSourcesAsyncRdfIterator extends LinkedRdfSourcesAsyncRdfI
         return !this.wasForcefullyClosed &&
             (this.aggregatedStore && this.aggregatedStore.hasRunningIterators()) || super.canStartNewIterator();
     }
+    canStartNewIteratorConsiderReadable() {
+        return !this.aggregatedStore;
+    }
     isRunning() {
         // Same as above
         return (this.aggregatedStore && this.aggregatedStore.hasRunningIterators()) || !this.done;

package/lib/MediatedLinkedRdfSourcesAsyncRdfIterator.js.map

@@ -1 +1 @@
-{"version":3,"file":"MediatedLinkedRdfSourcesAsyncRdfIterator.js","sourceRoot":"","sources":["MediatedLinkedRdfSourcesAsyncRdfIterator.ts"],"names":[],"mappings":";;;AAKA,+DAA+D;AAG/D,uDAA+C;AAE/C,qDAA0C;AAE1C,yFAAsF;AAEtF,MAAM,EAAE,GAAG,IAAI,8BAAW,EAAgB,CAAC;AAC3C,MAAM,EAAE,GAAG,IAAI,yBAAO,EAAE,CAAC;AAEzB;;;;;GAKG;AACH,MAAa,wCAAyC,SAAQ,mEAAgC;IAU5F,YACE,SAAiB,EACjB,SAA4B,EAC5B,oBAAuD,EACvD,OAAuB,EACvB,eAAmC,EACnC,QAAgB,EAChB,YAAoB,EACpB,iBAAoC,EACpC,eAA6C,EAC7C,0BAAyD,EACzD,iCAAoE,EACpE,sCAA8E;QAE9E,KAAK,CACH,SAAS,EACT,SAAS,EACT,oBAAoB,EACpB,OAAO,EACP,QAAQ,EACR,YAAY,EACZ,iBAAiB;QACjB,2GAA2G;QAC3G,6CAA6C;QAC7C,eAAe,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,MAAM,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,SAAS,CAC1E,CAAC;QA3BI,wBAAmB,GAAG,KAAK,CAAC;QA4BlC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,0BAA0B,GAAG,0BAA0B,CAAC;QAC7D,IAAI,CAAC,iCAAiC,GAAG,iCAAiC,CAAC;QAC3E,IAAI,CAAC,sCAAsC,GAAG,sCAAsC,CAAC;QACrF,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,CAAC;QACxC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACzC,CAAC;IAED,iGAAiG;IACjG,uEAAuE;IACvE,+FAA+F;IAExF,KAAK;QACV,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE;YACzB,KAAK,CAAC,KAAK,EAAE,CAAC;YACd,OAAO;SACR;QAED,IAAI,CAAC,YAAY,EAAE;aAChB,IAAI,CAAC,SAAS,CAAC,EAAE;YAChB,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,EAAE;gBAC/B,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,CAAC;gBAC5B,KAAK,CAAC,KAAK,EAAE,CAAC;aACf;iBAAM;gBACL,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;aACjC;QACH,CAAC,CAAC;aACD,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1C,CAAC;IAEM,OAAO,CAAC,KAAa;QAC1B,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE;YACzB,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACrB,OAAO;SACR;QAED,IAAI,CAAC,YAAY,EAAE;aAChB,IAAI,CAAC,SAAS,CAAC,EAAE;YAChB,IAAI,KAAK,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,EAAE;gBACxC,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,CAAC;gBAC5B,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;aACtB;iBAAM;gBACL,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;aACjC;QACH,CAAC,CAAC;aACD,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1C,CAAC;IAES,WAAW,CAAC,SAAqB;QACzC,OAAO,CAAC,IAAI,CAAC,mBAAmB,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC;IAC1F,CAAC;IAEkB,mBAAmB;QACpC,oGAAoG;QACpG,mHAAmH;QACnH,OAAO,CAAC,IAAI,CAAC,mBAAmB;YAC9B,CAAC,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe,CAAC,mBAAmB,EAAE,CAAC,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC;IACxG,CAAC;IAEkB,SAAS;QAC1B,gBAAgB;QAChB,OAAO,CAAC,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe,CAAC,mBAAmB,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;IAC5F,CAAC;IAEM,YAAY;QACjB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE;YACnB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,sCAAsC;iBACzD,OAAO,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC;iBAC3D,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;SACrC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAES,KAAK,CAAC,cAAc,CAAC,QAA6B;QAC1D,IAAI;YACF,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,iCAAiC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;YAE5G,8CAA8C;YAC9C,OAAO,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;gBACzB,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;oBAC9B,OAAO,KAAK,CAAC;iBACd;gBACD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;gBAClC,OAAO,IAAI,CAAC;YACd,CAAC,CAAC,CAAC;SACJ;QAAC,MAAM;YACN,uGAAuG;YACvG,OAAO,EAAE,CAAC;SACX;IACH,CAAC;IAES,aAAa,CAAC,WAAyB;QAC/C,IAAI,IAAI,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,gBAAgB,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;YAC5F,6GAA6G;YAC7G,2EAA2E;YAC3E,IAAI,CAAC,eAAe,EAAE,gBAAgB,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACjE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,aAAa,CAC7C,EAAE,CAAC,aAAa,CACd,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAChB,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAChB,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAChB,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CACjB,EACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,oCAAkB,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAC7D,CAAC,SAAS,CAAC;gBACV,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CACtB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAE,EAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAE,EAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAE,EAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAClB;gBACD,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,MAAM,CAAc,MAAM,CAAC;iBAC7C,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACd,KAAK,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;YACnC,CAAC,CAAC,CAAC;SACN;aAAM;YACL,KAAK,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;SAClC;IACH,CAAC;IAEM,KAAK,CAAC,kBAAkB,CAC7B,mBAAqC,EACrC,iBAAmC;QAEnC,OAA0B,CAAC,MAAM,IAAI,CAAC,0BAA0B,CAAC,OAAO,CAAC;YACvE,IAAI,EAAE,QAAQ;YACd,mBAAmB;YACnB,iBAAiB;YACjB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC,CAAC,QAAQ,CAAC;IACf,CAAC;IAES,cAAc,CAAC,WAA6B;QACpD,KAAK,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;QAClC,IAAI,CAAC,eAAe,EAAE,eAAe,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;IAC3D,CAAC;CACF;AA9KD,4FA8KC","sourcesContent":["import type { MediatorRdfMetadataAccumulate } from '@comunica/bus-rdf-metadata-accumulate';\nimport type { ILink,\n  MediatorRdfResolveHypermediaLinks } from '@comunica/bus-rdf-resolve-hypermedia-links';\nimport type { ILinkQueue,\n  MediatorRdfResolveHypermediaLinksQueue } from '@comunica/bus-rdf-resolve-hypermedia-links-queue';\nimport { KeysQueryOperation } from '@comunica/context-entries';\nimport type { IActionContext, IAggregatedStore, IQueryBindingsOptions, MetadataBindings } from '@comunica/types';\nimport type * as RDF from '@rdfjs/types';\nimport { DataFactory } from 'rdf-data-factory';\nimport type { Algebra } from 'sparqlalgebrajs';\nimport { Factory } from 'sparqlalgebrajs';\nimport type { SourceStateGetter, ISourceState } from './LinkedRdfSourcesAsyncRdfIterator';\nimport { LinkedRdfSourcesAsyncRdfIterator } from './LinkedRdfSourcesAsyncRdfIterator';\n\nconst DF = new DataFactory<RDF.BaseQuad>();\nconst AF = new Factory();\n\n/**\n * An quad iterator that can iterate over consecutive RDF sources\n * that are determined using the rdf-resolve-hypermedia-links bus.\n *\n * @see LinkedRdfSourcesAsyncRdfIterator\n */\nexport class MediatedLinkedRdfSourcesAsyncRdfIterator extends LinkedRdfSourcesAsyncRdfIterator {\n  private readonly mediatorMetadataAccumulate: MediatorRdfMetadataAccumulate;\n  private readonly mediatorRdfResolveHypermediaLinks: MediatorRdfResolveHypermediaLinks;\n  private readonly mediatorRdfResolveHypermediaLinksQueue: MediatorRdfResolveHypermediaLinksQueue;\n  private readonly forceSourceType?: string;\n  private readonly handledUrls: Record<string, boolean>;\n  private readonly aggregatedStore: IAggregatedStore | undefined;\n  private linkQueue: Promise<ILinkQueue> | undefined;\n  private wasForcefullyClosed = false;\n\n  public constructor(\n    cacheSize: number,\n    operation: Algebra.Operation,\n    queryBindingsOptions: IQueryBindingsOptions | undefined,\n    context: IActionContext,\n    forceSourceType: string | undefined,\n    firstUrl: string,\n    maxIterators: number,\n    sourceStateGetter: SourceStateGetter,\n    aggregatedStore: IAggregatedStore | undefined,\n    mediatorMetadataAccumulate: MediatorRdfMetadataAccumulate,\n    mediatorRdfResolveHypermediaLinks: MediatorRdfResolveHypermediaLinks,\n    mediatorRdfResolveHypermediaLinksQueue: MediatorRdfResolveHypermediaLinksQueue,\n  ) {\n    super(\n      cacheSize,\n      operation,\n      queryBindingsOptions,\n      context,\n      firstUrl,\n      maxIterators,\n      sourceStateGetter,\n      // Buffersize must be infinite for an aggregated store because it must keep filling until there are no more\n      // derived iterators in the aggregated store.\n      aggregatedStore ? { maxBufferSize: Number.POSITIVE_INFINITY } : undefined,\n    );\n    this.forceSourceType = forceSourceType;\n    this.mediatorMetadataAccumulate = mediatorMetadataAccumulate;\n    this.mediatorRdfResolveHypermediaLinks = mediatorRdfResolveHypermediaLinks;\n    this.mediatorRdfResolveHypermediaLinksQueue = mediatorRdfResolveHypermediaLinksQueue;\n    this.handledUrls = { [firstUrl]: true };\n    this.aggregatedStore = aggregatedStore;\n  }\n\n  // Mark the aggregated store as ended once we trigger the closing or destroying of this iterator.\n  // We don't override _end, because that would mean that we have to wait\n  // until the buffer of this iterator must be fully consumed, which will not always be the case.\n\n  public close(): void {\n    if (!this.aggregatedStore) {\n      super.close();\n      return;\n    }\n\n    this.getLinkQueue()\n      .then(linkQueue => {\n        if (this.isCloseable(linkQueue)) {\n          this.aggregatedStore?.end();\n          super.close();\n        } else {\n          this.wasForcefullyClosed = true;\n        }\n      })\n      .catch(error => super.destroy(error));\n  }\n\n  public destroy(cause?: Error): void {\n    if (!this.aggregatedStore) {\n      super.destroy(cause);\n      return;\n    }\n\n    this.getLinkQueue()\n      .then(linkQueue => {\n        if (cause || this.isCloseable(linkQueue)) {\n          this.aggregatedStore?.end();\n          super.destroy(cause);\n        } else {\n          this.wasForcefullyClosed = true;\n        }\n      })\n      .catch(error => super.destroy(error));\n  }\n\n  protected isCloseable(linkQueue: ILinkQueue): boolean {\n    return (this.wasForcefullyClosed || linkQueue.isEmpty()) && !this.areIteratorsRunning();\n  }\n\n  protected override canStartNewIterator(): boolean {\n    // Also allow sub-iterators to be started if the aggregated store has at least one running iterator.\n    // We need this because there are cases where these running iterators will be consumed before this linked iterator.\n    return !this.wasForcefullyClosed &&\n      (this.aggregatedStore && this.aggregatedStore.hasRunningIterators()) || super.canStartNewIterator();\n  }\n\n  protected override isRunning(): boolean {\n    // Same as above\n    return (this.aggregatedStore && this.aggregatedStore.hasRunningIterators()) || !this.done;\n  }\n\n  public getLinkQueue(): Promise<ILinkQueue> {\n    if (!this.linkQueue) {\n      this.linkQueue = this.mediatorRdfResolveHypermediaLinksQueue\n        .mediate({ firstUrl: this.firstUrl, context: this.context })\n        .then(result => result.linkQueue);\n    }\n    return this.linkQueue;\n  }\n\n  protected async getSourceLinks(metadata: Record<string, any>): Promise<ILink[]> {\n    try {\n      const { links } = await this.mediatorRdfResolveHypermediaLinks.mediate({ context: this.context, metadata });\n\n      // Filter URLs to avoid cyclic next-page loops\n      return links.filter(link => {\n        if (this.handledUrls[link.url]) {\n          return false;\n        }\n        this.handledUrls[link.url] = true;\n        return true;\n      });\n    } catch {\n      // No next URLs may be available, for example when we've reached the end of a Hydra next-page sequence.\n      return [];\n    }\n  }\n\n  protected startIterator(startSource: ISourceState): void {\n    if (this.aggregatedStore && !this.aggregatedStore.containedSources.has(startSource.link.url)) {\n      // A source that has been cached due to earlier query executions may not be part of the aggregated store yet.\n      // In that case, we add all quads from that source to the aggregated store.\n      this.aggregatedStore?.containedSources.add(startSource.link.url);\n      const stream = startSource.source.queryBindings(\n        AF.createPattern(\n          DF.variable('s'),\n          DF.variable('p'),\n          DF.variable('o'),\n          DF.variable('g'),\n        ),\n        this.context.set(KeysQueryOperation.unionDefaultGraph, true),\n      ).transform({\n        map: bindings => DF.quad(\n          bindings.get('s')!,\n          bindings.get('p')!,\n          bindings.get('o')!,\n          bindings.get('g'),\n        ),\n        autoStart: false,\n      });\n      this.aggregatedStore.import(<RDF.Stream> stream)\n        .on('end', () => {\n          super.startIterator(startSource);\n        });\n    } else {\n      super.startIterator(startSource);\n    }\n  }\n\n  public async accumulateMetadata(\n    accumulatedMetadata: MetadataBindings,\n    appendingMetadata: MetadataBindings,\n  ): Promise<MetadataBindings> {\n    return <MetadataBindings> (await this.mediatorMetadataAccumulate.mediate({\n      mode: 'append',\n      accumulatedMetadata,\n      appendingMetadata,\n      context: this.context,\n    })).metadata;\n  }\n\n  protected updateMetadata(metadataNew: MetadataBindings): void {\n    super.updateMetadata(metadataNew);\n    this.aggregatedStore?.setBaseMetadata(metadataNew, true);\n  }\n}\n"]}
+{"version":3,"file":"MediatedLinkedRdfSourcesAsyncRdfIterator.js","sourceRoot":"","sources":["MediatedLinkedRdfSourcesAsyncRdfIterator.ts"],"names":[],"mappings":";;;AAKA,+DAA+D;AAG/D,uDAA+C;AAE/C,qDAA0C;AAE1C,yFAAsF;AAEtF,MAAM,EAAE,GAAG,IAAI,8BAAW,EAAgB,CAAC;AAC3C,MAAM,EAAE,GAAG,IAAI,yBAAO,EAAE,CAAC;AAEzB;;;;;GAKG;AACH,MAAa,wCAAyC,SAAQ,mEAAgC;IAU5F,YACE,SAAiB,EACjB,SAA4B,EAC5B,oBAAuD,EACvD,OAAuB,EACvB,eAAmC,EACnC,QAAgB,EAChB,YAAoB,EACpB,iBAAoC,EACpC,eAA6C,EAC7C,0BAAyD,EACzD,iCAAoE,EACpE,sCAA8E;QAE9E,KAAK,CACH,SAAS,EACT,SAAS,EACT,oBAAoB,EACpB,OAAO,EACP,QAAQ,EACR,YAAY,EACZ,iBAAiB;QACjB,2GAA2G;QAC3G,6CAA6C;QAC7C,eAAe,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,MAAM,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,SAAS,CAC1E,CAAC;QA3BI,wBAAmB,GAAG,KAAK,CAAC;QA4BlC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,0BAA0B,GAAG,0BAA0B,CAAC;QAC7D,IAAI,CAAC,iCAAiC,GAAG,iCAAiC,CAAC;QAC3E,IAAI,CAAC,sCAAsC,GAAG,sCAAsC,CAAC;QACrF,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,CAAC;QACxC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACzC,CAAC;IAED,iGAAiG;IACjG,uEAAuE;IACvE,+FAA+F;IAExF,KAAK;QACV,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE;YACzB,KAAK,CAAC,KAAK,EAAE,CAAC;YACd,OAAO;SACR;QAED,IAAI,CAAC,YAAY,EAAE;aAChB,IAAI,CAAC,SAAS,CAAC,EAAE;YAChB,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE;gBACtC,0GAA0G;gBAC1G,IAAI,IAAI,CAAC,eAAe,EAAE;oBACxB,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,eAAgB,CAAC,GAAG,EAAE,CAAC,CAAC;iBAC/C;gBACD,KAAK,CAAC,KAAK,EAAE,CAAC;aACf;iBAAM;gBACL,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;aACjC;QACH,CAAC,CAAC;aACD,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1C,CAAC;IAEM,OAAO,CAAC,KAAa;QAC1B,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE;YACzB,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACrB,OAAO;SACR;QAED,IAAI,CAAC,YAAY,EAAE;aAChB,IAAI,CAAC,SAAS,CAAC,EAAE;YAChB,IAAI,KAAK,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE;gBAC/C,0GAA0G;gBAC1G,IAAI,IAAI,CAAC,eAAe,EAAE;oBACxB,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,eAAgB,CAAC,GAAG,EAAE,CAAC,CAAC;iBAC/C;gBACD,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;aACtB;iBAAM;gBACL,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;aACjC;QACH,CAAC,CAAC;aACD,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1C,CAAC;IAES,WAAW,CAAC,SAAqB,EAAE,iBAA0B;QACrE,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,mBAAmB,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;YAChG,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC;IAChC,CAAC;IAEkB,mBAAmB;QACpC,oGAAoG;QACpG,mHAAmH;QACnH,OAAO,CAAC,IAAI,CAAC,mBAAmB;YAC9B,CAAC,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe,CAAC,mBAAmB,EAAE,CAAC,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC;IACxG,CAAC;IAES,mCAAmC;QAC3C,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC;IAC/B,CAAC;IAEkB,SAAS;QAC1B,gBAAgB;QAChB,OAAO,CAAC,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe,CAAC,mBAAmB,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;IAC5F,CAAC;IAEM,YAAY;QACjB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE;YACnB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,sCAAsC;iBACzD,OAAO,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC;iBAC3D,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;SACrC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAES,KAAK,CAAC,cAAc,CAAC,QAA6B;QAC1D,IAAI;YACF,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,iCAAiC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;YAE5G,8CAA8C;YAC9C,OAAO,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;gBACzB,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;oBAC9B,OAAO,KAAK,CAAC;iBACd;gBACD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;gBAClC,OAAO,IAAI,CAAC;YACd,CAAC,CAAC,CAAC;SACJ;QAAC,MAAM;YACN,uGAAuG;YACvG,OAAO,EAAE,CAAC;SACX;IACH,CAAC;IAES,aAAa,CAAC,WAAyB;QAC/C,IAAI,IAAI,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,gBAAgB,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;YAC5F,6GAA6G;YAC7G,2EAA2E;YAC3E,IAAI,CAAC,eAAe,EAAE,gBAAgB,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACjE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,aAAa,CAC7C,EAAE,CAAC,aAAa,CACd,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAChB,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAChB,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAChB,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CACjB,EACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,oCAAkB,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAC7D,CAAC,SAAS,CAAC;gBACV,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CACtB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAE,EAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAE,EAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAE,EAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAClB;gBACD,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,MAAM,CAAc,MAAM,CAAC;iBAC7C,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACd,KAAK,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;YACnC,CAAC,CAAC,CAAC;SACN;aAAM;YACL,KAAK,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;SAClC;IACH,CAAC;IAEM,KAAK,CAAC,kBAAkB,CAC7B,mBAAqC,EACrC,iBAAmC;QAEnC,OAA0B,CAAC,MAAM,IAAI,CAAC,0BAA0B,CAAC,OAAO,CAAC;YACvE,IAAI,EAAE,QAAQ;YACd,mBAAmB;YACnB,iBAAiB;YACjB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC,CAAC,QAAQ,CAAC;IACf,CAAC;IAES,cAAc,CAAC,WAA6B;QACpD,KAAK,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;QAClC,IAAI,CAAC,eAAe,EAAE,eAAe,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;IAC3D,CAAC;CACF;AAzLD,4FAyLC","sourcesContent":["import type { MediatorRdfMetadataAccumulate } from '@comunica/bus-rdf-metadata-accumulate';\nimport type { ILink,\n  MediatorRdfResolveHypermediaLinks } from '@comunica/bus-rdf-resolve-hypermedia-links';\nimport type { ILinkQueue,\n  MediatorRdfResolveHypermediaLinksQueue } from '@comunica/bus-rdf-resolve-hypermedia-links-queue';\nimport { KeysQueryOperation } from '@comunica/context-entries';\nimport type { IActionContext, IAggregatedStore, IQueryBindingsOptions, MetadataBindings } from '@comunica/types';\nimport type * as RDF from '@rdfjs/types';\nimport { DataFactory } from 'rdf-data-factory';\nimport type { Algebra } from 'sparqlalgebrajs';\nimport { Factory } from 'sparqlalgebrajs';\nimport type { SourceStateGetter, ISourceState } from './LinkedRdfSourcesAsyncRdfIterator';\nimport { LinkedRdfSourcesAsyncRdfIterator } from './LinkedRdfSourcesAsyncRdfIterator';\n\nconst DF = new DataFactory<RDF.BaseQuad>();\nconst AF = new Factory();\n\n/**\n * An quad iterator that can iterate over consecutive RDF sources\n * that are determined using the rdf-resolve-hypermedia-links bus.\n *\n * @see LinkedRdfSourcesAsyncRdfIterator\n */\nexport class MediatedLinkedRdfSourcesAsyncRdfIterator extends LinkedRdfSourcesAsyncRdfIterator {\n  private readonly mediatorMetadataAccumulate: MediatorRdfMetadataAccumulate;\n  private readonly mediatorRdfResolveHypermediaLinks: MediatorRdfResolveHypermediaLinks;\n  private readonly mediatorRdfResolveHypermediaLinksQueue: MediatorRdfResolveHypermediaLinksQueue;\n  private readonly forceSourceType?: string;\n  private readonly handledUrls: Record<string, boolean>;\n  private readonly aggregatedStore: IAggregatedStore | undefined;\n  private linkQueue: Promise<ILinkQueue> | undefined;\n  private wasForcefullyClosed = false;\n\n  public constructor(\n    cacheSize: number,\n    operation: Algebra.Operation,\n    queryBindingsOptions: IQueryBindingsOptions | undefined,\n    context: IActionContext,\n    forceSourceType: string | undefined,\n    firstUrl: string,\n    maxIterators: number,\n    sourceStateGetter: SourceStateGetter,\n    aggregatedStore: IAggregatedStore | undefined,\n    mediatorMetadataAccumulate: MediatorRdfMetadataAccumulate,\n    mediatorRdfResolveHypermediaLinks: MediatorRdfResolveHypermediaLinks,\n    mediatorRdfResolveHypermediaLinksQueue: MediatorRdfResolveHypermediaLinksQueue,\n  ) {\n    super(\n      cacheSize,\n      operation,\n      queryBindingsOptions,\n      context,\n      firstUrl,\n      maxIterators,\n      sourceStateGetter,\n      // Buffersize must be infinite for an aggregated store because it must keep filling until there are no more\n      // derived iterators in the aggregated store.\n      aggregatedStore ? { maxBufferSize: Number.POSITIVE_INFINITY } : undefined,\n    );\n    this.forceSourceType = forceSourceType;\n    this.mediatorMetadataAccumulate = mediatorMetadataAccumulate;\n    this.mediatorRdfResolveHypermediaLinks = mediatorRdfResolveHypermediaLinks;\n    this.mediatorRdfResolveHypermediaLinksQueue = mediatorRdfResolveHypermediaLinksQueue;\n    this.handledUrls = { [firstUrl]: true };\n    this.aggregatedStore = aggregatedStore;\n  }\n\n  // Mark the aggregated store as ended once we trigger the closing or destroying of this iterator.\n  // We don't override _end, because that would mean that we have to wait\n  // until the buffer of this iterator must be fully consumed, which will not always be the case.\n\n  public close(): void {\n    if (!this.aggregatedStore) {\n      super.close();\n      return;\n    }\n\n    this.getLinkQueue()\n      .then(linkQueue => {\n        if (this.isCloseable(linkQueue, false)) {\n          // Wait a tick before ending the aggregatedStore, to ensure that pending match() calls to it have started.\n          if (this.aggregatedStore) {\n            setTimeout(() => this.aggregatedStore!.end());\n          }\n          super.close();\n        } else {\n          this.wasForcefullyClosed = true;\n        }\n      })\n      .catch(error => super.destroy(error));\n  }\n\n  public destroy(cause?: Error): void {\n    if (!this.aggregatedStore) {\n      super.destroy(cause);\n      return;\n    }\n\n    this.getLinkQueue()\n      .then(linkQueue => {\n        if (cause || this.isCloseable(linkQueue, false)) {\n          // Wait a tick before ending the aggregatedStore, to ensure that pending match() calls to it have started.\n          if (this.aggregatedStore) {\n            setTimeout(() => this.aggregatedStore!.end());\n          }\n          super.destroy(cause);\n        } else {\n          this.wasForcefullyClosed = true;\n        }\n      })\n      .catch(error => super.destroy(error));\n  }\n\n  protected isCloseable(linkQueue: ILinkQueue, requireQueueEmpty: boolean): boolean {\n    return (requireQueueEmpty ? linkQueue.isEmpty() : this.wasForcefullyClosed || linkQueue.isEmpty()) &&\n      !this.areIteratorsRunning();\n  }\n\n  protected override canStartNewIterator(): boolean {\n    // Also allow sub-iterators to be started if the aggregated store has at least one running iterator.\n    // We need this because there are cases where these running iterators will be consumed before this linked iterator.\n    return !this.wasForcefullyClosed &&\n      (this.aggregatedStore && this.aggregatedStore.hasRunningIterators()) || super.canStartNewIterator();\n  }\n\n  protected canStartNewIteratorConsiderReadable(): boolean {\n    return !this.aggregatedStore;\n  }\n\n  protected override isRunning(): boolean {\n    // Same as above\n    return (this.aggregatedStore && this.aggregatedStore.hasRunningIterators()) || !this.done;\n  }\n\n  public getLinkQueue(): Promise<ILinkQueue> {\n    if (!this.linkQueue) {\n      this.linkQueue = this.mediatorRdfResolveHypermediaLinksQueue\n        .mediate({ firstUrl: this.firstUrl, context: this.context })\n        .then(result => result.linkQueue);\n    }\n    return this.linkQueue;\n  }\n\n  protected async getSourceLinks(metadata: Record<string, any>): Promise<ILink[]> {\n    try {\n      const { links } = await this.mediatorRdfResolveHypermediaLinks.mediate({ context: this.context, metadata });\n\n      // Filter URLs to avoid cyclic next-page loops\n      return links.filter(link => {\n        if (this.handledUrls[link.url]) {\n          return false;\n        }\n        this.handledUrls[link.url] = true;\n        return true;\n      });\n    } catch {\n      // No next URLs may be available, for example when we've reached the end of a Hydra next-page sequence.\n      return [];\n    }\n  }\n\n  protected startIterator(startSource: ISourceState): void {\n    if (this.aggregatedStore && !this.aggregatedStore.containedSources.has(startSource.link.url)) {\n      // A source that has been cached due to earlier query executions may not be part of the aggregated store yet.\n      // In that case, we add all quads from that source to the aggregated store.\n      this.aggregatedStore?.containedSources.add(startSource.link.url);\n      const stream = startSource.source.queryBindings(\n        AF.createPattern(\n          DF.variable('s'),\n          DF.variable('p'),\n          DF.variable('o'),\n          DF.variable('g'),\n        ),\n        this.context.set(KeysQueryOperation.unionDefaultGraph, true),\n      ).transform({\n        map: bindings => DF.quad(\n          bindings.get('s')!,\n          bindings.get('p')!,\n          bindings.get('o')!,\n          bindings.get('g'),\n        ),\n        autoStart: false,\n      });\n      this.aggregatedStore.import(<RDF.Stream> stream)\n        .on('end', () => {\n          super.startIterator(startSource);\n        });\n    } else {\n      super.startIterator(startSource);\n    }\n  }\n\n  public async accumulateMetadata(\n    accumulatedMetadata: MetadataBindings,\n    appendingMetadata: MetadataBindings,\n  ): Promise<MetadataBindings> {\n    return <MetadataBindings> (await this.mediatorMetadataAccumulate.mediate({\n      mode: 'append',\n      accumulatedMetadata,\n      appendingMetadata,\n      context: this.context,\n    })).metadata;\n  }\n\n  protected updateMetadata(metadataNew: MetadataBindings): void {\n    super.updateMetadata(metadataNew);\n    this.aggregatedStore?.setBaseMetadata(metadataNew, true);\n  }\n}\n"]}

package/lib/QuerySourceHypermedia.js

@@ -39,6 +39,11 @@ class QuerySourceHypermedia {
         const it = new MediatedLinkedRdfSourcesAsyncRdfIterator_1.MediatedLinkedRdfSourcesAsyncRdfIterator(this.cacheSize, operation, options, context, this.forceSourceType, this.firstUrl, this.maxIterators, (link, handledDatasets) => this.getSourceCached(link, handledDatasets, context, aggregatedStore), aggregatedStore, this.mediators.mediatorMetadataAccumulate, this.mediators.mediatorRdfResolveHypermediaLinks, this.mediators.mediatorRdfResolveHypermediaLinksQueue);
         if (aggregatedStore) {
             aggregatedStore.started = true;
+            // Kickstart this iterator when derived iterators are created from the aggregatedStore,
+            // otherwise the traversal process will not start if this iterator is not the first one to be consumed.
+            const listener = () => it.kickstart();
+            aggregatedStore.addIteratorCreatedListener(listener);
+            it.on('end', () => aggregatedStore.removeIteratorCreatedListener(listener));
         }
         return it;
     }

package/lib/QuerySourceHypermedia.js.map

@@ -1 +1 @@
-{"version":3,"file":"QuerySourceHypermedia.js","sourceRoot":"","sources":["QuerySourceHypermedia.ts"],"names":[],"mappings":";;;AAAA,mGAA+E;AAS/E,+DAAoE;AAUpE,iDAAkD;AAClD,yCAAqC;AACrC,qDAA2C;AAG3C,yGAAsG;AACtG,qEAAkE;AAElE,MAAa,qBAAqB;IAiBhC,YACE,SAAiB,EACjB,QAAgB,EAChB,eAAmC,EACnC,YAAoB,EACpB,cAAuB,EACvB,SAAwB,EACxB,UAA4C,EAC5C,eAAgC;QAEhC,IAAI,CAAC,cAAc,GAAG,QAAQ,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,YAAY,GAAG,IAAI,oBAAQ,CAAgC,EAAE,GAAG,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;IAC3F,CAAC;IAEM,KAAK,CAAC,gBAAgB,CAAC,OAAuB;QACnD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;QAChH,OAAO,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;IACjD,CAAC;IAEM,aAAa,CAClB,SAA4B,EAC5B,OAAuB,EACvB,OAA+B;QAE/B,gEAAgE;QAChE,MAAM,eAAe,GAAiC,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACtF,IAAI,eAAe,IAAI,SAAS,CAAC,IAAI,KAAK,SAAS,IAAI,eAAe,CAAC,OAAO,EAAE;YAC9E,OAAO,IAAI,oDAAgB,CAAC,eAAe,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC,aAAa,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;SACtG;QAED,6CAA6C;QAC7C,IAAI,IAAI,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,EAAE;YAChC,IAAI,CAAC,eAAe,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,eAAe,CAAC;iBACvE,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;SACtC;QAED,MAAM,EAAE,GAA6C,IAAI,mFAAwC,CAC/F,IAAI,CAAC,SAAS,EACd,SAAS,EACT,OAAO,EACP,OAAO,EACP,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,YAAY,EACjB,CAAC,IAAI,EAAE,eAAe,EAAE,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,eAAe,EAAE,OAAO,EAAE,eAAe,CAAC,EAChG,eAAe,EACf,IAAI,CAAC,SAAS,CAAC,0BAA0B,EACzC,IAAI,CAAC,SAAS,CAAC,iCAAiC,EAChD,IAAI,CAAC,SAAS,CAAC,sCAAsC,CACtD,CAAC;QACF,IAAI,eAAe,EAAE;YACnB,eAAe,CAAC,OAAO,GAAG,IAAI,CAAC;SAChC;QAED,OAAO,EAAE,CAAC;IACZ,CAAC;IAEM,UAAU,CAAC,SAA4B,EAAE,OAAuB;QACrE,OAAO,IAAI,iCAAiB,CAAC,KAAK,IAAG,EAAE;YACrC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;YAChH,OAAO,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,YAAY,CAAC,SAAsB,EAAE,OAAuB;QACvE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;QAChH,OAAO,MAAM,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC9D,CAAC;IAEM,KAAK,CAAC,SAAS,CAAC,SAAyB,EAAE,OAAuB;QACvE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;QAChH,OAAO,MAAM,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC3D,CAAC;IAED;;;;;;OAMG;IACI,KAAK,CAAC,SAAS,CACpB,IAAW,EACX,eAAwC,EACxC,OAAuB,EACvB,eAA6C;QAE7C,oCAAoC;QACpC,IAAI,IAAI,CAAC,OAAO,EAAE;YAChB,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;SACvC;QAED,mDAAmD;QACnD,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;QACnB,IAAI,KAAiB,CAAC;QACtB,IAAI,QAA6B,CAAC;QAClC,IAAI;YACF,MAAM,oBAAoB,GAA+B,MAAM,IAAI,CAAC,SAAS,CAAC,sBAAsB;iBACjG,OAAO,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;YAC7B,GAAG,GAAG,oBAAoB,CAAC,GAAG,CAAC;YAE/B,yBAAyB;YACzB,MAAM,iBAAiB,GAA4B,MAAM,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,OAAO,CAC9F,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,oBAAoB,CAAC,IAAI,EAAE,OAAO,EAAE,oBAAoB,CAAC,QAAQ,EAAE,OAAO,EAAE,CACpG,CAAC;YAEF,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBACtC,qCAAqC;gBACrC,wDAAwD;gBACxD,kDAAkD;gBAClD,iEAAiE;YACnE,CAAC,CAAC,CAAC;YAEH,QAAQ,GAAG,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC,OAAO,CAAC;gBAC/D,OAAO;gBACP,GAAG;gBACH,2DAA2D;gBAC3D,QAAQ,EAAE,iBAAiB,CAAC,QAAQ;gBACpC,OAAO,EAAE,oBAAoB,CAAC,OAAO;gBACrC,WAAW,EAAE,oBAAoB,CAAC,WAAW;aAC9C,CAAC,CAAC,CAAC,QAAQ,CAAC;YACb,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC;YAE/B,uCAAuC;YACvC,IAAI,IAAI,CAAC,SAAS,EAAE;gBAClB,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;aACrC;SACF;QAAC,OAAO,KAAc,EAAE;YACvB,+FAA+F;YAC/F,kHAAkH;YAClH,KAAK,GAAG,IAAI,0BAAQ,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,GAAG,GAAG,EAAE;gBAChB,UAAU,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC7C,OAAO,IAAI,CAAC;YACd,CAAC,CAAC;YACF,CAAC,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,0BAA0B,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC;YAE1G,kGAAkG;YAClG,wEAAwE;YACxE,IAAI,CAAC,UAAU,CAAC,2BAA2B,GAAG,YAAqB,KAAM,CAAC,OAAO,EAAE,CAAC,CAAC;SACtF;QAED,+CAA+C;QAC/C,eAAe,EAAE,eAAe,CAAoB,QAAQ,EAAE,KAAK,CAAC,CAAC;QACrE,eAAe,EAAE,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChD,eAAe,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QAE/B,uBAAuB;QACvB,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,qCAAqC,CAAC,OAAO,CAAC;YAC7F,OAAO;YACP,eAAe,EAAE,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS;YAC9E,eAAe;YACf,QAAQ;YACR,KAAK;YACL,GAAG;SACJ,CAAC,CAAC;QAEH,IAAI,OAAO,EAAE;YACX,8BAA8B;YAC9B,uFAAuF;YACvF,+CAA+C;YAC/C,eAAe,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC;SACjC;QAED,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAqB,QAAQ,EAAE,eAAe,EAAE,CAAC;IAClF,CAAC;IAED;;;;;;;OAOG;IACO,eAAe,CACvB,IAAW,EACX,eAAwC,EACxC,OAAuB,EACvB,eAA6C;QAE7C,IAAI,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC7C,IAAI,MAAM,EAAE;YACV,OAAO,MAAM,CAAC;SACf;QACD,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,eAAe,EAAE,OAAO,EAAE,eAAe,CAAC,CAAC;QACzE,IAAI,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,QAAQ,IAAI,eAAe,KAAK,SAAS,EAAE;YAC/D,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;SACzC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEM,iBAAiB,CAAC,OAAuB;QAC9C,IAAI,eAA6C,CAAC;QAClD,IAAI,IAAI,CAAC,cAAc,EAAE;YACvB,MAAM,gBAAgB,GAA8C,OAAO;iBACxE,GAAG,CAAC,yCAAuB,CAAC,iCAAiC,CAAC,CAAC;YAClE,IAAI,gBAAgB,EAAE;gBACpB,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACtD,IAAI,CAAC,eAAe,EAAE;oBACpB,eAAe,GAAG,IAAI,+CAAsB,CAC1C,SAAS,EACT,KAAK,EAAC,mBAAmB,EAAE,iBAAiB,EAAE,EAAE,CAC9C,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,0BAA0B,CAAC,OAAO,CAAC;wBACvD,IAAI,EAAE,QAAQ;wBACd,mBAAmB;wBACnB,iBAAiB;wBACjB,OAAO;qBACR,CAAC,CAAC,CAAC,QAAQ,CACf,CAAC;oBACF,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;iBACtD;gBACD,OAAO,eAAe,CAAC;aACxB;SACF;IACH,CAAC;IAEM,QAAQ;QACb,OAAO,yBAAyB,IAAI,CAAC,QAAQ,GAAG,CAAC;IACnD,CAAC;CACF;AArPD,sDAqPC","sourcesContent":["import { QuerySourceRdfJs } from '@comunica/actor-query-source-identify-rdfjs';\nimport type { BindingsFactory } from '@comunica/bindings-factory';\nimport type { IActorDereferenceRdfOutput, MediatorDereferenceRdf } from '@comunica/bus-dereference-rdf';\nimport type { MediatorQuerySourceIdentifyHypermedia } from '@comunica/bus-query-source-identify-hypermedia';\nimport type { IActorRdfMetadataOutput, MediatorRdfMetadata } from '@comunica/bus-rdf-metadata';\nimport type { MediatorRdfMetadataAccumulate } from '@comunica/bus-rdf-metadata-accumulate';\nimport type { MediatorRdfMetadataExtract } from '@comunica/bus-rdf-metadata-extract';\nimport type { ILink, MediatorRdfResolveHypermediaLinks } from '@comunica/bus-rdf-resolve-hypermedia-links';\nimport type { MediatorRdfResolveHypermediaLinksQueue } from '@comunica/bus-rdf-resolve-hypermedia-links-queue';\nimport { KeysQuerySourceIdentify } from '@comunica/context-entries';\nimport type {\n  BindingsStream,\n  FragmentSelectorShape,\n  IActionContext,\n  IAggregatedStore, IQueryBindingsOptions,\n  IQuerySource, MetadataBindings,\n} from '@comunica/types';\nimport type * as RDF from '@rdfjs/types';\nimport type { AsyncIterator } from 'asynciterator';\nimport { TransformIterator } from 'asynciterator';\nimport { LRUCache } from 'lru-cache';\nimport { Readable } from 'readable-stream';\nimport type { Algebra } from 'sparqlalgebrajs';\nimport type { ISourceState } from './LinkedRdfSourcesAsyncRdfIterator';\nimport { MediatedLinkedRdfSourcesAsyncRdfIterator } from './MediatedLinkedRdfSourcesAsyncRdfIterator';\nimport { StreamingStoreMetadata } from './StreamingStoreMetadata';\n\nexport class QuerySourceHypermedia implements IQuerySource {\n  public readonly referenceValue: string;\n  public readonly firstUrl: string;\n  public readonly forceSourceType?: string;\n  public readonly aggregateStore: boolean;\n  public readonly mediators: IMediatorArgs;\n  public readonly logWarning: (warningMessage: string) => void;\n  public readonly bindingsFactory: BindingsFactory;\n\n  /**\n   * A cache for source URLs to source states.\n   */\n  public sourcesState: LRUCache<string, Promise<ISourceState>>;\n\n  private readonly cacheSize: number;\n  private readonly maxIterators: number;\n\n  public constructor(\n    cacheSize: number,\n    firstUrl: string,\n    forceSourceType: string | undefined,\n    maxIterators: number,\n    aggregateStore: boolean,\n    mediators: IMediatorArgs,\n    logWarning: (warningMessage: string) => void,\n    bindingsFactory: BindingsFactory,\n  ) {\n    this.referenceValue = firstUrl;\n    this.cacheSize = cacheSize;\n    this.firstUrl = firstUrl;\n    this.forceSourceType = forceSourceType;\n    this.maxIterators = maxIterators;\n    this.mediators = mediators;\n    this.aggregateStore = aggregateStore;\n    this.logWarning = logWarning;\n    this.bindingsFactory = bindingsFactory;\n    this.sourcesState = new LRUCache<string, Promise<ISourceState>>({ max: this.cacheSize });\n  }\n\n  public async getSelectorShape(context: IActionContext): Promise<FragmentSelectorShape> {\n    const source = await this.getSourceCached({ url: this.firstUrl }, {}, context, this.getAggregateStore(context));\n    return source.source.getSelectorShape(context);\n  }\n\n  public queryBindings(\n    operation: Algebra.Operation,\n    context: IActionContext,\n    options?: IQueryBindingsOptions,\n  ): BindingsStream {\n    // Optimized match with aggregated store if enabled and started.\n    const aggregatedStore: IAggregatedStore | undefined = this.getAggregateStore(context);\n    if (aggregatedStore && operation.type === 'pattern' && aggregatedStore.started) {\n      return new QuerySourceRdfJs(aggregatedStore, this.bindingsFactory).queryBindings(operation, context);\n    }\n\n    // Initialize the sources state on first call\n    if (this.sourcesState.size === 0) {\n      this.getSourceCached({ url: this.firstUrl }, {}, context, aggregatedStore)\n        .catch(error => it.destroy(error));\n    }\n\n    const it: MediatedLinkedRdfSourcesAsyncRdfIterator = new MediatedLinkedRdfSourcesAsyncRdfIterator(\n      this.cacheSize,\n      operation,\n      options,\n      context,\n      this.forceSourceType,\n      this.firstUrl,\n      this.maxIterators,\n      (link, handledDatasets) => this.getSourceCached(link, handledDatasets, context, aggregatedStore),\n      aggregatedStore,\n      this.mediators.mediatorMetadataAccumulate,\n      this.mediators.mediatorRdfResolveHypermediaLinks,\n      this.mediators.mediatorRdfResolveHypermediaLinksQueue,\n    );\n    if (aggregatedStore) {\n      aggregatedStore.started = true;\n    }\n\n    return it;\n  }\n\n  public queryQuads(operation: Algebra.Construct, context: IActionContext): AsyncIterator<RDF.Quad> {\n    return new TransformIterator(async() => {\n      const source = await this.getSourceCached({ url: this.firstUrl }, {}, context, this.getAggregateStore(context));\n      return source.source.queryQuads(operation, context);\n    });\n  }\n\n  public async queryBoolean(operation: Algebra.Ask, context: IActionContext): Promise<boolean> {\n    const source = await this.getSourceCached({ url: this.firstUrl }, {}, context, this.getAggregateStore(context));\n    return await source.source.queryBoolean(operation, context);\n  }\n\n  public async queryVoid(operation: Algebra.Update, context: IActionContext): Promise<void> {\n    const source = await this.getSourceCached({ url: this.firstUrl }, {}, context, this.getAggregateStore(context));\n    return await source.source.queryVoid(operation, context);\n  }\n\n  /**\n   * Resolve a source for the given URL.\n   * @param link A source link.\n   * @param handledDatasets A hash of dataset identifiers that have already been handled.\n   * @param context The action context.\n   * @param aggregatedStore An optional aggregated store.\n   */\n  public async getSource(\n    link: ILink,\n    handledDatasets: Record<string, boolean>,\n    context: IActionContext,\n    aggregatedStore: IAggregatedStore | undefined,\n  ): Promise<ISourceState> {\n    // Include context entries from link\n    if (link.context) {\n      context = context.merge(link.context);\n    }\n\n    // Get the RDF representation of the given document\n    let url = link.url;\n    let quads: RDF.Stream;\n    let metadata: Record<string, any>;\n    try {\n      const dereferenceRdfOutput: IActorDereferenceRdfOutput = await this.mediators.mediatorDereferenceRdf\n        .mediate({ context, url });\n      url = dereferenceRdfOutput.url;\n\n      // Determine the metadata\n      const rdfMetadataOutput: IActorRdfMetadataOutput = await this.mediators.mediatorMetadata.mediate(\n        { context, url, quads: dereferenceRdfOutput.data, triples: dereferenceRdfOutput.metadata?.triples },\n      );\n\n      rdfMetadataOutput.data.on('error', () => {\n        // Silence errors in the data stream,\n        // as they will be emitted again in the metadata stream,\n        // and will result in a promise rejection anyways.\n        // If we don't do this, we end up with an unhandled error message\n      });\n\n      metadata = (await this.mediators.mediatorMetadataExtract.mediate({\n        context,\n        url,\n        // The problem appears to be conflicting metadata keys here\n        metadata: rdfMetadataOutput.metadata,\n        headers: dereferenceRdfOutput.headers,\n        requestTime: dereferenceRdfOutput.requestTime,\n      })).metadata;\n      quads = rdfMetadataOutput.data;\n\n      // Optionally filter the resulting data\n      if (link.transform) {\n        quads = await link.transform(quads);\n      }\n    } catch (error: unknown) {\n      // Make sure that dereference errors are only emitted once an actor really needs the read quads\n      // This for example allows SPARQL endpoints that error on service description fetching to still be source-forcible\n      quads = new Readable();\n      quads.read = () => {\n        setTimeout(() => quads.emit('error', error));\n        return null;\n      };\n      ({ metadata } = await this.mediators.mediatorMetadataAccumulate.mediate({ context, mode: 'initialize' }));\n\n      // Log as warning, because the quads above may not always be consumed (e.g. for SPARQL endpoints),\n      // so the user would not be notified of something going wrong otherwise.\n      this.logWarning(`Metadata extraction for ${url} failed: ${(<Error> error).message}`);\n    }\n\n    // Aggregate all discovered quads into a store.\n    aggregatedStore?.setBaseMetadata(<MetadataBindings> metadata, false);\n    aggregatedStore?.containedSources.add(link.url);\n    aggregatedStore?.import(quads);\n\n    // Determine the source\n    const { source, dataset } = await this.mediators.mediatorQuerySourceIdentifyHypermedia.mediate({\n      context,\n      forceSourceType: link.url === this.firstUrl ? this.forceSourceType : undefined,\n      handledDatasets,\n      metadata,\n      quads,\n      url,\n    });\n\n    if (dataset) {\n      // Mark the dataset as applied\n      // This is needed to make sure that things like QPF search forms are only applied once,\n      // and next page links are followed after that.\n      handledDatasets[dataset] = true;\n    }\n\n    return { link, source, metadata: <MetadataBindings> metadata, handledDatasets };\n  }\n\n  /**\n   * Resolve a source for the given URL.\n   * This will first try to retrieve the source from cache.\n   * @param link A source ILink.\n   * @param handledDatasets A hash of dataset identifiers that have already been handled.\n   * @param context The action context.\n   * @param aggregatedStore An optional aggregated store.\n   */\n  protected getSourceCached(\n    link: ILink,\n    handledDatasets: Record<string, boolean>,\n    context: IActionContext,\n    aggregatedStore: IAggregatedStore | undefined,\n  ): Promise<ISourceState> {\n    let source = this.sourcesState.get(link.url);\n    if (source) {\n      return source;\n    }\n    source = this.getSource(link, handledDatasets, context, aggregatedStore);\n    if (link.url === this.firstUrl || aggregatedStore === undefined) {\n      this.sourcesState.set(link.url, source);\n    }\n    return source;\n  }\n\n  public getAggregateStore(context: IActionContext): IAggregatedStore | undefined {\n    let aggregatedStore: IAggregatedStore | undefined;\n    if (this.aggregateStore) {\n      const aggregatedStores: Map<string, IAggregatedStore> | undefined = context\n        .get(KeysQuerySourceIdentify.hypermediaSourcesAggregatedStores);\n      if (aggregatedStores) {\n        aggregatedStore = aggregatedStores.get(this.firstUrl);\n        if (!aggregatedStore) {\n          aggregatedStore = new StreamingStoreMetadata(\n            undefined,\n            async(accumulatedMetadata, appendingMetadata) => <MetadataBindings>\n              (await this.mediators.mediatorMetadataAccumulate.mediate({\n                mode: 'append',\n                accumulatedMetadata,\n                appendingMetadata,\n                context,\n              })).metadata,\n          );\n          aggregatedStores.set(this.firstUrl, aggregatedStore);\n        }\n        return aggregatedStore;\n      }\n    }\n  }\n\n  public toString(): string {\n    return `QuerySourceHypermedia(${this.firstUrl})`;\n  }\n}\n\nexport interface IMediatorArgs {\n  mediatorDereferenceRdf: MediatorDereferenceRdf;\n  mediatorMetadata: MediatorRdfMetadata;\n  mediatorMetadataExtract: MediatorRdfMetadataExtract;\n  mediatorMetadataAccumulate: MediatorRdfMetadataAccumulate;\n  mediatorQuerySourceIdentifyHypermedia: MediatorQuerySourceIdentifyHypermedia;\n  mediatorRdfResolveHypermediaLinks: MediatorRdfResolveHypermediaLinks;\n  mediatorRdfResolveHypermediaLinksQueue: MediatorRdfResolveHypermediaLinksQueue;\n}\n"]}
+{"version":3,"file":"QuerySourceHypermedia.js","sourceRoot":"","sources":["QuerySourceHypermedia.ts"],"names":[],"mappings":";;;AAAA,mGAA+E;AAS/E,+DAAoE;AAUpE,iDAAkD;AAClD,yCAAqC;AACrC,qDAA2C;AAG3C,yGAAsG;AACtG,qEAAkE;AAElE,MAAa,qBAAqB;IAiBhC,YACE,SAAiB,EACjB,QAAgB,EAChB,eAAmC,EACnC,YAAoB,EACpB,cAAuB,EACvB,SAAwB,EACxB,UAA4C,EAC5C,eAAgC;QAEhC,IAAI,CAAC,cAAc,GAAG,QAAQ,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,YAAY,GAAG,IAAI,oBAAQ,CAAgC,EAAE,GAAG,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;IAC3F,CAAC;IAEM,KAAK,CAAC,gBAAgB,CAAC,OAAuB;QACnD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;QAChH,OAAO,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;IACjD,CAAC;IAEM,aAAa,CAClB,SAA4B,EAC5B,OAAuB,EACvB,OAA+B;QAE/B,gEAAgE;QAChE,MAAM,eAAe,GAAiC,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACtF,IAAI,eAAe,IAAI,SAAS,CAAC,IAAI,KAAK,SAAS,IAAI,eAAe,CAAC,OAAO,EAAE;YAC9E,OAAO,IAAI,oDAAgB,CAAC,eAAe,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC,aAAa,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;SACtG;QAED,6CAA6C;QAC7C,IAAI,IAAI,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,EAAE;YAChC,IAAI,CAAC,eAAe,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,eAAe,CAAC;iBACvE,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;SACtC;QAED,MAAM,EAAE,GAA6C,IAAI,mFAAwC,CAC/F,IAAI,CAAC,SAAS,EACd,SAAS,EACT,OAAO,EACP,OAAO,EACP,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,YAAY,EACjB,CAAC,IAAI,EAAE,eAAe,EAAE,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,eAAe,EAAE,OAAO,EAAE,eAAe,CAAC,EAChG,eAAe,EACf,IAAI,CAAC,SAAS,CAAC,0BAA0B,EACzC,IAAI,CAAC,SAAS,CAAC,iCAAiC,EAChD,IAAI,CAAC,SAAS,CAAC,sCAAsC,CACtD,CAAC;QACF,IAAI,eAAe,EAAE;YACnB,eAAe,CAAC,OAAO,GAAG,IAAI,CAAC;YAE/B,uFAAuF;YACvF,uGAAuG;YACvG,MAAM,QAAQ,GAAG,GAAS,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC;YAC5C,eAAe,CAAC,0BAA0B,CAAC,QAAQ,CAAC,CAAC;YACrD,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,eAAe,CAAC,6BAA6B,CAAC,QAAQ,CAAC,CAAC,CAAC;SAC7E;QAED,OAAO,EAAE,CAAC;IACZ,CAAC;IAEM,UAAU,CAAC,SAA4B,EAAE,OAAuB;QACrE,OAAO,IAAI,iCAAiB,CAAC,KAAK,IAAG,EAAE;YACrC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;YAChH,OAAO,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,YAAY,CAAC,SAAsB,EAAE,OAAuB;QACvE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;QAChH,OAAO,MAAM,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC9D,CAAC;IAEM,KAAK,CAAC,SAAS,CAAC,SAAyB,EAAE,OAAuB;QACvE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;QAChH,OAAO,MAAM,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC3D,CAAC;IAED;;;;;;OAMG;IACI,KAAK,CAAC,SAAS,CACpB,IAAW,EACX,eAAwC,EACxC,OAAuB,EACvB,eAA6C;QAE7C,oCAAoC;QACpC,IAAI,IAAI,CAAC,OAAO,EAAE;YAChB,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;SACvC;QAED,mDAAmD;QACnD,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;QACnB,IAAI,KAAiB,CAAC;QACtB,IAAI,QAA6B,CAAC;QAClC,IAAI;YACF,MAAM,oBAAoB,GAA+B,MAAM,IAAI,CAAC,SAAS,CAAC,sBAAsB;iBACjG,OAAO,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;YAC7B,GAAG,GAAG,oBAAoB,CAAC,GAAG,CAAC;YAE/B,yBAAyB;YACzB,MAAM,iBAAiB,GAA4B,MAAM,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,OAAO,CAC9F,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,oBAAoB,CAAC,IAAI,EAAE,OAAO,EAAE,oBAAoB,CAAC,QAAQ,EAAE,OAAO,EAAE,CACpG,CAAC;YAEF,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBACtC,qCAAqC;gBACrC,wDAAwD;gBACxD,kDAAkD;gBAClD,iEAAiE;YACnE,CAAC,CAAC,CAAC;YAEH,QAAQ,GAAG,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC,OAAO,CAAC;gBAC/D,OAAO;gBACP,GAAG;gBACH,2DAA2D;gBAC3D,QAAQ,EAAE,iBAAiB,CAAC,QAAQ;gBACpC,OAAO,EAAE,oBAAoB,CAAC,OAAO;gBACrC,WAAW,EAAE,oBAAoB,CAAC,WAAW;aAC9C,CAAC,CAAC,CAAC,QAAQ,CAAC;YACb,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC;YAE/B,uCAAuC;YACvC,IAAI,IAAI,CAAC,SAAS,EAAE;gBAClB,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;aACrC;SACF;QAAC,OAAO,KAAc,EAAE;YACvB,+FAA+F;YAC/F,kHAAkH;YAClH,KAAK,GAAG,IAAI,0BAAQ,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,GAAG,GAAG,EAAE;gBAChB,UAAU,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC7C,OAAO,IAAI,CAAC;YACd,CAAC,CAAC;YACF,CAAC,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,0BAA0B,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC;YAE1G,kGAAkG;YAClG,wEAAwE;YACxE,IAAI,CAAC,UAAU,CAAC,2BAA2B,GAAG,YAAqB,KAAM,CAAC,OAAO,EAAE,CAAC,CAAC;SACtF;QAED,+CAA+C;QAC/C,eAAe,EAAE,eAAe,CAAoB,QAAQ,EAAE,KAAK,CAAC,CAAC;QACrE,eAAe,EAAE,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChD,eAAe,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QAE/B,uBAAuB;QACvB,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,qCAAqC,CAAC,OAAO,CAAC;YAC7F,OAAO;YACP,eAAe,EAAE,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS;YAC9E,eAAe;YACf,QAAQ;YACR,KAAK;YACL,GAAG;SACJ,CAAC,CAAC;QAEH,IAAI,OAAO,EAAE;YACX,8BAA8B;YAC9B,uFAAuF;YACvF,+CAA+C;YAC/C,eAAe,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC;SACjC;QAED,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAqB,QAAQ,EAAE,eAAe,EAAE,CAAC;IAClF,CAAC;IAED;;;;;;;OAOG;IACO,eAAe,CACvB,IAAW,EACX,eAAwC,EACxC,OAAuB,EACvB,eAA6C;QAE7C,IAAI,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC7C,IAAI,MAAM,EAAE;YACV,OAAO,MAAM,CAAC;SACf;QACD,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,eAAe,EAAE,OAAO,EAAE,eAAe,CAAC,CAAC;QACzE,IAAI,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,QAAQ,IAAI,eAAe,KAAK,SAAS,EAAE;YAC/D,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;SACzC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEM,iBAAiB,CAAC,OAAuB;QAC9C,IAAI,eAA6C,CAAC;QAClD,IAAI,IAAI,CAAC,cAAc,EAAE;YACvB,MAAM,gBAAgB,GAA8C,OAAO;iBACxE,GAAG,CAAC,yCAAuB,CAAC,iCAAiC,CAAC,CAAC;YAClE,IAAI,gBAAgB,EAAE;gBACpB,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACtD,IAAI,CAAC,eAAe,EAAE;oBACpB,eAAe,GAAG,IAAI,+CAAsB,CAC1C,SAAS,EACT,KAAK,EAAC,mBAAmB,EAAE,iBAAiB,EAAE,EAAE,CAC9C,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,0BAA0B,CAAC,OAAO,CAAC;wBACvD,IAAI,EAAE,QAAQ;wBACd,mBAAmB;wBACnB,iBAAiB;wBACjB,OAAO;qBACR,CAAC,CAAC,CAAC,QAAQ,CACf,CAAC;oBACF,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;iBACtD;gBACD,OAAO,eAAe,CAAC;aACxB;SACF;IACH,CAAC;IAEM,QAAQ;QACb,OAAO,yBAAyB,IAAI,CAAC,QAAQ,GAAG,CAAC;IACnD,CAAC;CACF;AA3PD,sDA2PC","sourcesContent":["import { QuerySourceRdfJs } from '@comunica/actor-query-source-identify-rdfjs';\nimport type { BindingsFactory } from '@comunica/bindings-factory';\nimport type { IActorDereferenceRdfOutput, MediatorDereferenceRdf } from '@comunica/bus-dereference-rdf';\nimport type { MediatorQuerySourceIdentifyHypermedia } from '@comunica/bus-query-source-identify-hypermedia';\nimport type { IActorRdfMetadataOutput, MediatorRdfMetadata } from '@comunica/bus-rdf-metadata';\nimport type { MediatorRdfMetadataAccumulate } from '@comunica/bus-rdf-metadata-accumulate';\nimport type { MediatorRdfMetadataExtract } from '@comunica/bus-rdf-metadata-extract';\nimport type { ILink, MediatorRdfResolveHypermediaLinks } from '@comunica/bus-rdf-resolve-hypermedia-links';\nimport type { MediatorRdfResolveHypermediaLinksQueue } from '@comunica/bus-rdf-resolve-hypermedia-links-queue';\nimport { KeysQuerySourceIdentify } from '@comunica/context-entries';\nimport type {\n  BindingsStream,\n  FragmentSelectorShape,\n  IActionContext,\n  IAggregatedStore, IQueryBindingsOptions,\n  IQuerySource, MetadataBindings,\n} from '@comunica/types';\nimport type * as RDF from '@rdfjs/types';\nimport type { AsyncIterator } from 'asynciterator';\nimport { TransformIterator } from 'asynciterator';\nimport { LRUCache } from 'lru-cache';\nimport { Readable } from 'readable-stream';\nimport type { Algebra } from 'sparqlalgebrajs';\nimport type { ISourceState } from './LinkedRdfSourcesAsyncRdfIterator';\nimport { MediatedLinkedRdfSourcesAsyncRdfIterator } from './MediatedLinkedRdfSourcesAsyncRdfIterator';\nimport { StreamingStoreMetadata } from './StreamingStoreMetadata';\n\nexport class QuerySourceHypermedia implements IQuerySource {\n  public readonly referenceValue: string;\n  public readonly firstUrl: string;\n  public readonly forceSourceType?: string;\n  public readonly aggregateStore: boolean;\n  public readonly mediators: IMediatorArgs;\n  public readonly logWarning: (warningMessage: string) => void;\n  public readonly bindingsFactory: BindingsFactory;\n\n  /**\n   * A cache for source URLs to source states.\n   */\n  public sourcesState: LRUCache<string, Promise<ISourceState>>;\n\n  private readonly cacheSize: number;\n  private readonly maxIterators: number;\n\n  public constructor(\n    cacheSize: number,\n    firstUrl: string,\n    forceSourceType: string | undefined,\n    maxIterators: number,\n    aggregateStore: boolean,\n    mediators: IMediatorArgs,\n    logWarning: (warningMessage: string) => void,\n    bindingsFactory: BindingsFactory,\n  ) {\n    this.referenceValue = firstUrl;\n    this.cacheSize = cacheSize;\n    this.firstUrl = firstUrl;\n    this.forceSourceType = forceSourceType;\n    this.maxIterators = maxIterators;\n    this.mediators = mediators;\n    this.aggregateStore = aggregateStore;\n    this.logWarning = logWarning;\n    this.bindingsFactory = bindingsFactory;\n    this.sourcesState = new LRUCache<string, Promise<ISourceState>>({ max: this.cacheSize });\n  }\n\n  public async getSelectorShape(context: IActionContext): Promise<FragmentSelectorShape> {\n    const source = await this.getSourceCached({ url: this.firstUrl }, {}, context, this.getAggregateStore(context));\n    return source.source.getSelectorShape(context);\n  }\n\n  public queryBindings(\n    operation: Algebra.Operation,\n    context: IActionContext,\n    options?: IQueryBindingsOptions,\n  ): BindingsStream {\n    // Optimized match with aggregated store if enabled and started.\n    const aggregatedStore: IAggregatedStore | undefined = this.getAggregateStore(context);\n    if (aggregatedStore && operation.type === 'pattern' && aggregatedStore.started) {\n      return new QuerySourceRdfJs(aggregatedStore, this.bindingsFactory).queryBindings(operation, context);\n    }\n\n    // Initialize the sources state on first call\n    if (this.sourcesState.size === 0) {\n      this.getSourceCached({ url: this.firstUrl }, {}, context, aggregatedStore)\n        .catch(error => it.destroy(error));\n    }\n\n    const it: MediatedLinkedRdfSourcesAsyncRdfIterator = new MediatedLinkedRdfSourcesAsyncRdfIterator(\n      this.cacheSize,\n      operation,\n      options,\n      context,\n      this.forceSourceType,\n      this.firstUrl,\n      this.maxIterators,\n      (link, handledDatasets) => this.getSourceCached(link, handledDatasets, context, aggregatedStore),\n      aggregatedStore,\n      this.mediators.mediatorMetadataAccumulate,\n      this.mediators.mediatorRdfResolveHypermediaLinks,\n      this.mediators.mediatorRdfResolveHypermediaLinksQueue,\n    );\n    if (aggregatedStore) {\n      aggregatedStore.started = true;\n\n      // Kickstart this iterator when derived iterators are created from the aggregatedStore,\n      // otherwise the traversal process will not start if this iterator is not the first one to be consumed.\n      const listener = (): void => it.kickstart();\n      aggregatedStore.addIteratorCreatedListener(listener);\n      it.on('end', () => aggregatedStore.removeIteratorCreatedListener(listener));\n    }\n\n    return it;\n  }\n\n  public queryQuads(operation: Algebra.Construct, context: IActionContext): AsyncIterator<RDF.Quad> {\n    return new TransformIterator(async() => {\n      const source = await this.getSourceCached({ url: this.firstUrl }, {}, context, this.getAggregateStore(context));\n      return source.source.queryQuads(operation, context);\n    });\n  }\n\n  public async queryBoolean(operation: Algebra.Ask, context: IActionContext): Promise<boolean> {\n    const source = await this.getSourceCached({ url: this.firstUrl }, {}, context, this.getAggregateStore(context));\n    return await source.source.queryBoolean(operation, context);\n  }\n\n  public async queryVoid(operation: Algebra.Update, context: IActionContext): Promise<void> {\n    const source = await this.getSourceCached({ url: this.firstUrl }, {}, context, this.getAggregateStore(context));\n    return await source.source.queryVoid(operation, context);\n  }\n\n  /**\n   * Resolve a source for the given URL.\n   * @param link A source link.\n   * @param handledDatasets A hash of dataset identifiers that have already been handled.\n   * @param context The action context.\n   * @param aggregatedStore An optional aggregated store.\n   */\n  public async getSource(\n    link: ILink,\n    handledDatasets: Record<string, boolean>,\n    context: IActionContext,\n    aggregatedStore: IAggregatedStore | undefined,\n  ): Promise<ISourceState> {\n    // Include context entries from link\n    if (link.context) {\n      context = context.merge(link.context);\n    }\n\n    // Get the RDF representation of the given document\n    let url = link.url;\n    let quads: RDF.Stream;\n    let metadata: Record<string, any>;\n    try {\n      const dereferenceRdfOutput: IActorDereferenceRdfOutput = await this.mediators.mediatorDereferenceRdf\n        .mediate({ context, url });\n      url = dereferenceRdfOutput.url;\n\n      // Determine the metadata\n      const rdfMetadataOutput: IActorRdfMetadataOutput = await this.mediators.mediatorMetadata.mediate(\n        { context, url, quads: dereferenceRdfOutput.data, triples: dereferenceRdfOutput.metadata?.triples },\n      );\n\n      rdfMetadataOutput.data.on('error', () => {\n        // Silence errors in the data stream,\n        // as they will be emitted again in the metadata stream,\n        // and will result in a promise rejection anyways.\n        // If we don't do this, we end up with an unhandled error message\n      });\n\n      metadata = (await this.mediators.mediatorMetadataExtract.mediate({\n        context,\n        url,\n        // The problem appears to be conflicting metadata keys here\n        metadata: rdfMetadataOutput.metadata,\n        headers: dereferenceRdfOutput.headers,\n        requestTime: dereferenceRdfOutput.requestTime,\n      })).metadata;\n      quads = rdfMetadataOutput.data;\n\n      // Optionally filter the resulting data\n      if (link.transform) {\n        quads = await link.transform(quads);\n      }\n    } catch (error: unknown) {\n      // Make sure that dereference errors are only emitted once an actor really needs the read quads\n      // This for example allows SPARQL endpoints that error on service description fetching to still be source-forcible\n      quads = new Readable();\n      quads.read = () => {\n        setTimeout(() => quads.emit('error', error));\n        return null;\n      };\n      ({ metadata } = await this.mediators.mediatorMetadataAccumulate.mediate({ context, mode: 'initialize' }));\n\n      // Log as warning, because the quads above may not always be consumed (e.g. for SPARQL endpoints),\n      // so the user would not be notified of something going wrong otherwise.\n      this.logWarning(`Metadata extraction for ${url} failed: ${(<Error> error).message}`);\n    }\n\n    // Aggregate all discovered quads into a store.\n    aggregatedStore?.setBaseMetadata(<MetadataBindings> metadata, false);\n    aggregatedStore?.containedSources.add(link.url);\n    aggregatedStore?.import(quads);\n\n    // Determine the source\n    const { source, dataset } = await this.mediators.mediatorQuerySourceIdentifyHypermedia.mediate({\n      context,\n      forceSourceType: link.url === this.firstUrl ? this.forceSourceType : undefined,\n      handledDatasets,\n      metadata,\n      quads,\n      url,\n    });\n\n    if (dataset) {\n      // Mark the dataset as applied\n      // This is needed to make sure that things like QPF search forms are only applied once,\n      // and next page links are followed after that.\n      handledDatasets[dataset] = true;\n    }\n\n    return { link, source, metadata: <MetadataBindings> metadata, handledDatasets };\n  }\n\n  /**\n   * Resolve a source for the given URL.\n   * This will first try to retrieve the source from cache.\n   * @param link A source ILink.\n   * @param handledDatasets A hash of dataset identifiers that have already been handled.\n   * @param context The action context.\n   * @param aggregatedStore An optional aggregated store.\n   */\n  protected getSourceCached(\n    link: ILink,\n    handledDatasets: Record<string, boolean>,\n    context: IActionContext,\n    aggregatedStore: IAggregatedStore | undefined,\n  ): Promise<ISourceState> {\n    let source = this.sourcesState.get(link.url);\n    if (source) {\n      return source;\n    }\n    source = this.getSource(link, handledDatasets, context, aggregatedStore);\n    if (link.url === this.firstUrl || aggregatedStore === undefined) {\n      this.sourcesState.set(link.url, source);\n    }\n    return source;\n  }\n\n  public getAggregateStore(context: IActionContext): IAggregatedStore | undefined {\n    let aggregatedStore: IAggregatedStore | undefined;\n    if (this.aggregateStore) {\n      const aggregatedStores: Map<string, IAggregatedStore> | undefined = context\n        .get(KeysQuerySourceIdentify.hypermediaSourcesAggregatedStores);\n      if (aggregatedStores) {\n        aggregatedStore = aggregatedStores.get(this.firstUrl);\n        if (!aggregatedStore) {\n          aggregatedStore = new StreamingStoreMetadata(\n            undefined,\n            async(accumulatedMetadata, appendingMetadata) => <MetadataBindings>\n              (await this.mediators.mediatorMetadataAccumulate.mediate({\n                mode: 'append',\n                accumulatedMetadata,\n                appendingMetadata,\n                context,\n              })).metadata,\n          );\n          aggregatedStores.set(this.firstUrl, aggregatedStore);\n        }\n        return aggregatedStore;\n      }\n    }\n  }\n\n  public toString(): string {\n    return `QuerySourceHypermedia(${this.firstUrl})`;\n  }\n}\n\nexport interface IMediatorArgs {\n  mediatorDereferenceRdf: MediatorDereferenceRdf;\n  mediatorMetadata: MediatorRdfMetadata;\n  mediatorMetadataExtract: MediatorRdfMetadataExtract;\n  mediatorMetadataAccumulate: MediatorRdfMetadataAccumulate;\n  mediatorQuerySourceIdentifyHypermedia: MediatorQuerySourceIdentifyHypermedia;\n  mediatorRdfResolveHypermediaLinks: MediatorRdfResolveHypermediaLinks;\n  mediatorRdfResolveHypermediaLinksQueue: MediatorRdfResolveHypermediaLinksQueue;\n}\n"]}

package/lib/StreamingStoreMetadata.d.ts

@@ -11,6 +11,7 @@ export declare class StreamingStoreMetadata extends StreamingStore implements IA
     started: boolean;
     containedSources: Set<string>;
     readonly runningIterators: Set<AsyncIterator<RDF.Quad>>;
+    protected readonly iteratorCreatedListeners: Set<() => void>;
     protected readonly metadataAccumulator: (accumulatedMetadata: MetadataBindings, appendingMetadata: MetadataBindings) => Promise<MetadataBindings>;
     protected baseMetadata: MetadataBindings;
     constructor(store: RDF.Store | undefined, metadataAccumulator: (accumulatedMetadata: MetadataBindings, appendingMetadata: MetadataBindings) => Promise<MetadataBindings>);
@@ -19,4 +20,6 @@ export declare class StreamingStoreMetadata extends StreamingStore implements IA
     match(subject?: RDF.Term | null, predicate?: RDF.Term | null, object?: RDF.Term | null, graph?: RDF.Term | null): AsyncIterator<RDF.Quad>;
     setBaseMetadata(metadata: MetadataBindings, updateStates: boolean): void;
     protected updateMetadataState(iterator: AsyncIterator<RDF.Quad>, count: number): void;
+    addIteratorCreatedListener(listener: () => void): void;
+    removeIteratorCreatedListener(listener: () => void): void;
 }

package/lib/StreamingStoreMetadata.js

@@ -13,6 +13,7 @@ class StreamingStoreMetadata extends rdf_streaming_store_1.StreamingStore {
         this.started = false;
         this.containedSources = new Set();
         this.runningIterators = new Set();
+        this.iteratorCreatedListeners = new Set();
         this.baseMetadata = {
             state: new metadata_1.MetadataValidationState(),
             cardinality: { type: 'exact', value: 0 },
@@ -59,6 +60,10 @@ class StreamingStoreMetadata extends rdf_streaming_store_1.StreamingStore {
         });
         // Store all running iterators until they close or are destroyed
         this.runningIterators.add(iterator);
+        // Invoke creation listeners
+        for (const listener of this.iteratorCreatedListeners) {
+            listener();
+        }
         return iterator;
     }
     setBaseMetadata(metadata, updateStates) {
@@ -94,6 +99,12 @@ class StreamingStoreMetadata extends rdf_streaming_store_1.StreamingStore {
             // Void errors
         });
     }
+    addIteratorCreatedListener(listener) {
+        this.iteratorCreatedListeners.add(listener);
+    }
+    removeIteratorCreatedListener(listener) {
+        this.iteratorCreatedListeners.delete(listener);
+    }
 }
 exports.StreamingStoreMetadata = StreamingStoreMetadata;
 //# sourceMappingURL=StreamingStoreMetadata.js.map

package/lib/StreamingStoreMetadata.js.map

@@ -1 +1 @@
-{"version":3,"file":"StreamingStoreMetadata.js","sourceRoot":"","sources":["StreamingStoreMetadata.ts"],"names":[],"mappings":";;;AAEA,uEAA0E;AAC1E,iDAA6D;AAI7D,6DAAqD;AAErD;;GAEG;AACH,MAAa,sBAAuB,SAAQ,oCAAc;IAcxD,YACE,KAA4B,EAC5B,mBACyG;QAEzG,KAAK,CAAC,KAAK,CAAC,CAAC;QAlBR,YAAO,GAAG,KAAK,CAAC;QAChB,qBAAgB,GAAG,IAAI,GAAG,EAAU,CAAC;QAC5B,qBAAgB,GAAiC,IAAI,GAAG,EAA2B,CAAC;QAI1F,iBAAY,GAAqB;YACzC,KAAK,EAAE,IAAI,kCAAuB,EAAE;YACpC,WAAW,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE;YACxC,gBAAgB,EAAE,KAAK;YACvB,SAAS,EAAE,EAAE;SACd,CAAC;QAQA,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;IACjD,CAAC;IAEM,MAAM,CAAC,MAAkB;QAC9B,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE;YACf,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;SACtB;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEM,mBAAmB;QACxB,OAAO,IAAI,CAAC,gBAAgB,CAAC,IAAI,GAAG,CAAC,CAAC;IACxC,CAAC;IAEM,KAAK,CACV,OAAyB,EACzB,SAA2B,EAC3B,MAAwB,EACxB,KAAuB;QAEvB,0CAA0C;QAC1C,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;QACjE,MAAM,QAAQ,GAAG,IAAI,+CAAyB,CACtC,SAAS,EAAE;YACf,SAAS,EAAE,KAAK;YAChB,OAAO,EAAE,GAAG,EAAE;gBACZ,yDAAyD;gBACzD,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACzC,CAAC;SACF,CACF,CAAC;QAEF,0DAA0D;QAC1D,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,OAAQ,EAAE,SAAU,EAAE,MAAO,EAAE,KAAM,CAAC,CAAC;QAC9E,MAAM,QAAQ,GAAkB;YAC9B,KAAK,EAAE,IAAI,kCAAuB,EAAE;YACpC,WAAW,EAAE;gBACX,IAAI,EAAE,UAAU;gBAChB,KAAK,EAAE,KAAK;aACb;YACD,gBAAgB,EAAE,KAAK;SACxB,CAAC;QACF,QAAQ,CAAC,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QAC3C,QAAQ,CAAC,WAAW,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QAEzC,yEAAyE;QACzE,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;YACxB,QAAQ,CAAC,WAAW,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,CAAC;YAC3C,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,gEAAgE;QAChE,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEpC,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEM,eAAe,CAAC,QAA0B,EAAE,YAAqB;QACtE,IAAI,CAAC,YAAY,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;QACpC,IAAI,CAAC,YAAY,CAAC,WAAW,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAE5D,IAAI,YAAY,EAAE;YAChB,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,gBAAgB,EAAE;gBAC5C,MAAM,KAAK,GAAW,QAAQ,CAAC,WAAW,CAAC,WAAW,CAAE,CAAC;gBACzD,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;aAC3C;SACF;IACH,CAAC;IAES,mBAAmB,CAAC,QAAiC,EAAE,KAAa;QAC5E,oDAAoD;QACpD,MAAM,WAAW,GAAqB;YACpC,KAAK,EAAE,IAAI,kCAAuB,EAAE;YACpC,WAAW,EAAE;gBACX,IAAI,EAAE,UAAU;gBAChB,KAAK,EAAE,KAAK;aACb;YACD,gBAAgB,EAAE,KAAK;YACvB,SAAS,EAAE,EAAE;SACd,CAAC;QAEF,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,YAAY,EAAE,WAAW,CAAC;aACrD,IAAI,CAAC,mBAAmB,CAAC,EAAE;YAC1B,mBAAmB,CAAC,KAAK,GAAG,IAAI,kCAAuB,EAAE,CAAC;YAE1D,0DAA0D;YAC1D,MAAM,oBAAoB,GAAG,QAAQ,CAAC,WAAW,CAAgB,UAAU,CAAC,CAAC;YAC7E,QAAQ,CAAC,WAAW,CAAC,UAAU,EAAE,mBAAmB,CAAC,CAAC;YACtD,oBAAoB,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC;QAC3C,CAAC,CAAC;aACD,KAAK,CAAC,GAAG,EAAE;YACV,cAAc;QAChB,CAAC,CAAC,CAAC;IACP,CAAC;CACF;AAlHD,wDAkHC","sourcesContent":["// eslint-disable-next-line import/no-nodejs-modules\nimport type { EventEmitter } from 'events';\nimport { ClosableTransformIterator } from '@comunica/bus-query-operation';\nimport { MetadataValidationState } from '@comunica/metadata';\nimport type { MetadataQuads, IAggregatedStore, MetadataBindings } from '@comunica/types';\nimport type * as RDF from '@rdfjs/types';\nimport type { AsyncIterator } from 'asynciterator';\nimport { StreamingStore } from 'rdf-streaming-store';\n\n/**\n * A StreamingStore that returns an AsyncIterator with a valid MetadataQuads property.\n */\nexport class StreamingStoreMetadata extends StreamingStore implements IAggregatedStore {\n  public started = false;\n  public containedSources = new Set<string>();\n  public readonly runningIterators: Set<AsyncIterator<RDF.Quad>> = new Set<AsyncIterator<RDF.Quad>>();\n  protected readonly metadataAccumulator:\n  (accumulatedMetadata: MetadataBindings, appendingMetadata: MetadataBindings) => Promise<MetadataBindings>;\n\n  protected baseMetadata: MetadataBindings = {\n    state: new MetadataValidationState(),\n    cardinality: { type: 'exact', value: 0 },\n    canContainUndefs: false,\n    variables: [],\n  };\n\n  public constructor(\n    store: RDF.Store | undefined,\n    metadataAccumulator:\n    (accumulatedMetadata: MetadataBindings, appendingMetadata: MetadataBindings) => Promise<MetadataBindings>,\n  ) {\n    super(store);\n    this.metadataAccumulator = metadataAccumulator;\n  }\n\n  public import(stream: RDF.Stream): EventEmitter {\n    if (!this.ended) {\n      super.import(stream);\n    }\n    return stream;\n  }\n\n  public hasRunningIterators(): boolean {\n    return this.runningIterators.size > 0;\n  }\n\n  public match(\n    subject?: RDF.Term | null,\n    predicate?: RDF.Term | null,\n    object?: RDF.Term | null,\n    graph?: RDF.Term | null,\n  ): AsyncIterator<RDF.Quad> {\n    // Wrap the raw stream in an AsyncIterator\n    const rawStream = super.match(subject, predicate, object, graph);\n    const iterator = new ClosableTransformIterator<RDF.Quad, RDF.Quad>(\n      <any> rawStream, {\n        autoStart: false,\n        onClose: () => {\n          // Running iterators are deleted once closed or destroyed\n          this.runningIterators.delete(iterator);\n        },\n      },\n    );\n\n    // Expose the metadata property containing the cardinality\n    let count = this.getStore().countQuads(subject!, predicate!, object!, graph!);\n    const metadata: MetadataQuads = {\n      state: new MetadataValidationState(),\n      cardinality: {\n        type: 'estimate',\n        value: count,\n      },\n      canContainUndefs: false,\n    };\n    iterator.setProperty('metadata', metadata);\n    iterator.setProperty('lastCount', count);\n\n    // Every time a new quad is pushed into the iterator, update the metadata\n    rawStream.on('quad', () => {\n      iterator.setProperty('lastCount', ++count);\n      this.updateMetadataState(iterator, count);\n    });\n\n    // Store all running iterators until they close or are destroyed\n    this.runningIterators.add(iterator);\n\n    return iterator;\n  }\n\n  public setBaseMetadata(metadata: MetadataBindings, updateStates: boolean): void {\n    this.baseMetadata = { ...metadata };\n    this.baseMetadata.cardinality = { type: 'exact', value: 0 };\n\n    if (updateStates) {\n      for (const iterator of this.runningIterators) {\n        const count: number = iterator.getProperty('lastCount')!;\n        this.updateMetadataState(iterator, count);\n      }\n    }\n  }\n\n  protected updateMetadataState(iterator: AsyncIterator<RDF.Quad>, count: number): void {\n    // Append the given cardinality to the base metadata\n    const metadataNew: MetadataBindings = {\n      state: new MetadataValidationState(),\n      cardinality: {\n        type: 'estimate',\n        value: count,\n      },\n      canContainUndefs: false,\n      variables: [],\n    };\n\n    this.metadataAccumulator(this.baseMetadata, metadataNew)\n      .then(accumulatedMetadata => {\n        accumulatedMetadata.state = new MetadataValidationState();\n\n        // Set the new metadata, and invalidate the previous state\n        const metadataToInvalidate = iterator.getProperty<MetadataQuads>('metadata');\n        iterator.setProperty('metadata', accumulatedMetadata);\n        metadataToInvalidate?.state.invalidate();\n      })\n      .catch(() => {\n        // Void errors\n      });\n  }\n}\n"]}
+{"version":3,"file":"StreamingStoreMetadata.js","sourceRoot":"","sources":["StreamingStoreMetadata.ts"],"names":[],"mappings":";;;AAEA,uEAA0E;AAC1E,iDAA6D;AAI7D,6DAAqD;AAErD;;GAEG;AACH,MAAa,sBAAuB,SAAQ,oCAAc;IAexD,YACE,KAA4B,EAC5B,mBACyG;QAEzG,KAAK,CAAC,KAAK,CAAC,CAAC;QAnBR,YAAO,GAAG,KAAK,CAAC;QAChB,qBAAgB,GAAG,IAAI,GAAG,EAAU,CAAC;QAC5B,qBAAgB,GAAiC,IAAI,GAAG,EAA2B,CAAC;QACjF,6BAAwB,GAAoB,IAAI,GAAG,EAAE,CAAC;QAI/D,iBAAY,GAAqB;YACzC,KAAK,EAAE,IAAI,kCAAuB,EAAE;YACpC,WAAW,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE;YACxC,gBAAgB,EAAE,KAAK;YACvB,SAAS,EAAE,EAAE;SACd,CAAC;QAQA,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;IACjD,CAAC;IAEM,MAAM,CAAC,MAAkB;QAC9B,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE;YACf,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;SACtB;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEM,mBAAmB;QACxB,OAAO,IAAI,CAAC,gBAAgB,CAAC,IAAI,GAAG,CAAC,CAAC;IACxC,CAAC;IAEM,KAAK,CACV,OAAyB,EACzB,SAA2B,EAC3B,MAAwB,EACxB,KAAuB;QAEvB,0CAA0C;QAC1C,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;QACjE,MAAM,QAAQ,GAAG,IAAI,+CAAyB,CACtC,SAAS,EAAE;YACf,SAAS,EAAE,KAAK;YAChB,OAAO,EAAE,GAAG,EAAE;gBACZ,yDAAyD;gBACzD,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACzC,CAAC;SACF,CACF,CAAC;QAEF,0DAA0D;QAC1D,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,OAAQ,EAAE,SAAU,EAAE,MAAO,EAAE,KAAM,CAAC,CAAC;QAC9E,MAAM,QAAQ,GAAkB;YAC9B,KAAK,EAAE,IAAI,kCAAuB,EAAE;YACpC,WAAW,EAAE;gBACX,IAAI,EAAE,UAAU;gBAChB,KAAK,EAAE,KAAK;aACb;YACD,gBAAgB,EAAE,KAAK;SACxB,CAAC;QACF,QAAQ,CAAC,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QAC3C,QAAQ,CAAC,WAAW,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QAEzC,yEAAyE;QACzE,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;YACxB,QAAQ,CAAC,WAAW,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,CAAC;YAC3C,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,gEAAgE;QAChE,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEpC,4BAA4B;QAC5B,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,wBAAwB,EAAE;YACpD,QAAQ,EAAE,CAAC;SACZ;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEM,eAAe,CAAC,QAA0B,EAAE,YAAqB;QACtE,IAAI,CAAC,YAAY,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;QACpC,IAAI,CAAC,YAAY,CAAC,WAAW,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAE5D,IAAI,YAAY,EAAE;YAChB,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,gBAAgB,EAAE;gBAC5C,MAAM,KAAK,GAAW,QAAQ,CAAC,WAAW,CAAC,WAAW,CAAE,CAAC;gBACzD,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;aAC3C;SACF;IACH,CAAC;IAES,mBAAmB,CAAC,QAAiC,EAAE,KAAa;QAC5E,oDAAoD;QACpD,MAAM,WAAW,GAAqB;YACpC,KAAK,EAAE,IAAI,kCAAuB,EAAE;YACpC,WAAW,EAAE;gBACX,IAAI,EAAE,UAAU;gBAChB,KAAK,EAAE,KAAK;aACb;YACD,gBAAgB,EAAE,KAAK;YACvB,SAAS,EAAE,EAAE;SACd,CAAC;QAEF,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,YAAY,EAAE,WAAW,CAAC;aACrD,IAAI,CAAC,mBAAmB,CAAC,EAAE;YAC1B,mBAAmB,CAAC,KAAK,GAAG,IAAI,kCAAuB,EAAE,CAAC;YAE1D,0DAA0D;YAC1D,MAAM,oBAAoB,GAAG,QAAQ,CAAC,WAAW,CAAgB,UAAU,CAAC,CAAC;YAC7E,QAAQ,CAAC,WAAW,CAAC,UAAU,EAAE,mBAAmB,CAAC,CAAC;YACtD,oBAAoB,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC;QAC3C,CAAC,CAAC;aACD,KAAK,CAAC,GAAG,EAAE;YACV,cAAc;QAChB,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,0BAA0B,CAAC,QAAoB;QACpD,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC9C,CAAC;IAEM,6BAA6B,CAAC,QAAoB;QACvD,IAAI,CAAC,wBAAwB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACjD,CAAC;CACF;AAhID,wDAgIC","sourcesContent":["// eslint-disable-next-line import/no-nodejs-modules\nimport type { EventEmitter } from 'events';\nimport { ClosableTransformIterator } from '@comunica/bus-query-operation';\nimport { MetadataValidationState } from '@comunica/metadata';\nimport type { MetadataQuads, IAggregatedStore, MetadataBindings } from '@comunica/types';\nimport type * as RDF from '@rdfjs/types';\nimport type { AsyncIterator } from 'asynciterator';\nimport { StreamingStore } from 'rdf-streaming-store';\n\n/**\n * A StreamingStore that returns an AsyncIterator with a valid MetadataQuads property.\n */\nexport class StreamingStoreMetadata extends StreamingStore implements IAggregatedStore {\n  public started = false;\n  public containedSources = new Set<string>();\n  public readonly runningIterators: Set<AsyncIterator<RDF.Quad>> = new Set<AsyncIterator<RDF.Quad>>();\n  protected readonly iteratorCreatedListeners: Set<() => void> = new Set();\n  protected readonly metadataAccumulator:\n  (accumulatedMetadata: MetadataBindings, appendingMetadata: MetadataBindings) => Promise<MetadataBindings>;\n\n  protected baseMetadata: MetadataBindings = {\n    state: new MetadataValidationState(),\n    cardinality: { type: 'exact', value: 0 },\n    canContainUndefs: false,\n    variables: [],\n  };\n\n  public constructor(\n    store: RDF.Store | undefined,\n    metadataAccumulator:\n    (accumulatedMetadata: MetadataBindings, appendingMetadata: MetadataBindings) => Promise<MetadataBindings>,\n  ) {\n    super(store);\n    this.metadataAccumulator = metadataAccumulator;\n  }\n\n  public import(stream: RDF.Stream): EventEmitter {\n    if (!this.ended) {\n      super.import(stream);\n    }\n    return stream;\n  }\n\n  public hasRunningIterators(): boolean {\n    return this.runningIterators.size > 0;\n  }\n\n  public match(\n    subject?: RDF.Term | null,\n    predicate?: RDF.Term | null,\n    object?: RDF.Term | null,\n    graph?: RDF.Term | null,\n  ): AsyncIterator<RDF.Quad> {\n    // Wrap the raw stream in an AsyncIterator\n    const rawStream = super.match(subject, predicate, object, graph);\n    const iterator = new ClosableTransformIterator<RDF.Quad, RDF.Quad>(\n      <any> rawStream, {\n        autoStart: false,\n        onClose: () => {\n          // Running iterators are deleted once closed or destroyed\n          this.runningIterators.delete(iterator);\n        },\n      },\n    );\n\n    // Expose the metadata property containing the cardinality\n    let count = this.getStore().countQuads(subject!, predicate!, object!, graph!);\n    const metadata: MetadataQuads = {\n      state: new MetadataValidationState(),\n      cardinality: {\n        type: 'estimate',\n        value: count,\n      },\n      canContainUndefs: false,\n    };\n    iterator.setProperty('metadata', metadata);\n    iterator.setProperty('lastCount', count);\n\n    // Every time a new quad is pushed into the iterator, update the metadata\n    rawStream.on('quad', () => {\n      iterator.setProperty('lastCount', ++count);\n      this.updateMetadataState(iterator, count);\n    });\n\n    // Store all running iterators until they close or are destroyed\n    this.runningIterators.add(iterator);\n\n    // Invoke creation listeners\n    for (const listener of this.iteratorCreatedListeners) {\n      listener();\n    }\n\n    return iterator;\n  }\n\n  public setBaseMetadata(metadata: MetadataBindings, updateStates: boolean): void {\n    this.baseMetadata = { ...metadata };\n    this.baseMetadata.cardinality = { type: 'exact', value: 0 };\n\n    if (updateStates) {\n      for (const iterator of this.runningIterators) {\n        const count: number = iterator.getProperty('lastCount')!;\n        this.updateMetadataState(iterator, count);\n      }\n    }\n  }\n\n  protected updateMetadataState(iterator: AsyncIterator<RDF.Quad>, count: number): void {\n    // Append the given cardinality to the base metadata\n    const metadataNew: MetadataBindings = {\n      state: new MetadataValidationState(),\n      cardinality: {\n        type: 'estimate',\n        value: count,\n      },\n      canContainUndefs: false,\n      variables: [],\n    };\n\n    this.metadataAccumulator(this.baseMetadata, metadataNew)\n      .then(accumulatedMetadata => {\n        accumulatedMetadata.state = new MetadataValidationState();\n\n        // Set the new metadata, and invalidate the previous state\n        const metadataToInvalidate = iterator.getProperty<MetadataQuads>('metadata');\n        iterator.setProperty('metadata', accumulatedMetadata);\n        metadataToInvalidate?.state.invalidate();\n      })\n      .catch(() => {\n        // Void errors\n      });\n  }\n\n  public addIteratorCreatedListener(listener: () => void): void {\n    this.iteratorCreatedListeners.add(listener);\n  }\n\n  public removeIteratorCreatedListener(listener: () => void): void {\n    this.iteratorCreatedListeners.delete(listener);\n  }\n}\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@comunica/actor-query-source-identify-hypermedia",
-  "version": "3.0.1-alpha.43.0",
+  "version": "3.0.1-alpha.45.0",
   "description": "A hypermedia query-source-identify actor",
   "lsd:module": true,
   "main": "lib/index.js",
@@ -32,27 +32,27 @@
     "lib/**/*.js.map"
   ],
   "dependencies": {
-    "@comunica/actor-query-source-identify-rdfjs": "3.0.1-alpha.43.0",
-    "@comunica/bindings-factory": "3.0.1-alpha.43.0",
-    "@comunica/bus-dereference-rdf": "3.0.1-alpha.43.0",
-    "@comunica/bus-merge-bindings-context": "3.0.1-alpha.43.0",
-    "@comunica/bus-query-operation": "3.0.1-alpha.43.0",
-    "@comunica/bus-query-source-identify": "3.0.1-alpha.43.0",
-    "@comunica/bus-query-source-identify-hypermedia": "3.0.1-alpha.43.0",
-    "@comunica/bus-rdf-metadata": "3.0.1-alpha.43.0",
-    "@comunica/bus-rdf-metadata-accumulate": "3.0.1-alpha.43.0",
-    "@comunica/bus-rdf-metadata-extract": "3.0.1-alpha.43.0",
-    "@comunica/bus-rdf-resolve-hypermedia-links": "3.0.1-alpha.43.0",
-    "@comunica/bus-rdf-resolve-hypermedia-links-queue": "3.0.1-alpha.43.0",
-    "@comunica/context-entries": "3.0.1-alpha.43.0",
-    "@comunica/core": "3.0.1-alpha.43.0",
-    "@comunica/metadata": "3.0.1-alpha.43.0",
-    "@comunica/types": "3.0.1-alpha.43.0",
+    "@comunica/actor-query-source-identify-rdfjs": "3.0.1-alpha.45.0",
+    "@comunica/bindings-factory": "3.0.1-alpha.45.0",
+    "@comunica/bus-dereference-rdf": "3.0.1-alpha.45.0",
+    "@comunica/bus-merge-bindings-context": "3.0.1-alpha.45.0",
+    "@comunica/bus-query-operation": "3.0.1-alpha.45.0",
+    "@comunica/bus-query-source-identify": "3.0.1-alpha.45.0",
+    "@comunica/bus-query-source-identify-hypermedia": "3.0.1-alpha.45.0",
+    "@comunica/bus-rdf-metadata": "3.0.1-alpha.45.0",
+    "@comunica/bus-rdf-metadata-accumulate": "3.0.1-alpha.45.0",
+    "@comunica/bus-rdf-metadata-extract": "3.0.1-alpha.45.0",
+    "@comunica/bus-rdf-resolve-hypermedia-links": "3.0.1-alpha.45.0",
+    "@comunica/bus-rdf-resolve-hypermedia-links-queue": "3.0.1-alpha.45.0",
+    "@comunica/context-entries": "3.0.1-alpha.45.0",
+    "@comunica/core": "3.0.1-alpha.45.0",
+    "@comunica/metadata": "3.0.1-alpha.45.0",
+    "@comunica/types": "3.0.1-alpha.45.0",
     "@rdfjs/types": "*",
     "asynciterator": "^3.8.1",
     "lru-cache": "^10.0.0",
     "rdf-data-factory": "^1.1.2",
-    "rdf-streaming-store": "^1.1.0",
+    "rdf-streaming-store": "^1.1.4",
     "readable-stream": "^4.4.2",
     "sparqlalgebrajs": "^4.3.3"
   },
@@ -61,5 +61,5 @@
     "build:ts": "node \"../../node_modules/typescript/bin/tsc\"",
     "build:components": "componentsjs-generator"
   },
-  "gitHead": "d11e44cf07d4699f9d2c51d5851b5ed443de1997"
+  "gitHead": "b5a3900c24e29fa7b3243ab50bc9745d286fbce8"
 }