@netwerk-digitaal-erfgoed/network-of-terms-query 6.2.8 → 6.2.10

package/src/lookup/lookup.ts

@@ -0,0 +1,147 @@
+import { Catalog, Dataset, Distribution, IRI } from '../catalog.js';
+import {
+  Error,
+  QueryTermsService,
+  ServerError,
+  Terms,
+  TermsResponse,
+  TermsResult,
+  TimeoutError,
+} from '../query.js';
+import { Term } from '../terms.js';
+import { clientQueriesCounter } from '../instrumentation.js';
+
+export type LookupQueryResult = {
+  uri: string;
+  distribution: SourceResult;
+  result: LookupResult;
+
+  responseTimeMs: number;
+};
+
+export type SourceResult = Distribution | SourceNotFoundError;
+
+export type LookupResult = Term | NotFoundError | TimeoutError | ServerError;
+
+export class SourceNotFoundError {
+  readonly message: string;
+
+  constructor(readonly iri: string) {
+    this.message = `No source found that can provide term with URI ${iri}`;
+  }
+}
+
+export class NotFoundError {
+  readonly message: string;
+
+  constructor(readonly iri: string) {
+    this.message = `No term found with URI ${iri}`;
+  }
+}
+
+export class LookupService {
+  constructor(
+    private catalog: Catalog,
+    private queryService: QueryTermsService,
+  ) {}
+
+  public async lookup(
+    iris: string[],
+    timeoutMs: number,
+  ): Promise<LookupQueryResult[]> {
+    const irisToDataset = iris.reduce((acc, iri) => {
+      const dataset = this.catalog.getDatasetByTermIri(iri);
+      if (dataset) {
+        acc.set(iri.toString(), dataset);
+      }
+      return acc;
+    }, new Map<string, Dataset>());
+
+    const datasetToIris = [...irisToDataset].reduce(
+      (datasetMap, [iri, dataset]) => {
+        datasetMap.set(dataset, [...(datasetMap.get(dataset) ?? []), iri]);
+        return datasetMap;
+      },
+      new Map<Dataset, IRI[]>(),
+    );
+
+    const lookups = [...datasetToIris].map(([dataset]) =>
+      this.queryService.lookup(iris, dataset.distributions[0], timeoutMs),
+    );
+
+    const termsPerSource: TermsResponse[] = await Promise.all(lookups);
+
+    const datasetToTerms = termsPerSource.reduce(
+      (acc, response: TermsResponse) => {
+        let dataset = this.catalog.getDatasetByDistributionIri(
+          response.result.distribution.iri,
+        )!;
+        if (response.result instanceof Terms) {
+          const termsResult =
+            (acc.get(dataset)?.result as Terms) ??
+            new Terms(response.result.distribution, []);
+          for (const term of response.result.terms) {
+            if (term.datasetIri !== undefined) {
+              const termsDataset = this.catalog.getDatasetByIri(
+                term.datasetIri.value,
+              );
+              if (termsDataset !== undefined) {
+                dataset = termsDataset;
+                irisToDataset.set(term.id.value, dataset);
+              }
+            }
+            termsResult.terms.push(term);
+          }
+          acc.set(
+            dataset,
+            new TermsResponse(termsResult, response.responseTimeMs),
+          );
+        } else {
+          const dataset = this.catalog.getDatasetByDistributionIri(
+            response.result.distribution.iri,
+          )!;
+          acc.set(dataset, response);
+        }
+        return acc;
+      },
+      new Map<Dataset, TermsResponse>(),
+    );
+
+    return iris.map((iri) => {
+      const dataset = irisToDataset.get(iri.toString());
+      if (dataset === undefined) {
+        clientQueriesCounter.add(1, {
+          type: 'lookup',
+          error: 'SourceNotFound',
+        });
+        return {
+          uri: iri,
+          distribution: new SourceNotFoundError(iri),
+          result: new NotFoundError(iri),
+          responseTimeMs: 0,
+        };
+      }
+
+      const response = datasetToTerms.get(dataset)!;
+      clientQueriesCounter.add(1, { type: 'lookup' });
+
+      return {
+        uri: iri,
+        distribution: dataset.distributions[0],
+        result: result(response.result, iri),
+        responseTimeMs: response.responseTimeMs,
+      };
+    });
+  }
+}
+
+function result(result: TermsResult, iri: string): LookupResult {
+  if (result instanceof Error) {
+    return result;
+  }
+
+  return (
+    result.terms.find((term) => term.id.value === iri.toString()) ??
+    new NotFoundError(iri)
+  );
+}

package/src/query.ts

@@ -0,0 +1,302 @@
+import * as Hoek from '@hapi/hoek';
+import Joi from 'joi';
+import { LoggerPino } from './helpers/logger-pino.js';
+import Pino from 'pino';
+import PrettyMilliseconds from 'pretty-ms';
+import * as RDF from '@rdfjs/types';
+import { Term, TermsTransformer } from './terms.js';
+import { QueryMode, queryVariants } from './search/query-mode.js';
+import { Dataset, Distribution, IRI } from './catalog.js';
+import { QueryEngine } from '@comunica/query-sparql';
+import { BindingsFactory } from '@comunica/utils-bindings-factory';
+import { DataFactory } from 'rdf-data-factory';
+import { sourceQueriesHistogram } from './instrumentation.js';
+import { config } from './config.js';
+
+/**
+ * Check if a query requires string substitution instead of initialBindings.
+ * Workaround for Comunica v5 traqula bug that crashes with:
+ * - SERVICE clauses
+ * - VALUES combination
+ */
+function requiresStringSubstitution(query: string): boolean {
+  const hasService = /\bSERVICE\b/i.test(query);
+  const hasValues = /\bVALUES\b/i.test(query);
+  return hasService || hasValues;
+}
+
+/**
+ * Substitute bindings directly into a SPARQL query string.
+ * This is a workaround for Comunica v5's initialBindings bug with SERVICE clauses.
+ */
+function substituteBindings(
+  query: string,
+  bindings: Record<string, RDF.Term>,
+): string {
+  let result = query;
+  for (const [name, term] of Object.entries(bindings)) {
+    const pattern = new RegExp(`\\?${name}\\b`, 'g');
+    if (term.termType === 'NamedNode') {
+      result = result.replace(pattern, `<${term.value}>`);
+    } else if (term.termType === 'Literal') {
+      const literal = term as RDF.Literal;
+      const datatype = literal.datatype?.value;
+      if (
+        datatype &&
+        datatype !== 'http://www.w3.org/2001/XMLSchema#string' &&
+        datatype !== 'http://www.w3.org/1999/02/22-rdf-syntax-ns#langString'
+      ) {
+        result = result.replace(pattern, `"${term.value}"^^<${datatype}>`);
+      } else if (literal.language) {
+        result = result.replace(pattern, `"${term.value}"@${literal.language}`);
+      } else {
+        result = result.replace(pattern, `"${term.value}"`);
+      }
+    }
+  }
+  return result;
+}
+
+export type TermsResult = Terms | TimeoutError | ServerError;
+
+export class TermsResponse {
+  constructor(
+    readonly result: TermsResult,
+    readonly responseTimeMs: number,
+  ) {}
+}
+
+export class Terms {
+  constructor(
+    readonly distribution: Distribution,
+    readonly terms: Term[],
+  ) {}
+}
+
+export class Error {
+  constructor(
+    readonly distribution: Distribution,
+    readonly message: string,
+  ) {}
+}
+
+export class TimeoutError extends Error {
+  constructor(
+    override readonly distribution: Distribution,
+    timeoutMs: number,
+  ) {
+    super(distribution, `Source timed out after ${timeoutMs}ms`);
+  }
+}
+
+export class ServerError extends Error {}
+
+export class QueryTermsService {
+  private readonly logger: Pino.Logger;
+  private readonly engine: QueryEngine;
+
+  constructor(options: { comunica?: QueryEngine; logger?: Pino.Logger } = {}) {
+    this.engine = options.comunica || new QueryEngine();
+    this.logger = options.logger || Pino.pino();
+  }
+
+  /**
+   * Parameterize the SPARQL query’s limit in two ways:
+   * - as a pre-bound variable ?limit (for GraphDB’s luc:limit, Wikidata and text:query);
+   * - by replacing the #LIMIT# placeholder (for LIMIT 123).
+   */
+  parameterizeLimit(args: {
+    query: string;
+    bindings: Record<string, RDF.Term>;
+    limit: number;
+  }): { queryWithLimit: string; bindingsWithLimit: Record<string, RDF.Term> } {
+    return {
+      queryWithLimit: args.query.replace('#LIMIT#', `LIMIT ${args.limit}`),
+      bindingsWithLimit: {
+        ...args.bindings,
+        limit: dataFactory.literal(
+          args.limit.toString(),
+          dataFactory.namedNode('http://www.w3.org/2001/XMLSchema#integer'),
+        ),
+      },
+    };
+  }
+
+  async search(
+    searchQuery: string,
+    queryMode: QueryMode,
+    dataset: Dataset,
+    distribution: Distribution,
+    limit: number,
+    timeoutMs: number,
+  ): Promise<TermsResponse> {
+    const bindings = [...queryVariants(searchQuery, queryMode)].reduce(
+      (record: Record<string, RDF.Term>, [k, v]) => {
+        record[k] = dataFactory.literal(v);
+        return record;
+      },
+      {},
+    );
+    bindings['datasetUri'] = dataFactory.namedNode(dataset.iri.toString());
+
+    const { queryWithLimit, bindingsWithLimit } = this.parameterizeLimit({
+      query: distribution.searchQuery,
+      bindings,
+      limit,
+    });
+
+    return this.run(
+      // For plain SPARQL LIMIT (LIMIT 123) that cannot be pre-bound
+      queryWithLimit,
+      distribution,
+      timeoutMs,
+      bindingsWithLimit,
+    );
+  }
+
+  async lookup(iris: IRI[], distribution: Distribution, timeoutMs: number) {
+    return this.run(
+      distribution.lookupQuery.replace(
+        '?uris',
+        iris.map((iri) => `<${iri}>`).join(' '),
+      ),
+      distribution,
+      timeoutMs,
+    );
+  }
+
+  async run(
+    query: string,
+    distribution: Distribution,
+    timeoutMs: number,
+    bindings: Record<string, RDF.Term> = {},
+  ): Promise<TermsResponse> {
+    Joi.attempt(
+      timeoutMs,
+      Joi.number()
+        .integer()
+        .min(1)
+        .max(config.MAX_QUERY_TIMEOUT)
+        .default(config.DEFAULT_QUERY_TIMEOUT),
+    );
+
+    const timer = new Hoek.Bench();
+    const logger = new LoggerPino({ logger: this.logger });
+    // Extract HTTP credentials if the distribution URL contains any.
+    const url = new URL(distribution.endpoint.toString());
+
+    // Workaround for https://github.com/comunica/comunica/issues/1655, so use
+    // string substitution instead of initialBindings for:
+    // - SERVICE clauses crash with initialBindings
+    // - VALUES crashes in some combinations
+    const useStringSubstitution = requiresStringSubstitution(query);
+    const finalQuery = useStringSubstitution
+      ? substituteBindings(query, bindings)
+      : query;
+
+    this.logger.info(`Querying "${url}" with "${finalQuery}"...`);
+    const quadStream = await this.engine.queryQuads(finalQuery, {
+      log: logger,
+      httpAuth:
+        url.username === '' ? undefined : url.username + ':' + url.password,
+      httpTimeout: timeoutMs,
+      noCache: true,
+      sources: [
+        {
+          type: 'sparql',
+          value: url.origin + url.pathname,
+        },
+      ],
+      // Only pass initialBindings when NOT using string substitution
+      ...(useStringSubstitution
+        ? {}
+        : { initialBindings: bindingsFactory.fromRecord(bindings) }),
+    });
+
+    return new Promise((resolve) => {
+      const termsTransformer = new TermsTransformer();
+      quadStream.on('error', (error) => {
+        const elapsed = Math.round(timer.elapsed());
+        this.logger.error(
+          `An error occurred when querying "${distribution.endpoint}": ${error} with %o`,
+          error,
+        );
+
+        if (error.message.startsWith('Fetch timed out')) {
+          sourceQueriesHistogram.record(Math.round(timer.elapsed()), {
+            distribution: distribution.iri.toString(),
+            error: 'TimeoutError',
+          });
+          resolve(
+            new TermsResponse(
+              new TimeoutError(distribution, timeoutMs),
+              elapsed,
+            ),
+          );
+        } else {
+          sourceQueriesHistogram.record(Math.round(timer.elapsed()), {
+            distribution: distribution.iri.toString(),
+            error: 'ServerError',
+          });
+          resolve(
+            new TermsResponse(
+              new ServerError(
+                distribution,
+                obfuscateHttpCredentials(error.message),
+              ),
+              elapsed,
+            ),
+          );
+        }
+      });
+      quadStream.on('data', (quad: RDF.Quad) => {
+        termsTransformer.fromQuad(quad);
+      });
+      quadStream.on('end', () => {
+        const terms = termsTransformer
+          .asArray()
+          .sort(byScoreThenAlphabetically);
+        this.logger.info(
+          `Found ${terms.length} terms matching "${query}" in "${
+            distribution.endpoint
+          }" in ${PrettyMilliseconds(timer.elapsed())}`,
+        );
+        sourceQueriesHistogram.record(Math.round(timer.elapsed()), {
+          distribution: distribution.iri.toString(),
+        });
+        resolve(
+          new TermsResponse(
+            new Terms(distribution, terms),
+            Math.round(timer.elapsed()),
+          ),
+        );
+      });
+    });
+  }
+}
+
+const byScoreThenAlphabetically = (a: Term, b: Term) => {
+  const scoreA = parseFloat(a.score?.value ?? '0');
+  const scoreB = parseFloat(b.score?.value ?? '0');
+  if (scoreA === scoreB) {
+    return alphabeticallyByLabels(a, b);
+  } else {
+    return scoreA < scoreB ? 1 : -1;
+  }
+};
+
+const alphabeticallyByLabels = (a: Term, b: Term) => {
+  const prefLabelA = a.prefLabels[0]?.value ?? '';
+  const altLabelA = a.altLabels[0]?.value ?? '';
+  const sortLabelA = prefLabelA + altLabelA;
+  const prefLabelB = b.prefLabels[0]?.value ?? '';
+  const altLabelB = b.altLabels[0]?.value ?? '';
+  const sortLabelB = prefLabelB + altLabelB;
+  return sortLabelA.localeCompare(sortLabelB);
+};
+
+const dataFactory = new DataFactory();
+const bindingsFactory = new BindingsFactory(dataFactory);
+
+const obfuscateHttpCredentials = (message: string) =>
+  message.replace(/(https?):\/\/.+:.+@/, '$1://***@');

package/src/search/query-mode.ts

@@ -0,0 +1,54 @@
+export enum QueryMode {
+  RAW = 'raw',
+  OPTIMIZED = 'optimized',
+}
+
+export function queryVariants(query: string, type: QueryMode) {
+  switch (type) {
+    case QueryMode.RAW:
+      return new Map([
+        ['query', query],
+        ['virtuosoQuery', query],
+      ]);
+    case QueryMode.OPTIMIZED:
+      return new Map([
+        ['query', stringQuery(query)],
+        ['virtuosoQuery', virtuosoQuery(stringQuery(query))],
+      ]);
+  }
+}
+
+const stringQuery = (query: string) =>
+  query.toLowerCase().replace(/\s+/g, ' ').trim();
+
+const virtuosoQuery = (query: string) =>
+  join(quote(filterStopWords(split(escape(query)))));
+
+const escape = (query: string) => query.replace(/'/g, "\\'");
+
+const split = (query: string) => query.split(/\s+/);
+
+/**
+ * Quote parts that are not boolean operators.
+ */
+const quote = (queryParts: string[]) =>
+  queryParts.map((part) => (isBooleanOperator(part) ? part : `'${part}'`));
+
+const filterStopWords = (queryParts: string[]) =>
+  queryParts.filter((part) => part !== '&');
+
+/**
+ * Join query parts with boolean AND if they are not yet connected with a boolean.
+ */
+const join = (queryParts: string[]) =>
+  queryParts.reduce((previousValue, currentValue, currentIndex, array) => {
+    const previous = array[currentIndex - 1];
+    if (!isBooleanOperator(previous) && !isBooleanOperator(currentValue)) {
+      return `${previousValue} AND ${currentValue}`;
+    }
+
+    return `${previousValue} ${currentValue}`;
+  });
+
+const isBooleanOperator = (maybeBool: string) =>
+  maybeBool.toLowerCase() === 'and' || maybeBool.toLowerCase() === 'or';

package/src/terms.ts

@@ -0,0 +1,141 @@
+import type RDF from '@rdfjs/types';
+
+export class Term {
+  constructor(
+    readonly id: RDF.Term,
+    readonly type: RDF.Term | undefined,
+    readonly prefLabels: RDF.Literal[],
+    readonly altLabels: RDF.Literal[],
+    readonly hiddenLabels: RDF.Literal[],
+    readonly scopeNotes: RDF.Literal[],
+    readonly seeAlso: RDF.NamedNode[],
+    readonly broaderTerms: RelatedTerm[],
+    readonly narrowerTerms: RelatedTerm[],
+    readonly relatedTerms: RelatedTerm[],
+    readonly exactMatches: RelatedTerm[],
+    readonly datasetIri: RDF.Term | undefined,
+    readonly score: RDF.Literal | undefined,
+  ) {}
+}
+
+export class RelatedTerm {
+  constructor(
+    readonly id: RDF.Term,
+    readonly prefLabels: RDF.Literal[],
+  ) {}
+}
+
+class SparqlResultTerm {
+  constructor(readonly id: RDF.Term) {}
+  type: RDF.Term | undefined = undefined;
+  prefLabels: RDF.Literal[] = [];
+  altLabels: RDF.Literal[] = [];
+  hiddenLabels: RDF.Literal[] = [];
+  scopeNotes: RDF.Literal[] = [];
+  seeAlso: RDF.NamedNode[] = [];
+  broaderTerms: RDF.Term[] = [];
+  narrowerTerms: RDF.Term[] = [];
+  relatedTerms: RDF.Term[] = [];
+  exactMatches: RDF.Term[] = [];
+  inScheme: RDF.Term | undefined = undefined;
+  score: RDF.Literal | undefined = undefined;
+}
+
+export class TermsTransformer {
+  private termsIris: Set<string> = new Set();
+  private termsMap: Map<string, SparqlResultTerm> = new Map();
+  private readonly predicateToPropertyMap = new Map<string, string>([
+    ['http://www.w3.org/1999/02/22-rdf-syntax-ns#type', 'type'],
+    ['http://www.w3.org/2000/01/rdf-schema#seeAlso', 'seeAlso'],
+    ['http://www.w3.org/2004/02/skos/core#prefLabel', 'prefLabels'],
+    ['http://www.w3.org/2008/05/skos#prefLabel', 'prefLabels'],
+    ['http://www.w3.org/2004/02/skos/core#altLabel', 'altLabels'],
+    ['http://www.w3.org/2008/05/skos#altLabel', 'altLabels'],
+    ['http://www.w3.org/2004/02/skos/core#hiddenLabel', 'hiddenLabels'],
+    ['http://www.w3.org/2008/05/skos#hiddenLabel', 'hiddenLabels'],
+    ['http://www.w3.org/2004/02/skos/core#scopeNote', 'scopeNotes'],
+    ['http://www.w3.org/2008/05/skos#scopeNote', 'scopeNotes'],
+    ['http://www.w3.org/2004/02/skos/core#broader', 'broaderTerms'],
+    ['http://www.w3.org/2008/05/skos#broader', 'broaderTerms'],
+    ['http://www.w3.org/2004/02/skos/core#narrower', 'narrowerTerms'],
+    ['http://www.w3.org/2008/05/skos#narrower', 'narrowerTerms'],
+    ['http://www.w3.org/2004/02/skos/core#related', 'relatedTerms'],
+    ['http://www.w3.org/2008/05/skos#related', 'relatedTerms'],
+    ['http://www.w3.org/2004/02/skos/core#exactMatch', 'exactMatches'],
+    ['http://www.w3.org/2008/05/skos#exactMatch', 'exactMatches'],
+    ['http://www.w3.org/2004/02/skos/core#inScheme', 'inScheme'],
+    ['http://purl.org/voc/vrank#simpleRank', 'score'],
+  ]);
+
+  fromQuad(quad: RDF.Quad): void {
+    const subject = quad.subject;
+    const propertyName = this.predicateToPropertyMap.get(quad.predicate.value);
+    const currentTerm =
+      this.termsMap.get(subject.value) ?? new SparqlResultTerm(subject);
+    this.termsMap.set(subject.value, currentTerm);
+
+    // skos:Concepts are the top-level search results, which we track in termsIris.
+    if (
+      propertyName === 'type' &&
+      (quad.object.value === 'http://www.w3.org/2004/02/skos/core#Concept' ||
+        quad.object.value === 'http://www.w3.org/2008/05/skos#Concept')
+    ) {
+      this.termsIris.add(subject.value);
+    }
+
+    if (propertyName !== undefined) {
+      const propertyValue = (currentTerm as any)[propertyName]; // eslint-disable-line @typescript-eslint/no-explicit-any
+      if (Array.isArray(propertyValue)) {
+        // Prevent duplicate prefLabel values due to duplicate quads.
+        if (!propertyValue.find((value) => value.equals(quad.object))) {
+          propertyValue.push(quad.object);
+        }
+      } else {
+        (currentTerm as any)[propertyName] = quad.object; // eslint-disable-line @typescript-eslint/no-explicit-any
+      }
+    }
+  }
+
+  asArray(): Term[] {
+    return [...this.termsIris].map((iri) => {
+      const term = this.termsMap.get(iri)!;
+
+      return new Term(
+        term.id,
+        term.type,
+        term.prefLabels,
+        term.altLabels,
+        term.hiddenLabels,
+        term.scopeNotes,
+        term.seeAlso,
+        this.mapRelatedTerms(term.broaderTerms).sort(alphabeticallyByPrefLabel),
+        this.mapRelatedTerms(term.narrowerTerms).sort(
+          alphabeticallyByPrefLabel,
+        ),
+        this.mapRelatedTerms(term.relatedTerms).sort(alphabeticallyByPrefLabel),
+        this.mapRelatedTerms(term.exactMatches).sort(alphabeticallyByPrefLabel),
+        term.inScheme,
+        term.score,
+      );
+    });
+  }
+
+  /**
+   * Map related IRIs to their related terms, making sure to only accept complete related terms.
+   *
+   * Related terms can be incomplete because of the SPARQL query limit (see
+   * https://github.com/netwerk-digitaal-erfgoed/network-of-terms/issues/36).
+   */
+  private mapRelatedTerms = (terms: RDF.Term[]) =>
+    terms.reduce((acc: RelatedTerm[], iri: RDF.Term) => {
+      const term = this.termsMap.get(iri.value);
+      acc.push(new RelatedTerm(iri, term?.prefLabels ?? []));
+      return acc;
+    }, []);
+}
+
+const alphabeticallyByPrefLabel = (a: RelatedTerm, b: RelatedTerm) => {
+  const prefLabelA = a.prefLabels[0]?.value ?? '';
+  const prefLabelB = b.prefLabels[0]?.value ?? '';
+  return prefLabelA.localeCompare(prefLabelB);
+};