@netwerk-digitaal-erfgoed/network-of-terms-query 6.2.7 → 6.2.9

package/src/catalog.ts

@@ -0,0 +1,157 @@
+import { URL } from 'url';
+
+export class Catalog {
+  private readonly prefixToDataset: Map<string, IRI>;
+
+  constructor(readonly datasets: ReadonlyArray<Dataset>) {
+    this.prefixToDataset = this.indexPrefixesByStringLength();
+  }
+
+  /**
+   * Get dataset by IRI, accepting distribution IRIs too for BC.
+   */
+  public getDatasetByIri(iri: IRI): Dataset | undefined {
+    return (
+      this.datasets.find(
+        (dataset) => dataset.iri.toString() === iri.toString(),
+      ) ?? this.getDatasetByDistributionIri(iri)
+    );
+  }
+
+  public getDatasetsSortedByName(languageCode: string): Dataset[] {
+    return [...this.datasets].sort((a, b) =>
+      a.name[languageCode].localeCompare(b.name[languageCode]),
+    );
+  }
+
+  public getDatasetByDistributionIri(iri: IRI): Dataset | undefined {
+    return this.datasets.find(
+      (dataset) => dataset.getDistributionByIri(iri) !== undefined,
+    );
+  }
+
+  public getDatasetByTermIri(iri: IRI): Dataset | undefined {
+    for (const [prefix, datasetIri] of this.prefixToDataset) {
+      if (iri.toString().startsWith(prefix)) {
+        return this.getDatasetByIri(datasetIri);
+      }
+    }
+
+    return undefined;
+  }
+
+  public getDistributionsProvidingFeature(
+    featureType: FeatureType,
+  ): Distribution[] {
+    return this.datasets.reduce<Distribution[]>((acc, dataset) => {
+      return [
+        ...acc,
+        ...dataset.distributions.filter((distribution) =>
+          distribution.hasFeature(featureType),
+        ),
+      ];
+    }, []);
+  }
+
+  /**
+   * Get all languages provided by datasets in this catalog.
+   */
+  public getLanguages(): string[] {
+    return [
+      ...new Set(
+        this.datasets.reduce<string[]>((acc, dataset) => {
+          return [...acc, ...dataset.inLanguage];
+        }, []),
+      ),
+    ];
+  }
+
+  /**
+   * Index the prefixes of all datasets by their string length in descending order for matching
+   * term IRIs against during lookup. When looking up terms, we want to match the longest possible prefix
+   * in case prefixes overlap.
+   */
+  private indexPrefixesByStringLength() {
+    return new Map(
+      [
+        ...this.datasets
+          .reduce((acc, dataset) => {
+            dataset.termsPrefixes.forEach((prefix) => {
+              acc.set(prefix.toString(), dataset.iri);
+            });
+            return acc;
+          }, new Map<string, IRI>())
+          .entries(),
+      ].sort(([a], [b]) => b.localeCompare(a)),
+    );
+  }
+}
+
+export type StringDictionary = Record<string, string>;
+
+export class Dataset {
+  constructor(
+    readonly iri: IRI,
+    readonly name: StringDictionary,
+    readonly description: StringDictionary,
+    readonly genres: IRI[],
+    readonly termsPrefixes: IRI[],
+    readonly mainEntityOfPage: string,
+    readonly inLanguage: string[],
+    readonly creators: [Organization],
+    readonly distributions: [Distribution],
+    readonly alternateName: StringDictionary = {},
+  ) {}
+
+  public getSparqlDistribution(): Distribution | undefined {
+    return this.distributions.find(
+      (distribution) => distribution instanceof SparqlDistribution,
+    );
+  }
+
+  public getDistributionByIri(iri: IRI): Distribution | undefined {
+    return this.distributions.find(
+      (distribution) => distribution.iri.toString() === iri.toString(),
+    );
+  }
+}
+
+export class Organization {
+  constructor(
+    readonly iri: IRI,
+    readonly name: StringDictionary,
+    readonly alternateName: StringDictionary,
+  ) {}
+}
+
+export class SparqlDistribution {
+  constructor(
+    readonly iri: IRI,
+    readonly endpoint: IRI,
+    readonly searchQuery: string,
+    readonly lookupQuery: string,
+    readonly features: Feature[] = [],
+  ) {}
+
+  public hasFeature(feature: FeatureType) {
+    return this.features.some((value: Feature) => value.type === feature);
+  }
+}
+
+export class Feature {
+  constructor(
+    readonly type: FeatureType,
+    readonly url: URL,
+  ) {}
+}
+
+export enum FeatureType {
+  RECONCILIATION = 'https://reconciliation-api.github.io/specs/latest/',
+}
+
+/**
+ * A union type to be extended in the future with other distribution types.
+ */
+export type Distribution = SparqlDistribution;
+
+export type IRI = string;

package/src/config.ts

@@ -0,0 +1,24 @@
+import { envSchema, JSONSchemaType } from 'env-schema';
+
+const schema = {
+  type: 'object',
+  properties: {
+    MAX_QUERY_TIMEOUT: {
+      type: 'number',
+      default: 60000,
+    },
+    DEFAULT_QUERY_TIMEOUT: {
+      type: 'number',
+      default: 5000,
+    },
+  },
+};
+
+interface Env {
+  MAX_QUERY_TIMEOUT: number;
+  DEFAULT_QUERY_TIMEOUT: number;
+}
+
+export const config: JSONSchemaType<Env> = envSchema({
+  schema,
+});

package/src/distributions.ts

@@ -0,0 +1,94 @@
+import { QueryEngine } from '@comunica/query-sparql';
+import Joi from 'joi';
+import Pino from 'pino';
+import { QueryTermsService, TermsResponse } from './query.js';
+import { QueryMode } from './search/query-mode.js';
+import { Catalog, IRI } from './catalog.js';
+import { comunica } from './index.js';
+import { clientQueriesCounter } from './instrumentation.js';
+
+interface BaseQueryOptions {
+  query: string;
+  queryMode: QueryMode;
+  limit: number;
+  timeoutMs: number;
+}
+
+export interface QueryOptions extends BaseQueryOptions {
+  source: IRI;
+}
+
+export interface QueryAllOptions extends BaseQueryOptions {
+  sources: IRI[];
+}
+
+const schemaBase = Joi.object({
+  query: Joi.string().required(),
+  queryMode: Joi.string().required(),
+  limit: Joi.number().integer(),
+  timeoutMs: Joi.number().integer(),
+});
+
+const schemaQuery = schemaBase.append({
+  source: Joi.string().required(),
+});
+
+const schemaQueryAll = schemaBase.append({
+  sources: Joi.array().items(Joi.string().required()).min(1).required(),
+});
+
+export class DistributionsService {
+  private logger: Pino.Logger;
+  private catalog: Catalog;
+  private comunica: QueryEngine;
+
+  constructor(options: {
+    logger: Pino.Logger;
+    catalog: Catalog;
+    comunica?: QueryEngine;
+  }) {
+    this.logger = options.logger;
+    this.catalog = options.catalog;
+    this.comunica = options.comunica || comunica();
+  }
+
+  async query(options: QueryOptions): Promise<TermsResponse> {
+    const args = Joi.attempt(options, schemaQuery);
+    this.logger.info(`Preparing to query source "${args.source}"...`);
+    const dataset = this.catalog.getDatasetByIri(args.source);
+    if (dataset === undefined) {
+      throw Error(`Source with URI "${args.source}" not found`);
+    }
+    const distribution = dataset.getSparqlDistribution()!;
+    const queryService = new QueryTermsService({
+      logger: this.logger,
+      comunica: this.comunica,
+    });
+    return queryService.search(
+      args.query,
+      args.queryMode,
+      dataset,
+      distribution,
+      args.limit,
+      args.timeoutMs,
+    );
+  }
+
+  async queryAll(options: QueryAllOptions): Promise<TermsResponse[]> {
+    const args = Joi.attempt(options, schemaQueryAll);
+    clientQueriesCounter.add(1, {
+      numberOfSources: args.sources.length,
+      type: 'search',
+    });
+    const requests = args.sources.map((source: IRI) =>
+      this.query({
+        source,
+        query: args.query,
+        queryMode: args.queryMode,
+        limit: args.limit,
+        timeoutMs: args.timeoutMs,
+      }),
+    );
+    return Promise.all(requests);
+  }
+}

package/src/helpers/logger-pino.ts

@@ -0,0 +1,45 @@
+import { Logger } from '@comunica/types';
+import Pino from 'pino';
+
+export interface ConstructorOptions {
+  logger: Pino.Logger;
+}
+
+export class LoggerPino extends Logger {
+  protected logger: Pino.Logger;
+
+  constructor(options: ConstructorOptions) {
+    super();
+    this.logger = options.logger;
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  debug(message: string, data?: any): void {
+    this.logger.debug(message, data);
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  error(message: string, data?: any): void {
+    this.logger.error(message, data);
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  fatal(message: string, data?: any): void {
+    this.logger.fatal(message, data);
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  info(message: string, data?: any): void {
+    this.logger.info(message, data);
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  trace(message: string, data?: any): void {
+    this.logger.trace(message, data);
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  warn(message: string, data?: any): void {
+    this.logger.warn(message, data);
+  }
+}

package/src/helpers/logger.ts

@@ -0,0 +1,52 @@
+import Joi from 'joi';
+import Pino from 'pino';
+
+export interface GetLoggerOptions {
+  name: string;
+  level: string;
+}
+
+const schemaGetLogger = Joi.object({
+  name: Joi.string().required(),
+  level: Joi.string().required(),
+});
+
+const baseOptions: Pino.LoggerOptions = {
+  base: {
+    name: undefined, // Don't log PID and hostname
+  },
+  level: 'warn',
+  messageKey: 'message',
+  formatters: {
+    level(label) {
+      return {
+        level: label,
+      };
+    },
+  },
+};
+
+export function getCliLogger(options: GetLoggerOptions): Pino.Logger {
+  const args = Joi.attempt(options, schemaGetLogger);
+  const loggerOptions = Object.assign(baseOptions, {
+    base: {
+      name: args.name,
+    },
+    level: args.level,
+    prettyPrint: {
+      colorize: true,
+    },
+  });
+  const destinationStdErr = Pino.destination(2);
+  return Pino.pino(loggerOptions, destinationStdErr);
+}
+
+export function getHttpLogger(options: GetLoggerOptions): Pino.LoggerOptions {
+  const args = Joi.attempt(options, schemaGetLogger);
+  return Object.assign(baseOptions, {
+    base: {
+      name: args.name,
+    },
+    level: args.level,
+  });
+}

package/src/index.ts

@@ -0,0 +1,12 @@
+export * from './query.js';
+export * from './catalog.js';
+export * from './literal.js';
+export * from './lookup/lookup.js';
+export * from './terms.js';
+export * from './search/query-mode.js';
+export * from './distributions.js';
+export * from './helpers/logger.js';
+
+import { QueryEngine } from '@comunica/query-sparql';
+
+export const comunica = () => new QueryEngine();

package/src/instrumentation.ts

@@ -0,0 +1,51 @@
+import {
+  MeterProvider,
+  PeriodicExportingMetricReader,
+} from '@opentelemetry/sdk-metrics';
+import {
+  defaultResource,
+  resourceFromAttributes,
+} from '@opentelemetry/resources';
+import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-proto';
+import { ATTR_SERVICE_NAME } from '@opentelemetry/semantic-conventions';
+import { metrics, ValueType } from '@opentelemetry/api';
+
+const sourceQueriesHistogramName = 'queries.source';
+
+const meterProvider = new MeterProvider({
+  resource: defaultResource().merge(
+    resourceFromAttributes({ [ATTR_SERVICE_NAME]: 'network-of-terms' }),
+  ),
+  readers:
+    'test' === process.env.NODE_ENV
+      ? []
+      : [
+          new PeriodicExportingMetricReader({
+            exporter: new OTLPMetricExporter(),
+            exportIntervalMillis:
+              (process.env.OTEL_METRIC_EXPORT_INTERVAL as unknown as number) ??
+              60000,
+          }),
+        ],
+});
+
+metrics.setGlobalMeterProvider(meterProvider);
+
+const meter = metrics.getMeter('default');
+
+export const clientQueriesCounter = meter.createCounter(
+  'queries.client.counter',
+  {
+    description: 'Number of user queries',
+    valueType: ValueType.INT,
+  },
+);
+
+export const sourceQueriesHistogram = meter.createHistogram(
+  sourceQueriesHistogramName,
+  {
+    description: 'Queries to terminology sources and their response times',
+    valueType: ValueType.INT,
+    unit: 'ms',
+  },
+);

package/src/literal.ts

@@ -0,0 +1,42 @@
+import * as RDF from '@rdfjs/types';
+import { DataFactory } from 'rdf-data-factory';
+
+const dataFactory = new DataFactory();
+
+export function filterLiteralsByLanguage(
+  literals: RDF.Literal[],
+  languages: string[],
+) {
+  const preferredLanguageLiterals = literals.filter((literal) =>
+    languages.includes(literal.language),
+  );
+  if (preferredLanguageLiterals.length > 0) {
+    return preferredLanguageLiterals;
+  }
+
+  // If literal has no language tag, we assume it is in the Network of Terms’ default language, Dutch.
+  return literals
+    .filter((literal) => literal.language === '')
+    .map((literal) => dataFactory.literal(literal.value, 'nl'));
+}
+
+/**
+ * Return value from {@link Literal} in the given languages.
+ *
+ * @param literals
+ * @param languages
+ */
+export function literalValues(
+  literals: RDF.Literal[],
+  languages: string[] = ['nl'],
+) {
+  const languageLiterals = filterLiteralsByLanguage(literals, languages);
+  if (languageLiterals.length > 0) {
+    return languageLiterals.map((literal) => literal.value);
+  }
+
+  // Fall back to English for sources that provide no Dutch labels.
+  return filterLiteralsByLanguage(literals, ['en']).map(
+    (literal) => literal.value,
+  );
+}

package/src/lookup/lookup.ts

@@ -0,0 +1,147 @@
+import { Catalog, Dataset, Distribution, IRI } from '../catalog.js';
+import {
+  Error,
+  QueryTermsService,
+  ServerError,
+  Terms,
+  TermsResponse,
+  TermsResult,
+  TimeoutError,
+} from '../query.js';
+import { Term } from '../terms.js';
+import { clientQueriesCounter } from '../instrumentation.js';
+
+export type LookupQueryResult = {
+  uri: string;
+  distribution: SourceResult;
+  result: LookupResult;
+
+  responseTimeMs: number;
+};
+
+export type SourceResult = Distribution | SourceNotFoundError;
+
+export type LookupResult = Term | NotFoundError | TimeoutError | ServerError;
+
+export class SourceNotFoundError {
+  readonly message: string;
+
+  constructor(readonly iri: string) {
+    this.message = `No source found that can provide term with URI ${iri}`;
+  }
+}
+
+export class NotFoundError {
+  readonly message: string;
+
+  constructor(readonly iri: string) {
+    this.message = `No term found with URI ${iri}`;
+  }
+}
+
+export class LookupService {
+  constructor(
+    private catalog: Catalog,
+    private queryService: QueryTermsService,
+  ) {}
+
+  public async lookup(
+    iris: string[],
+    timeoutMs: number,
+  ): Promise<LookupQueryResult[]> {
+    const irisToDataset = iris.reduce((acc, iri) => {
+      const dataset = this.catalog.getDatasetByTermIri(iri);
+      if (dataset) {
+        acc.set(iri.toString(), dataset);
+      }
+      return acc;
+    }, new Map<string, Dataset>());
+
+    const datasetToIris = [...irisToDataset].reduce(
+      (datasetMap, [iri, dataset]) => {
+        datasetMap.set(dataset, [...(datasetMap.get(dataset) ?? []), iri]);
+        return datasetMap;
+      },
+      new Map<Dataset, IRI[]>(),
+    );
+
+    const lookups = [...datasetToIris].map(([dataset]) =>
+      this.queryService.lookup(iris, dataset.distributions[0], timeoutMs),
+    );
+
+    const termsPerSource: TermsResponse[] = await Promise.all(lookups);
+
+    const datasetToTerms = termsPerSource.reduce(
+      (acc, response: TermsResponse) => {
+        let dataset = this.catalog.getDatasetByDistributionIri(
+          response.result.distribution.iri,
+        )!;
+        if (response.result instanceof Terms) {
+          const termsResult =
+            (acc.get(dataset)?.result as Terms) ??
+            new Terms(response.result.distribution, []);
+          for (const term of response.result.terms) {
+            if (term.datasetIri !== undefined) {
+              const termsDataset = this.catalog.getDatasetByIri(
+                term.datasetIri.value,
+              );
+              if (termsDataset !== undefined) {
+                dataset = termsDataset;
+                irisToDataset.set(term.id.value, dataset);
+              }
+            }
+            termsResult.terms.push(term);
+          }
+          acc.set(
+            dataset,
+            new TermsResponse(termsResult, response.responseTimeMs),
+          );
+        } else {
+          const dataset = this.catalog.getDatasetByDistributionIri(
+            response.result.distribution.iri,
+          )!;
+          acc.set(dataset, response);
+        }
+        return acc;
+      },
+      new Map<Dataset, TermsResponse>(),
+    );
+
+    return iris.map((iri) => {
+      const dataset = irisToDataset.get(iri.toString());
+      if (dataset === undefined) {
+        clientQueriesCounter.add(1, {
+          type: 'lookup',
+          error: 'SourceNotFound',
+        });
+        return {
+          uri: iri,
+          distribution: new SourceNotFoundError(iri),
+          result: new NotFoundError(iri),
+          responseTimeMs: 0,
+        };
+      }
+
+      const response = datasetToTerms.get(dataset)!;
+      clientQueriesCounter.add(1, { type: 'lookup' });
+
+      return {
+        uri: iri,
+        distribution: dataset.distributions[0],
+        result: result(response.result, iri),
+        responseTimeMs: response.responseTimeMs,
+      };
+    });
+  }
+}
+
+function result(result: TermsResult, iri: string): LookupResult {
+  if (result instanceof Error) {
+    return result;
+  }
+
+  return (
+    result.terms.find((term) => term.id.value === iri.toString()) ??
+    new NotFoundError(iri)
+  );
+}