jskos-server 2.4.0

package/utils/searchHelper.js

@@ -0,0 +1,153 @@
+import _ from "lodash"
+import jskos from "jskos-tools"
+
+// from https://web.archive.org/web/20170609122132/http://jam.sg/blog/efficient-partial-keyword-searches/
+export function makeSuffixes(values) {
+  const results = []
+  values.forEach(function (val) {
+    val = val.toUpperCase().trim()
+    let tmp, hasSuffix
+    for (let i = 0; i < val.length - 1; i++) {
+      tmp = val.substr(i)
+      hasSuffix = results.includes(tmp)
+      if (!hasSuffix) {
+        results.push(tmp)
+      }
+    }
+  })
+  return results
+}
+// adapted from above
+export function makePrefixes(values) {
+  const results = []
+  values.forEach(function (val) {
+    val = val.toUpperCase().trim()
+    let tmp, hasPrefix
+    results.push(val)
+    for (let i = 2; i < val.length; i++) {
+      tmp = val.substr(0, i)
+      hasPrefix = results.includes(tmp)
+      if (!hasPrefix) {
+        results.push(tmp)
+      }
+    }
+  })
+  return results
+}
+
+/**
+ * Enumerates and sorts all labels (prefLabel and altLabel) in an item.
+ *
+ * Sorting:
+ * - prefLabel over altLabel
+ * - provided language over other languages
+ * - other languages by the order of the `languages` property of the item if available
+ *
+ * @param {Object} item JSKOS item (scheme or concept)
+ */
+export function getAllLabelsSorted(item, language = "en") {
+  function extractAndSortLabels(labels, languages) {
+    return _.toPairs(labels).sort((a, b) => {
+      const bIndex = languages.indexOf(b[0]), aIndex = languages.indexOf(a[0])
+      if (bIndex === -1) {
+        return -1
+      }
+      if (aIndex === -1) {
+        return 1
+      }
+      return aIndex - bIndex
+    }).map(v => v[1])
+  }
+  const languages = [language].concat(item.languages || [])
+  return _.flattenDeep(extractAndSortLabels(item.prefLabel || {}, languages).concat(extractAndSortLabels(item.altLabel || {}, languages)))
+}
+
+/**
+ * Adds necessary properties required by indexes for search.
+ *
+ * @param {Object} item JSKOS item (scheme or concept)
+ */
+export function addKeywords(item) {
+  item._keywordsNotation = makePrefixes(item.notation || [])
+  // Do not write text index keywords for synthetic concepts
+  if (!item.type || !item.type.includes("http://rdf-vocabulary.ddialliance.org/xkos#CombinedConcept")) {
+    // Labels
+    // Assemble all labels
+    let labels = getAllLabelsSorted(item)
+    // Split labels by space and dash
+    item._keywordsLabels = makeSuffixes(labels)
+    // Other properties
+    item._keywordsOther = []
+    for (let map of (item.creator || []).concat(item.scopeNote, item.editorialNote, item.definition)) {
+      if (map) {
+        item._keywordsOther = item._keywordsOther.concat(Object.values(map))
+      }
+    }
+    // Make sure to flatten both arrays and objects
+    item._keywordsOther = _.flattenDeep(item._keywordsOther)
+    item._keywordsOther = item._keywordsOther.map(v => {
+      if (_.isObject(v)) {
+        return Object.values(v)
+      }
+      return v
+    })
+    item._keywordsOther = _.flattenDeep(item._keywordsOther)
+    if (item.publisher) {
+      item._keywordsPublisher = _.flattenDeep(
+        item.publisher.map(publisher => {
+          return [publisher.uri].concat(Object.values(publisher.prefLabel || {}))
+        }),
+      )
+    }
+  }
+}
+
+/**
+ * Converts search results to Open Search Suggest Format for /suggest endpoints.
+ *
+ * @param {Object} options.query query object for request
+ * @param {Array} options.results results as JSKOS array
+ */
+export function toOpenSearchSuggestFormat({ query, results }) {
+  // Transform to OpenSearch Suggest Format
+  let labels = []
+  let descriptions = []
+  let uris = []
+  let currentOffset = query.offset
+  for (let result of results) {
+    // Skip if offset is not reached
+    if (currentOffset) {
+      currentOffset -= 1
+      continue
+    }
+    // Skip if limit is reached
+    if (labels.length >= query.limit) {
+      break
+    }
+    // Determine prefLabel via `language` parameter
+    const language = (query.language || "").split(",").filter(lang => lang)
+    let prefLabel
+    for (const lang of language) {
+      prefLabel = _.get(result, `prefLabel.${lang}`)
+      if (prefLabel) {
+        break
+      }
+    }
+    if (!prefLabel) {
+      prefLabel = jskos.prefLabel(result, { fallbackToUri: false })
+    }
+    let label = jskos.notation(result)
+    if (label && prefLabel) {
+      label += " "
+    }
+    label += prefLabel
+    labels.push(label) // + " (" + result.priority + ")")
+    descriptions.push("")
+    uris.push(result.uri)
+  }
+  const searchResults = [
+    query.search || "", labels, descriptions, uris,
+  ]
+  searchResults.totalCount = results.length
+  return searchResults
+}

package/utils/status.js

@@ -0,0 +1,77 @@
+import _ from "lodash"
+
+export function serverStatus(config, ok) {
+  const { baseUrl } = config
+  const status = {
+    config: _.omit(_.cloneDeep(config), ["verbosity", "port", "mongo", "namespace", "proxies", "ips"]),
+  }
+  // Remove `ips` property from all actions
+  for (let type of ["schemes", "concepts", "mappings", "concordances", "annotations"]) {
+    if (status.config[type]) {
+      delete status.config[type].ips
+      for (let action of ["read", "create", "update", "delete"]) {
+        if (status.config[type][action]) {
+          delete status.config[type][action].ips
+        }
+      }
+    }
+  }
+  // Remove `key` from auth config if a symmetric algorithm is used
+  if (["HS256", "HS384", "HS512"].includes(status?.config?.auth?.algorithm)) {
+    delete status.config.auth.key
+  }
+  status.config.baseUrl = baseUrl
+  // Set all available endpoints to `null` first
+  for (let type of [
+    "data",
+    "schemes",
+    "top",
+    "voc-search",
+    "voc-suggest",
+    "voc-concepts",
+    "concepts",
+    "narrower",
+    "ancestors",
+    "search",
+    "suggest",
+    "mappings",
+    "concordances",
+    "annotations",
+    "registries",
+  ]) {
+    status[type] = null
+  }
+  status.data = `${baseUrl}data`
+  if (status.config.schemes) {
+    status.schemes = `${baseUrl}voc`
+    status.top = `${baseUrl}voc/top`
+    status["voc-search"] = `${baseUrl}voc/search`
+    status["voc-suggest"] = `${baseUrl}voc/suggest`
+    status["voc-concepts"] = `${baseUrl}voc/concepts`
+  }
+  if (status.config.concepts) {
+    status.concepts = `${baseUrl}concepts`
+    status.narrower = `${baseUrl}concepts/narrower`
+    status.ancestors = `${baseUrl}concepts/ancestors`
+    status.search = `${baseUrl}concepts/search`
+    status.suggest = `${baseUrl}concepts/suggest`
+  }
+  if (status.config.mappings) {
+    status.mappings = `${baseUrl}mappings`
+  }
+  if (status.config.concordances) {
+    status.concordances = `${baseUrl}concordances`
+  }
+  if (status.config.annotations) {
+    status.annotations = `${baseUrl}annotations`
+  }
+  if (status.config.registries) {
+    status.registries = `${baseUrl}registries`
+  }
+  status.types = null // not supported in jskos-server yet
+  status.validate = `${baseUrl}validate`
+
+  status.ok = ok ? 1 : 0
+
+  return status
+}

package/utils/users.js

@@ -0,0 +1,7 @@
+export const getUrisOfUser = user => {
+  if (!user) {
+    return []
+  }
+  return [user.uri].concat(Object.values(user.identities || {})
+    .map(identity => identity.uri)).filter(uri => uri)
+}

package/utils/utils.js

@@ -0,0 +1,114 @@
+import _ from "lodash"
+import * as jskos from "jskos-tools"
+
+// remove object properties when its value is null
+export function removeNullProperties(obj) {
+  return Object.keys(obj).filter(key => obj[key] === null).forEach(key => delete obj[key])
+}
+
+/**
+ * Recursively remove certain fields from response
+ *
+ * Gets called in `returnJSON` and `handleDownload`. Shouldn't be used anywhere else.
+ *
+ * @param {(Object|Object[])} json JSON object or array of objects
+ * @param {number} [depth=0] Should not be set when called from outside
+ */
+export function cleanJSON(json, depth = 0, closedWorld = true) {
+  if (Array.isArray(json)) {
+    json.forEach(value => cleanJSON(value, depth, closedWorld))
+  } else if (_.isObject(json)) {
+    _.forOwn(json, (value, key) => {
+      if (
+        // Remove top level empty arrays/objects if closedWorldAssumption is set to false
+        (depth === 0 && !closedWorld && (_.isEqual(value, {}) || _.isEqual(value, [])))
+        // Remove all fields started with _
+        || key.startsWith("_")
+      ) {
+        _.unset(json, key)
+      } else {
+        cleanJSON(value, depth + 1, closedWorld)
+      }
+    })
+  }
+}
+
+export function bulkOperationForEntities({ entities, replace = true }) {
+  return entities.map(e => (replace ? {
+    replaceOne: {
+      filter: { _id: e._id },
+      replacement: e,
+      upsert: true,
+    },
+  } : {
+    insertOne: {
+      document: e,
+    },
+  }))
+}
+
+/**
+ * Converts a MongoDB "find" query to an aggregation pipeline.
+ *
+ * In most cases, this will simply be a single $match stage, but there's special handling for
+ * $nearSquere queries on the field `location` that is converted into a $geoNear stage.
+ *
+ * @param {*} query
+ * @returns array with aggregation pipeline
+ */
+export function queryToAggregation(query) {
+  const pipeline = []
+  // Transform location $nearSphere query into $geoNear aggregation stage
+  if (query.location) {
+    const locationQuery = query.location.$nearSphere
+    pipeline.push({
+      $geoNear: {
+        spherical: true,
+        maxDistance: locationQuery.$maxDistance,
+        query: _.omit(query, ["location"]),
+        near: locationQuery.$geometry,
+        distanceField: "_distance",
+      },
+    })
+  } else {
+    pipeline.push({
+      $match: query,
+    })
+  }
+  return pipeline
+}
+
+/**
+ *
+ * @param {Object} mapping mapping to be adjusted
+ * @param {Object} [options]
+ * @param {Object} [options.concordance] concordance object of mapping
+ * @param {Object} [options.fromScheme] manual override for `fromScheme`
+ * @param {Object} [options.toScheme] manual override for `toScheme`
+ * @returns
+ */
+export function addMappingSchemes(mapping, options = {}) {
+  mapping && ["from", "to"].forEach(side => {
+    const field = `${side}Scheme`
+    if (mapping[field]) {
+      return
+    }
+    if (options[field]) {
+      mapping[field] = options[field]
+      return
+    }
+    options.concordance = options.concordance || mapping.partOf?.[0]
+    if (options.concordance?.[field]) {
+      mapping[field] = options.concordance[field]
+      return
+    }
+    const concepts = jskos.conceptsOfMapping(mapping, side)
+    const schemeFromConcept = concepts.find(concept => concept?.inScheme?.[0]?.uri)?.inScheme[0]
+    if (schemeFromConcept) {
+      mapping[field] = schemeFromConcept
+    }
+  })
+  return mapping
+}
+
+

package/utils/uuid.js

@@ -0,0 +1,6 @@
+import { v4 as uuid } from "uuid"
+
+const uuidRegex = new RegExp(/^[0-9A-F]{8}-[0-9A-F]{4}-4[0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$/i)
+const isValidUuid = id => id.match(uuidRegex) != null
+
+export { uuid, isValidUuid }

package/utils/version.js

@@ -0,0 +1,324 @@
+import { addMappingSchemes } from "./utils.js"
+import { uuid } from "./uuid.js"
+import _ from "lodash"
+import yesno from "yesno"
+
+import { Scheme, Concordance, Mapping, Annotation } from "../models/index.js"
+import { SchemeService } from "../services/schemes.js"
+import { ConcordanceService } from "../services/concordances.js"
+import { AnnotationService } from "../services/annotations.js"
+import { MappingService } from "../services/mappings.js"
+import { ConceptService } from "../services/concepts.js"
+import { RegistryService } from "../services/registries.js"
+
+import { addKeywords } from "./searchHelper.js"
+
+export class Version {
+
+  constructor(version) {
+    this.version = version
+    const [major = 0, minor = 0, patch = 0] = version.split(".").map(v => parseInt(v))
+    this.major = major
+    this.minor = minor
+    this.patch = patch
+  }
+
+  gt(version) {
+    version = Version.from(version)
+    return this.major > version.major ||
+      (this.major == version.major && this.minor > version.minor) ||
+      (this.major == version.major && this.minor == version.minor && this.patch > version.patch)
+  }
+  gte(version) {
+    version = Version.from(version)
+    return this.major > version.major ||
+      (this.major == version.major && this.minor > version.minor) ||
+      (this.major == version.major && this.minor == version.minor && this.patch >= version.patch)
+  }
+  eq(version) {
+    return !this.lt(version) && !this.gt(version)
+  }
+  lt(version) {
+    return !this.gte(version)
+  }
+  lte(version) {
+    return !this.gt(version)
+  }
+
+  static from(version) {
+    return version instanceof Version ? version : new Version(version)
+  }
+
+}
+
+export class Upgrader {
+
+  constructor(config) {
+    this.baseUrl = config.baseUrl
+    this.schemeService = new SchemeService(config)
+    this.concordanceService = new ConcordanceService(config)
+    this.annotationService = new AnnotationService(config)
+    this.mappingService = new MappingService(config)
+    this.conceptService = new ConceptService(config)
+    this.registryService = new RegistryService(config)
+  }
+
+  // Returns an array of version strings for which upgrades are necessary.
+  getUpgrades(fromVersion, { forceLatest = false }) {
+    const list = []
+    fromVersion = Version.from(fromVersion)
+
+    for (let version of Object.getOwnPropertyNames(Upgrader).filter(/^[0-9.]+$/)) {
+      if (fromVersion.lt(version) || forceLatest && fromVersion.eq(version)) {
+        list.push(version)
+      }
+    }
+    return list
+  }
+
+  // Methods that perform upgrades necessary for that version
+
+  async "1.2.0"() {
+    // 1. Additional fields for schemes (full-text search)
+    console.log("Creating additional fields for schemes...")
+    const schemes = await Scheme.find().lean()
+    for (let scheme of schemes) {
+      addKeywords(scheme)
+      await Scheme.findByIdAndUpdate(scheme._id, scheme)
+    }
+    console.log("... done.")
+    // 2. Create indexes for concordances
+    console.log("Creating indexes for concordances...")
+    await this.concordanceService.createIndexes()
+    console.log("... done.")
+    // 3. Create indexes for schemes
+    console.log("Creating indexes for schemes...")
+    await this.schemeService.createIndexes()
+    console.log("... done.")
+  }
+  async "1.2.2"() {
+    // Update text search fields for schemes (full-text search)
+    console.log("Updating text search fields for schemes...")
+    const schemes = await Scheme.find().lean()
+    for (let scheme of schemes) {
+      addKeywords(scheme)
+      // Also add modified if it doesn't exist
+      scheme.modified = scheme.modified || scheme.created
+      await Scheme.findByIdAndUpdate(scheme._id, scheme)
+    }
+    console.log("... done.")
+  }
+  async "1.2.3"() {
+    // Update text search fields for schemes (full-text search)
+    console.log("Updating text search fields for schemes...")
+    const schemes = await Scheme.find().lean()
+    for (let scheme of schemes) {
+      addKeywords(scheme)
+      await Scheme.findByIdAndUpdate(scheme._id, scheme)
+    }
+    console.log("... done.")
+  }
+  async "1.2.7"() {
+    // 1. Update publisher keywords field for schemes
+    console.log("Updating publisher keywords fields for schemes...")
+    const schemes = await Scheme.find().lean()
+    for (let scheme of schemes) {
+      addKeywords(scheme)
+      await Scheme.findByIdAndUpdate(scheme._id, scheme)
+    }
+    console.log("... done.")
+    // 2. Create indexes for schemes
+    console.log("Creating indexes for schemes...")
+    await this.schemeService.createIndexes()
+    console.log("... done.")
+  }
+  async "1.3"() {
+    console.log("Creating indexes for annotations...")
+    await this.annotationService.createIndexes()
+    console.log("... done.")
+  }
+  async "1.4"() {
+    console.log("Concordances will be upgraded:")
+    console.log("- _id will be changed to notation (if available) or a new UUID")
+    console.log(`- URI will be adjusted to start with ${this.baseUrl}`)
+    console.log("- Previous URI(s) will be moved to identifier")
+    console.log("- Distribtions which are served from this instance are removed (will be added dynamically)")
+    const ok = await yesno({
+      question: "Is that okay?",
+      defaultValue: false,
+    })
+    if (!ok) {
+      throw new Error("Aborted due to missing user confirmation.")
+    }
+
+    let adjusted = 0
+    let failed = 0
+    let skipped = 0
+    const concordances = await Concordance.find().lean()
+    for (const concordance of concordances) {
+      const previous_id = concordance._id
+      if (!previous_id.startsWith("http")) {
+        skipped += 1
+        continue
+      }
+      let _id = concordance.notation[0]
+      if (!_id) {
+        _id = uuid()
+        concordance.notation = [_id].concat((concordance.notation || []).slice(1))
+      }
+      const identifier = []
+      // Add previous _id to identifier
+      identifier.push(previous_id)
+      // Add previous URI if necessary
+      if (previous_id != concordance.uri) {
+        identifier.push(concordance.uri)
+      }
+      // Set new _id and URI, add previous to identifier
+      concordance._id = _id
+      concordance.uri = `${this.baseUrl}concordances/${_id}`
+      console.log(`- Updating concordance ${previous_id} to _id ${_id} (${concordance.uri})`)
+      concordance.identifier = (concordance.identifier || []).concat(identifier)
+      // Remove distributions that are served by jskos-server and instead add them dynamically
+      concordance.distributions = (concordance.distributions || []).filter(dist => !dist.download || !dist.download.startsWith(this.baseUrl))
+      if (!concordance.distributions.length) {
+        delete concordance.distributions
+      }
+      // Save concordance
+      try {
+        await Concordance.insertMany([concordance])
+        await Concordance.deleteOne({ _id: previous_id })
+        adjusted += 1
+      } catch (error) {
+        console.error(error)
+        failed += 1
+      }
+    }
+
+    console.log(`- Adjusted: ${adjusted}, skipped: ${skipped}, failed: ${failed}.`)
+    console.log("... done")
+    if (failed > 0) {
+      throw new Error("Not all concordances could be adjusted. Please check the errors and try again.")
+    }
+  }
+  async "1.4.5"() {
+    console.log("Upgrades to annotations (see #173):")
+
+    console.log("- Update indexes for annotations...")
+    await this.annotationService.createIndexes()
+    console.log("... done.")
+
+    console.log("- Annotations will be updated to use an object for property `target` and to include mapping state if possible...")
+    const ok = await yesno({
+      question: "Is that okay?",
+      defaultValue: false,
+    })
+    if (!ok) {
+      throw new Error("Aborted due to missing user confirmation.")
+    }
+
+    let updatedCount = 0
+    const annotations = await Annotation.find({ "target.state.id": { $exists: false } }).exec()
+    for (const annotation of annotations) {
+      const target = _.get(annotation, "target.id", annotation.target)
+      const mapping = await Mapping.findOne({ uri: target })
+      const contentId = mapping && (mapping.identifier || []).find(id => id.startsWith("urn:jskos:mapping:content:"))
+      const update = contentId ? {
+        target: {
+          id: target,
+          state: {
+            id: contentId,
+          },
+        },
+      } : {
+        target: { id: target },
+      }
+      await Annotation.updateOne({ _id: annotation._id }, update)
+      updatedCount += 1
+    }
+    console.log(`... done (${updatedCount} annotations updated).`)
+
+  }
+  async "1.5.3"() {
+    // Create indexes for mappings with index for mappingRelevance
+    console.log("Creating indexes for mappings...")
+    await this.mappingService.createIndexes()
+    console.log("... done.")
+  }
+  async "1.6.3"() {
+    // Create compound indexes for mappings to create a stable sorting order
+    console.log("Creating indexes for mappings...")
+    await this.mappingService.createIndexes()
+    console.log("... done.")
+  }
+  async "2.0.3"() {
+    console.log("Adding missing `fromScheme`/`toScheme` fields to mappings...")
+    const mappings = await Mapping.aggregate([
+      {
+        $match: { $or: [{ "fromScheme.uri": { $exists: false } }, { "toScheme.uri": { $exists: false } }] },
+      },
+      {
+        $lookup: {
+          from: Concordance.collection.name,
+          localField: "partOf.0.uri",
+          foreignField: "uri",
+          as: "CONCORDANCE",
+        },
+      },
+    ])
+    let adjustedCount = 0
+    for (const mapping of mappings) {
+      const concordance = mapping.CONCORDANCE?.[0]
+      const _id = mapping._id
+      const hasFromScheme = !!mapping.fromScheme, hasToScheme = !!mapping.toScheme
+      delete mapping._id
+      delete mapping.CONCORDANCE
+      addMappingSchemes(mapping, { concordance })
+      if (!hasFromScheme && mapping.fromScheme || !hasToScheme && mapping.toScheme) {
+        await Mapping.replaceOne({ _id }, mapping)
+        adjustedCount += 1
+      }
+    }
+    console.log(`... done (${adjustedCount} out of ${mappings.length} were adjusted).`)
+  }
+  async "2.1.0"() {
+    console.log("Creating indexes for concepts and annotations...")
+    await this.conceptService.createIndexes()
+    await this.annotationService.createIndexes()
+    console.log("... done.")
+  }
+  async "2.1.6"() {
+    console.log("Rewriting concordance URIs for mappings...")
+
+    // Find all concordances with "identifier" set
+    const concordances = await Concordance.find({
+      "identifier.0": { $exists: true },
+    })
+    console.log(`- Found ${concordances.length} concordances where updates might be necessary.`)
+
+    let updatedConcordaces = 0, updatedMappings = 0
+    for (const concordance of concordances) {
+      // Update concordance URIs
+      const result = await Mapping.updateMany({
+        "partOf.0.uri": {
+          $in: concordance.identifier,
+        },
+      }, {
+        "partOf.0.uri": concordance.uri,
+      })
+      if (result.modifiedCount) {
+        updatedConcordaces += 1
+        updatedMappings += result.modifiedCount
+      }
+    }
+    if (concordances.length) {
+      console.log(`- Updated ${updatedMappings} mappings in ${updatedConcordaces} concordances.`)
+    }
+
+    console.log("... done.")
+  }
+  async "2.4.0"() {
+    console.log("Creating index for registries...")
+    await this.registryService.createIndexes()
+    console.log("... done.")
+  }
+}