jskos-server 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/.dockerignore +20 -0
  2. package/.editorconfig +9 -0
  3. package/.github/workflows/docker.yml +59 -0
  4. package/.github/workflows/gh-pages.yml +23 -0
  5. package/.github/workflows/gh-release.yml +19 -0
  6. package/.github/workflows/test.yml +39 -0
  7. package/.husky/pre-commit +1 -0
  8. package/CHANGELOG.md +18 -0
  9. package/LICENSE +21 -0
  10. package/README.md +2710 -0
  11. package/bin/extra.js +81 -0
  12. package/bin/import.js +438 -0
  13. package/bin/mongodb.js +21 -0
  14. package/bin/reset.js +257 -0
  15. package/bin/upgrade.js +34 -0
  16. package/config/config.default.json +88 -0
  17. package/config/config.schema.json +877 -0
  18. package/config/config.test.json +107 -0
  19. package/config/index.js +77 -0
  20. package/config/setup.js +212 -0
  21. package/depdendencies.png +0 -0
  22. package/docker/.env +1 -0
  23. package/docker/Dockerfile +20 -0
  24. package/docker/README.md +175 -0
  25. package/docker/docker-compose.yml +29 -0
  26. package/docker/docker-entrypoint.sh +8 -0
  27. package/docker/mongo-initdb.d/mongo_setup.sh +22 -0
  28. package/ecosystem.example.json +7 -0
  29. package/errors/index.js +94 -0
  30. package/eslint.config.js +17 -0
  31. package/index.js +10 -0
  32. package/models/annotations.js +13 -0
  33. package/models/concepts.js +12 -0
  34. package/models/concordances.js +12 -0
  35. package/models/index.js +33 -0
  36. package/models/mappings.js +20 -0
  37. package/models/meta.js +21 -0
  38. package/models/registries.js +12 -0
  39. package/models/schemes.js +12 -0
  40. package/package.json +91 -0
  41. package/routes/annotations.js +83 -0
  42. package/routes/common.js +86 -0
  43. package/routes/concepts.js +64 -0
  44. package/routes/concordances.js +86 -0
  45. package/routes/data.js +19 -0
  46. package/routes/mappings.js +108 -0
  47. package/routes/registries.js +24 -0
  48. package/routes/schemes.js +72 -0
  49. package/routes/validate.js +37 -0
  50. package/server.js +190 -0
  51. package/services/abstract.js +328 -0
  52. package/services/annotations.js +237 -0
  53. package/services/concepts.js +459 -0
  54. package/services/concordances.js +264 -0
  55. package/services/data.js +30 -0
  56. package/services/index.js +34 -0
  57. package/services/mappings.js +978 -0
  58. package/services/registries.js +319 -0
  59. package/services/schemes.js +318 -0
  60. package/services/validate.js +39 -0
  61. package/status.schema.json +145 -0
  62. package/test/abstract-service.js +36 -0
  63. package/test/annotations/annotation.json +13 -0
  64. package/test/api.js +2481 -0
  65. package/test/chai.js +14 -0
  66. package/test/changes.js +179 -0
  67. package/test/concepts/conceptNoFileEnding +4 -0
  68. package/test/concepts/concepts-ddc-6-60-61-62.json +123 -0
  69. package/test/concordances/concordances.ndjson +2 -0
  70. package/test/config.js +26 -0
  71. package/test/configs/complex-config.json +90 -0
  72. package/test/configs/empty-object.json +1 -0
  73. package/test/configs/fail-array.json +1 -0
  74. package/test/configs/fail-empty.json +0 -0
  75. package/test/configs/fail-mapping-only-props1.json +5 -0
  76. package/test/configs/fail-mapping-only-props2.json +5 -0
  77. package/test/configs/fail-mapping-only-props3.json +5 -0
  78. package/test/configs/fail-mapping-only-props4.json +5 -0
  79. package/test/configs/fail-nonexisting-prop.json +3 -0
  80. package/test/configs/fail-port-string.json +3 -0
  81. package/test/configs/fail-registry-types.json +16 -0
  82. package/test/configs/registry-types.json +16 -0
  83. package/test/data-write.js +784 -0
  84. package/test/eslint.js +22 -0
  85. package/test/import-reset.js +287 -0
  86. package/test/infer-mappings.js +340 -0
  87. package/test/ipcheck.js +287 -0
  88. package/test/mappings/README.md +1 -0
  89. package/test/mappings/ddc-gnd-1.mapping.json +33 -0
  90. package/test/mappings/ddc-gnd-2.mapping.json +67 -0
  91. package/test/mappings/mapping-ddc-gnd-noScheme.json +145 -0
  92. package/test/mappings/mapping-ddc-gnd.json +175 -0
  93. package/test/mappings/mappings-ddc.json +214 -0
  94. package/test/registries/registries.ndjson +2 -0
  95. package/test/services.js +557 -0
  96. package/test/terminologies/terminologies.json +94 -0
  97. package/test/test-utils.js +182 -0
  98. package/test/utils.js +425 -0
  99. package/test/validate.js +226 -0
  100. package/utils/adjust.js +206 -0
  101. package/utils/auth.js +154 -0
  102. package/utils/changes.js +88 -0
  103. package/utils/db.js +106 -0
  104. package/utils/ipcheck.js +76 -0
  105. package/utils/middleware.js +636 -0
  106. package/utils/searchHelper.js +153 -0
  107. package/utils/status.js +77 -0
  108. package/utils/users.js +7 -0
  109. package/utils/utils.js +114 -0
  110. package/utils/uuid.js +6 -0
  111. package/utils/version.js +324 -0
  112. package/views/base.ejs +172 -0
@@ -0,0 +1,978 @@
1
+ import _ from "lodash"
2
+ import { uuid, isValidUuid } from "../utils/uuid.js"
3
+ import { removeNullProperties, addMappingSchemes } from "../utils/utils.js"
4
+ import jskos from "jskos-tools"
5
+ import { validate } from "jskos-validate"
6
+ import { cdk } from "cocoda-sdk"
7
+
8
+ import { Mapping } from "../models/mappings.js"
9
+ import { Annotation } from "../models/annotations.js"
10
+ import { SchemeService } from "./schemes.js"
11
+ import { ConcordanceService } from "./concordances.js"
12
+ import { MalformedRequestError, EntityNotFoundError, InvalidBodyError, DatabaseAccessError, BackendError } from "../errors/index.js"
13
+
14
+ const validateMapping = (mapping) => {
15
+ const valid = validate.mapping(mapping)
16
+ if (!valid) {
17
+ return false
18
+ }
19
+ // Reject mappings without concepts in `from`
20
+ if (jskos.conceptsOfMapping(mapping, "from").length === 0) {
21
+ return false
22
+ }
23
+ return true
24
+ }
25
+
26
+ import { AbstractService } from "./abstract.js"
27
+
28
+ export class MappingService extends AbstractService {
29
+
30
+ constructor(config) {
31
+ super(config)
32
+ this.baseUri = config.baseUrl + "mappings/"
33
+ this.config = config.mappings || {}
34
+ this.model = Mapping
35
+ this.schemeService = new SchemeService(config)
36
+ this.concordanceService = new ConcordanceService(config)
37
+ this.loadWhitelists()
38
+ }
39
+
40
+ /**
41
+ * Loads all schemes from whitelists (if they exists) from the database.
42
+ */
43
+ async loadWhitelists() {
44
+ // Load schemes from fromSchemeWhitelist and toSchemeWhitelist
45
+ for (let type of ["fromSchemeWhitelist", "toSchemeWhitelist"]) {
46
+ let whitelist = []
47
+ for (let scheme of this.config[type] || []) {
48
+ scheme = (await this.schemeService.getScheme(scheme.uri)) || scheme
49
+ whitelist.push(scheme)
50
+ }
51
+ if (whitelist.length) {
52
+ this[type] = whitelist
53
+ }
54
+ }
55
+ }
56
+
57
+ /**
58
+ * Checks a mapping againgst scheme whitelists and throws an error if it doesn't match.
59
+ *
60
+ * @param {*} mapping
61
+ */
62
+ checkWhitelists(mapping) {
63
+ for (let type of ["fromScheme", "toScheme"]) {
64
+ const whitelist = this[`${type}Whitelist`]
65
+ const scheme = mapping[type]
66
+ if (whitelist && scheme) {
67
+ if (!whitelist.find(s => jskos.compare(s, scheme))) {
68
+ throw new InvalidBodyError(`Value in ${type} is not allowed.`)
69
+ }
70
+ }
71
+ }
72
+ }
73
+
74
+ async queryItems({ uri, identifier, from, to, fromScheme, toScheme, mode, direction, type, partOf, creator, sort, order, limit, offset, download, annotatedWith, annotatedFor, annotatedBy, cardinality }) {
75
+ direction = direction || "forward"
76
+
77
+ let count = 0
78
+ let fromTo = fromTo => {
79
+ let result
80
+ if (count == 1) {
81
+ result = fromTo == "from" && direction == "backward" || fromTo == "to" && direction != "backward" ? "to" : "from"
82
+ } else if (count <= 0) {
83
+ return null
84
+ } else {
85
+ result = fromTo == "from" ? "to" : "from"
86
+ }
87
+ return result
88
+ }
89
+ // Converts value to regex object for MongoDB if it ends with `*`.
90
+ // Currently only supports truncated search like that, no arbitrary regex possible.
91
+ const regex = value => {
92
+ if (value.endsWith("*")) {
93
+ return { $regex: `^${_.escapeRegExp(value.substring(0, value.length - 1))}` }
94
+ } else {
95
+ return value
96
+ }
97
+ }
98
+
99
+ // Note that mode is only applied to from(Scheme), to(Scheme), uri, and identifier
100
+ if (!["and", "or"].includes(mode)) {
101
+ mode = "and"
102
+ }
103
+
104
+ // Prepare fromScheme / toScheme
105
+ let fromToScheme = { fromScheme, toScheme }
106
+ for (let part of ["fromScheme", "toScheme"]) {
107
+ // Replace query.fromScheme and query.toScheme with array of URIs
108
+ if (fromToScheme[part]) {
109
+ // load scheme from database
110
+ let searchStrings = fromToScheme[part].split("|")
111
+ let allUris = []
112
+ for (let search of searchStrings) {
113
+ let scheme = await this.schemeService.getScheme(search)
114
+ let uris
115
+ if (!scheme) {
116
+ uris = [search]
117
+ } else {
118
+ uris = [scheme.uri].concat(scheme.identifier || [])
119
+ }
120
+ allUris = allUris.concat(uris)
121
+ }
122
+ fromToScheme[part] = allUris.length ? allUris : null
123
+ }
124
+ }
125
+
126
+ // Handle from/fromScheme/to/toScheme here
127
+ let criteria = []
128
+ let or = [], orEfficientFirstStage = []
129
+ count = direction == "both" ? 2 : 1
130
+ while (count > 0) {
131
+ let current = []
132
+ for (const part of ["from", "to"]) {
133
+ const conceptOr = [], schemeOr = []
134
+ // Depending on `count` and `direction`, the value of `side` will either be "from" or "to"
135
+ const side = fromTo(part)
136
+ // Deal with concepts
137
+ for (let searchString of ({ from, to }[part] || "").split("|").filter(Boolean)) {
138
+ conceptOr.push({
139
+ [`${side}.memberSet.uri`]: regex(searchString),
140
+ })
141
+ conceptOr.push({
142
+ [`${side}.memberChoice.uri`]: regex(searchString),
143
+ })
144
+ conceptOr.push({
145
+ [`${side}.memberSet.notation`]: regex(searchString),
146
+ })
147
+ conceptOr.push({
148
+ [`${side}.memberChoice.notation`]: regex(searchString),
149
+ })
150
+ }
151
+ // Deal with schemes
152
+ for (let uri of fromToScheme[part + "Scheme"] || []) {
153
+ schemeOr.push({ [`${side}Scheme.uri`]: uri })
154
+ schemeOr.push({ [`${side}Scheme.notation`]: uri })
155
+ }
156
+ if (conceptOr.length && schemeOr.length) {
157
+ // Concept and scheme from same side are always connected via "and'
158
+ current.push({
159
+ $and: [
160
+ { $or: conceptOr },
161
+ { $or: schemeOr },
162
+ ],
163
+ })
164
+ } else if (conceptOr.length) {
165
+ current.push({ $or: conceptOr })
166
+ } else if (schemeOr.length) {
167
+ current.push({ $or: schemeOr })
168
+ }
169
+ if (conceptOr.length) {
170
+ // Here we're implementing a first stage where only concept URIs/notations are matched, because we know that
171
+ // it is a lot more specific and can speed up the query significantly.
172
+ orEfficientFirstStage = orEfficientFirstStage.concat(conceptOr)
173
+ }
174
+ }
175
+ if (current.length) {
176
+ // Add current conditions depending on `mode`
177
+ or.push({ [`$${mode}`]: current })
178
+ }
179
+ count -= 1
180
+ }
181
+ // Only add the efficiency first stage for mode "and"
182
+ if (orEfficientFirstStage.length && mode === "and") {
183
+ criteria.push({ $or: orEfficientFirstStage })
184
+ }
185
+ if (or.length) {
186
+ criteria.push({ $or: or })
187
+ }
188
+ if (identifier) {
189
+ // Add identifier to criteria
190
+ criteria.push({ $or: identifier.split("|").map(id => ({ $or: [{ identifier: id }, { uri: id }] })) })
191
+ }
192
+ if (uri) {
193
+ // Add URI to criteria
194
+ criteria.push({ $or: uri.split("|").map(id => ({ uri: id })) })
195
+ }
196
+
197
+ let mongoQuery1 = criteria.length ? { [`$${mode}`]: criteria } : {}
198
+
199
+ // Type
200
+ criteria = []
201
+ if (type) {
202
+ for (let t of type.split("|")) {
203
+ criteria.push({
204
+ type: t,
205
+ })
206
+ // FIXME: Replace with default type from jskos-tools (does not exist yet).
207
+ if (t == "http://www.w3.org/2004/02/skos/core#mappingRelation") {
208
+ criteria.push({
209
+ type: { $exists: false },
210
+ })
211
+ }
212
+ }
213
+ }
214
+ let mongoQuery3 = criteria.length ? { $or: criteria } : {}
215
+
216
+ // Concordances
217
+ let mongoQuery4 = {}
218
+ if (partOf) {
219
+ if (partOf === "any") {
220
+ // Mapping is part of any concordance
221
+ mongoQuery4 = { "partOf.0": { $exists: true } }
222
+ } else if (partOf === "none") {
223
+ // Mapping is part of no concordance
224
+ mongoQuery4 = { "partOf.0": { $exists: false } }
225
+ } else {
226
+ let uris = partOf.split("|")
227
+ let allUris = []
228
+ for (const uri of uris) {
229
+ // Get concordance from database, then add all its identifiers
230
+ try {
231
+ const concordance = await this.concordanceService.getItem(uri)
232
+ allUris = allUris.concat(concordance.uri, concordance.identifier || [])
233
+ } catch (error) {
234
+ // Ignore error and push URI only
235
+ allUris.push(uri)
236
+ }
237
+ }
238
+ mongoQuery4 = {
239
+ $or: allUris.map(uri => ({ "partOf.uri": uri })),
240
+ }
241
+ }
242
+ }
243
+
244
+ // Concordances
245
+ let mongoQuery5 = {}
246
+ if (creator) {
247
+ let creators = creator.split("|")
248
+ mongoQuery5 = {
249
+ $or: _.flatten(creators.map(creator => [
250
+ jskos.isValidUri(creator) ? null : { "creator.prefLabel.de": new RegExp(_.escapeRegExp(creator), "i") },
251
+ jskos.isValidUri(creator) ? null : { "creator.prefLabel.en": new RegExp(_.escapeRegExp(creator), "i") },
252
+ jskos.isValidUri(creator) ? { "creator.uri": creator } : null,
253
+ ].filter(Boolean))),
254
+ }
255
+ }
256
+
257
+ // Cardinality
258
+ let mongoQuery6 = {}
259
+ if (cardinality === "1-to-1") {
260
+ mongoQuery6 = { "to.memberSet.1": { $exists: false } }
261
+ }
262
+
263
+ const query = { $and: [mongoQuery1, mongoQuery3, mongoQuery4, mongoQuery5, mongoQuery6] }
264
+
265
+ // Sorting (default: modified descending)
266
+ sort = ["created", "modified", "mappingRelevance"].includes(sort) ? sort : "modified"
267
+ order = order == "asc" ? 1 : -1
268
+ // Currently default sort by modified descending
269
+ const sorting = { [sort]: order, "from.memberSet.uri": order, _id: order }
270
+
271
+ // Annotation assertions need special handling (see #176)
272
+ const isNegativeAnnotationAssertion = (annotatedFor) => annotatedFor === "none" || (annotatedFor || "").startsWith("!")
273
+
274
+ // Handle restrictions on sum of assessment annotations in `annotatedWith`
275
+ let assessmentSumQuery, assessmentSumMatch
276
+ if (annotatedWith && (!annotatedFor || annotatedFor === "assessing") && (assessmentSumMatch = annotatedWith.match(/^([<>]?)(=?)(-?\d+)$/)) && (assessmentSumMatch[1] || assessmentSumMatch[2])) {
277
+
278
+ // Parameter `from` or `to` is required to use sum of assessment annotations
279
+ if (!from && !to) {
280
+ // Do nothing here; annotatedWith parameter will be completely ignored
281
+ } else if (assessmentSumMatch[1]) {
282
+ // > or <
283
+ assessmentSumQuery = {
284
+ _assessmentSum: { [`$${{ "<": "lt", ">": "gt" }[assessmentSumMatch[1]]}${{ "=": "e", "": "" }[assessmentSumMatch[2]]}`]: parseInt(assessmentSumMatch[3]) },
285
+ }
286
+ } else {
287
+ assessmentSumQuery = {
288
+ _assessmentSum: parseInt(assessmentSumMatch[3]),
289
+ }
290
+ }
291
+
292
+ annotatedWith = null
293
+ }
294
+
295
+ const buildAnnotationQuery = ({ annotatedWith, annotatedFor, annotatedBy, prefix = "" }) => {
296
+ const annotationQuery = {}
297
+ if (annotatedWith) {
298
+ annotationQuery[prefix + "bodyValue"] = annotatedWith
299
+ }
300
+ if (annotatedFor) {
301
+ let annotatedForQuery = annotatedFor
302
+ if (annotatedFor === "none") {
303
+ annotatedForQuery = { $exists: false }
304
+ } else if (annotatedFor === "any") {
305
+ annotatedForQuery = { $exists: true }
306
+ } else if (annotatedFor.startsWith("!")) {
307
+ annotatedForQuery = { $ne: annotatedFor.slice(1) }
308
+ }
309
+ annotationQuery[prefix + "motivation"] = annotatedForQuery
310
+ }
311
+ if (annotatedBy) {
312
+ annotationQuery[prefix + "creator.id"] = { $in: annotatedBy.split("|") }
313
+ }
314
+ return annotationQuery
315
+ }
316
+
317
+ const buildPipeline = ({ query, sorting, annotatedWith, annotatedBy, annotatedFor }) => {
318
+ let pipeline = []
319
+ const negativeAnnotationAssertion = isNegativeAnnotationAssertion(annotatedFor)
320
+
321
+ // Filter by annotations
322
+ // Three different paths
323
+ if (!annotatedWith && !annotatedBy && !annotatedFor && !assessmentSumQuery) {
324
+ // 1. No filter by annotations
325
+ // Simply match mapping query
326
+ pipeline = [
327
+ { $match: query },
328
+ ...(sorting ? [{ $sort: sorting }] : []),
329
+ ]
330
+ pipeline.model = Mapping
331
+ } else if (from || to || creator || negativeAnnotationAssertion) {
332
+ // 2. Filter by annotation, and from/to/creator is defined
333
+ // We'll first filter the mappings, then add annotations and filter by those
334
+ const annotationQuery = buildAnnotationQuery({ annotatedWith, annotatedFor, annotatedBy, prefix: "annotations." })
335
+ pipeline = [
336
+ { $match: query },
337
+ ...(sorting ? [{ $sort: sorting }] : []),
338
+ {
339
+ $lookup: {
340
+ from: "annotations",
341
+ localField: "uri",
342
+ foreignField: "target.id",
343
+ as: "annotations",
344
+ },
345
+ },
346
+ {
347
+ $match: annotationQuery,
348
+ },
349
+ // Deal with assessmentSumQuery here
350
+ ...(assessmentSumQuery ? [
351
+ // 1. Calculate assessment sum
352
+ {
353
+ $set: {
354
+ _assessmentSum: {
355
+ $function: {
356
+ lang: "js",
357
+ args: ["$annotations"],
358
+ body: function (annotations) {
359
+ return annotations.reduce((prev, cur) => {
360
+ if (cur.motivation === "assessing") {
361
+ if (cur.bodyValue === "+1") {
362
+ return prev + 1
363
+ }
364
+ if (cur.bodyValue === "-1") {
365
+ return prev - 1
366
+ }
367
+ }
368
+ return prev
369
+ }, 0)
370
+ },
371
+ },
372
+ },
373
+ },
374
+ },
375
+ // 2. Add query
376
+ { $match: assessmentSumQuery },
377
+ ] : []),
378
+ { $project: { annotations: 0, _assessmentSum: 0 } },
379
+ ]
380
+ pipeline.model = this.model
381
+ } else {
382
+ // 3. Filter by annotation, and none of the properties is given
383
+ // We'll first filter the annotations, then get the associated mappings, remove duplicates, and filter those
384
+ const annotationQuery = buildAnnotationQuery({ annotatedWith, annotatedFor, annotatedBy })
385
+ pipeline = [
386
+ // First, match annotations
387
+ {
388
+ $match: annotationQuery,
389
+ },
390
+ // Get mappings for annotations
391
+ {
392
+ $lookup: {
393
+ from: "mappings",
394
+ localField: "target.id",
395
+ foreignField: "uri",
396
+ as: "mappings",
397
+ },
398
+ },
399
+ // Unwind and replace root
400
+ { $unwind: "$mappings" },
401
+ { $replaceRoot: { newRoot: "$mappings" } },
402
+ // Filter duplicates by grouping by _id and getting only the first element
403
+ { $group: { _id: "$_id", data: { $push: "$$ROOT" } } },
404
+ { $addFields: { mapping: { $arrayElemAt: ["$data", 0] } } },
405
+ // Replace root with mapping
406
+ { $replaceRoot: { newRoot: "$mapping" } },
407
+ // Sort
408
+ ...(sorting ? [{ $sort: sorting }] : []),
409
+ // Match mappings
410
+ { $match: query },
411
+ ]
412
+ pipeline.model = Annotation
413
+ }
414
+ return pipeline
415
+ }
416
+
417
+ const pipeline = buildPipeline({ query, sorting, annotatedWith, annotatedBy, annotatedFor })
418
+ const negativeAnnotationAssertion = isNegativeAnnotationAssertion(annotatedFor)
419
+
420
+ if (download) {
421
+ // For a download, return a stream
422
+ return pipeline.model.aggregate(pipeline).cursor()
423
+ } else {
424
+ // Otherwise, return results
425
+ const normalizedPagination = this._getLimitAndOffset({ limit, offset })
426
+ const mappings = await pipeline.model.aggregate(pipeline.concat({ $skip: normalizedPagination.offset }, { $limit: normalizedPagination.limit }), { allowDiskUse: true }).exec()
427
+ // Handle negative annotation assertions differently because counting is inefficient
428
+ if (negativeAnnotationAssertion) {
429
+ // Instead, count by building a pipeline without `annotatedFor`, then another pipeline with the opposite `annotatedFor`, count for both and calculate the difference
430
+ const totalCountPipeline = buildPipeline({ query, annotatedWith, annotatedBy })
431
+ const oppositeCountPipeline = buildPipeline({ query, annotatedWith, annotatedBy, annotatedFor: annotatedFor === "none" ? "any" : annotatedFor.slice(1) })
432
+ mappings.totalCount = await this._count(totalCountPipeline.model, totalCountPipeline) - await this._count(oppositeCountPipeline.model, oppositeCountPipeline)
433
+ } else {
434
+ mappings.totalCount = await this._count(pipeline.model, pipeline.filter(p => !p.$sort))
435
+ }
436
+ return mappings
437
+ }
438
+ }
439
+
440
+ async getItem(_id) {
441
+ return this.getMapping(_id)
442
+ }
443
+
444
+ /**
445
+ * Returns a promise with a single mapping with ObjectId in req.params._id.
446
+ */
447
+ async getMapping(_id) {
448
+ if (!_id) {
449
+ throw new MalformedRequestError()
450
+ }
451
+ const result = await this.model.findById(_id).lean()
452
+ if (!result) {
453
+ throw new EntityNotFoundError(null, _id)
454
+ }
455
+ return result
456
+ }
457
+
458
+ /**
459
+ * Infer mappings based on the source concept's ancestors. (see https://github.com/gbv/jskos-server/issues/177)
460
+ */
461
+ async inferMappings({ strict, depth, ...query }) {
462
+ if (query.to) {
463
+ // `to` parameter not supported
464
+ throw new MalformedRequestError("Query parameter \"to\" is not supported in /mappings/infer.")
465
+ }
466
+
467
+ // Remove unsupported query parameters
468
+ delete query.cardinality
469
+ query.cardinality = "1-to-1"
470
+ delete query.download
471
+
472
+ if (query.direction && query.direction !== "forward") {
473
+ throw new MalformedRequestError("Only direction \"forward\" is supported in /mappings/infer.")
474
+ }
475
+ let { from, fromScheme, type } = query
476
+
477
+ // Do not continue with empty `from` parameter
478
+ if (!from || !fromScheme) {
479
+ return []
480
+ }
481
+
482
+ // Try queryItems first; return if there are results
483
+ let mappings = await this.queryItems(query)
484
+ if (mappings.length) {
485
+ return mappings
486
+ }
487
+
488
+ strict = ["true", "1"].includes(strict) ? true : false
489
+ depth = parseInt(depth)
490
+ depth = (isNaN(depth) || depth < 0) ? null : depth
491
+
492
+ if (depth === 0) {
493
+ return []
494
+ }
495
+
496
+ fromScheme = await this.schemeService.getScheme(fromScheme)
497
+ try {
498
+ const registry = fromScheme && cdk.registryForScheme(fromScheme)
499
+ registry && await registry.init()
500
+ // If fromScheme is not found or has no JSKOS API, return empty result
501
+ if (!fromScheme || !registry || !registry.has.ancestors) {
502
+ return []
503
+ }
504
+ fromScheme = new jskos.ConceptScheme(fromScheme)
505
+
506
+ // Build URI from notation if necessary
507
+ if (!jskos.isValidUri(from)) {
508
+ from = fromScheme.uriFromNotation(from)
509
+ if (!from) {
510
+ return []
511
+ }
512
+ }
513
+
514
+ // Build new type set
515
+ type = (type || "").split("|").filter(Boolean)
516
+ const types = []
517
+ if (type.includes("http://www.w3.org/2004/02/skos/core#mappingRelation") || type.length === 0) {
518
+ types.push("http://www.w3.org/2004/02/skos/core#mappingRelation")
519
+ }
520
+ if (type.includes("http://www.w3.org/2004/02/skos/core#narrowMatch") || type.length === 0) {
521
+ types.push("http://www.w3.org/2004/02/skos/core#exactMatch")
522
+ types.push("http://www.w3.org/2004/02/skos/core#narrowMatch")
523
+ if (!strict) {
524
+ types.push("http://www.w3.org/2004/02/skos/core#closeMatch")
525
+ }
526
+ }
527
+ if (type.includes("http://www.w3.org/2004/02/skos/core#relatedMatch") || type.length === 0) {
528
+ types.push("http://www.w3.org/2004/02/skos/core#relatedMatch")
529
+ }
530
+
531
+ // If there are no types in new type set, return empty result
532
+ if (types.length === 0) {
533
+ return []
534
+ }
535
+
536
+ // Retrieve ancestors from API
537
+ let ancestors = await registry.getAncestors({ concept: { uri: from } })
538
+ if (depth !== null) {
539
+ ancestors = ancestors.slice(0, depth)
540
+ }
541
+ for (const uri of ancestors.map(a => a && a.uri).filter(Boolean)) {
542
+ mappings = await this.queryItems(Object.assign({}, query, { from: uri, type: types.join("|") }))
543
+ if (mappings.length) {
544
+ return mappings.map(m => {
545
+ const mapping = {
546
+ from: {},
547
+ fromScheme: m.fromScheme,
548
+ to: m.to,
549
+ toScheme: m.toScheme,
550
+ }
551
+ if (m.uri) {
552
+ mapping.source = [{
553
+ uri: m.uri,
554
+ creator: m.creator,
555
+ created: m.created,
556
+ modified: m.modified,
557
+ from: m.from,
558
+ type: m.type,
559
+ }]
560
+ }
561
+ const fromConcept = {
562
+ uri: from,
563
+ }
564
+ const notation = fromScheme.notationFromUri(from)
565
+ if (notation) {
566
+ fromConcept.notation = [notation]
567
+ }
568
+ mapping.from.memberSet = [fromConcept]
569
+ const type = m?.type?.[0] || "http://www.w3.org/2004/02/skos/core#mappingRelation"
570
+ if (type === "http://www.w3.org/2004/02/skos/core#exactMatch" || !strict && type === "http://www.w3.org/2004/02/skos/core#closeMatch") {
571
+ mapping.type = ["http://www.w3.org/2004/02/skos/core#narrowMatch"]
572
+ } else {
573
+ mapping.type = [type]
574
+ }
575
+ return jskos.addMappingIdentifiers(mapping)
576
+ })
577
+ }
578
+ }
579
+ } catch (error) {
580
+ // This mainly catches errors related to the API requests for ancestors and mappings
581
+ throw new BackendError(`There was an error retrieving ancestors for concept ${from}: ${error.message}`)
582
+ }
583
+
584
+ return []
585
+
586
+ }
587
+
588
+ async prepareAndCheckItemForAction(mapping, action, { bulk }) {
589
+ if (action !== "create") {
590
+ return mapping
591
+ }
592
+
593
+ // Add created and modified dates.
594
+ const now = (new Date()).toISOString()
595
+ if (!bulk || !mapping.created) {
596
+ mapping.created = now
597
+ }
598
+ mapping.modified = now
599
+ // Validate mapping
600
+ if (!validateMapping(mapping)) {
601
+ throw new InvalidBodyError()
602
+ }
603
+ if (mapping.partOf) {
604
+ throw new InvalidBodyError("Property `partOf` is currently not allowed.")
605
+ }
606
+ // Check cardinality for 1-to-1
607
+ if (this.config.cardinality == "1-to-1" && jskos.conceptsOfMapping(mapping, "to").length > 1) {
608
+ throw new InvalidBodyError("Only 1-to-1 items are supported.")
609
+ }
610
+ // Add mapping schemes if necessary (e.g. from concepts' `inScheme` property)
611
+ addMappingSchemes(mapping)
612
+ // Check if schemes are available and replace them with URI/notation only
613
+ await this.schemeService.replaceSchemeProperties(mapping, ["fromScheme", "toScheme"])
614
+ // Reject mapping if either fromScheme or toScheme is missing
615
+ for (let field of ["fromScheme", "toScheme"]) {
616
+ if (!mapping[field]) {
617
+ throw new InvalidBodyError(`Property \`${field}\` is missing.`)
618
+ }
619
+ }
620
+ this.checkWhitelists(mapping)
621
+ // _id and URI
622
+ delete mapping._id
623
+ if (mapping.uri) {
624
+ let uri = mapping.uri
625
+ // URI already exists, use if it's valid, otherwise move to identifier
626
+ if (uri.startsWith(this.baseUri) && isValidUuid(uri.slice(this.baseUri.length, uri.length))) {
627
+ mapping._id = uri.slice(this.baseUri.length, uri.length)
628
+ } else {
629
+ mapping.identifier = (mapping.identifier || []).concat([uri])
630
+ }
631
+ }
632
+ if (!mapping._id) {
633
+ mapping._id = uuid()
634
+ mapping.uri = this.baseUri + mapping._id
635
+ }
636
+ // Make sure URI is a https URI when in production
637
+ if (this.config.env === "production") {
638
+ mapping.uri = mapping.uri.replace("http:", "https:")
639
+ }
640
+ // Set mapping identifier
641
+ mapping.identifier = jskos.addMappingIdentifiers(mapping).identifier
642
+ // Set mapping type to mappingRelation if not set
643
+ if (!mapping.type || !mapping.type.length) {
644
+ mapping.type = ["http://www.w3.org/2004/02/skos/core#mappingRelation"]
645
+ }
646
+
647
+ return mapping
648
+ }
649
+
650
+ async putMapping({ body, existing }) {
651
+ let mapping = body
652
+ if (!mapping) {
653
+ throw new InvalidBodyError()
654
+ }
655
+ // Add modified date.
656
+ mapping.modified = (new Date()).toISOString()
657
+ // Validate mapping
658
+ if (!validateMapping(mapping)) {
659
+ throw new InvalidBodyError()
660
+ }
661
+ if (this.config.cardinality == "1-to-1" && jskos.conceptsOfMapping(mapping, "to").length > 1) {
662
+ throw new InvalidBodyError("Only 1-to-1 mappings are supported.")
663
+ }
664
+ // If it's part of a concordance, don't allow changing fromScheme/toScheme
665
+ if (existing.partOf && existing.partOf.length) {
666
+ if (!jskos.compare(existing.fromScheme, mapping.fromScheme) || !jskos.compare(existing.toScheme, mapping.toScheme)) {
667
+ throw new InvalidBodyError("Can't change fromScheme/toScheme on a mapping that belongs to a concordance.")
668
+ }
669
+ }
670
+ this.checkWhitelists(mapping)
671
+
672
+ // Override _id, uri, and created properties
673
+ mapping._id = existing._id
674
+ mapping.uri = existing.uri
675
+ mapping.created = existing.created
676
+ // Set mapping identifier
677
+ mapping.identifier = jskos.addMappingIdentifiers(mapping).identifier
678
+ // Set mapping type to mappingRelation if not set
679
+ if (!mapping.type || !mapping.type.length) {
680
+ mapping.type = ["http://www.w3.org/2004/02/skos/core#mappingRelation"]
681
+ }
682
+
683
+ const result = await this.model.replaceOne({ _id: existing._id }, mapping)
684
+ if (result.acknowledged && result.matchedCount) {
685
+ // Update concordances if necessary
686
+ if (existing.partOf && existing.partOf[0]) {
687
+ await this.concordanceService.postAdjustmentForConcordance(existing.partOf[0].uri)
688
+ }
689
+ if (body.partOf && body.partOf[0]) {
690
+ await this.concordanceService.postAdjustmentForConcordance(body.partOf[0].uri)
691
+ }
692
+ return mapping
693
+ } else {
694
+ throw new DatabaseAccessError()
695
+ }
696
+ }
697
+
698
+ async patchMapping({ body, existing }) {
699
+ let mapping = body
700
+ if (!mapping) {
701
+ throw new InvalidBodyError()
702
+ }
703
+
704
+ for (let key of ["_id", "uri", "created"]) {
705
+ delete mapping[key]
706
+ }
707
+ // Remove creator/contributor if there are no changes
708
+ // TODO: Possibly check this is handleCreatorForObject
709
+ if (mapping.creator && _.isEqual(mapping.creator, existing.creator)) {
710
+ delete mapping.creator
711
+ }
712
+ if (mapping.contributor && _.isEqual(mapping.contributor, existing.contributor)) {
713
+ delete mapping.contributor
714
+ }
715
+ // Add modified date, except if only updating `partOf`
716
+ const keys = Object.keys(mapping)
717
+ if (keys.length === 1 && keys[0] === "partOf") {
718
+ delete mapping.modified
719
+ } else {
720
+ mapping.modified = (new Date()).toISOString()
721
+ }
722
+ // If it's part of a concordance, don't allow changing fromScheme/toScheme
723
+ if (existing.partOf && existing.partOf.length) {
724
+ if (mapping.fromScheme && !jskos.compare(existing.fromScheme, mapping.fromScheme) || mapping.toScheme && !jskos.compare(existing.toScheme, mapping.toScheme)) {
725
+ throw new InvalidBodyError("Can't change fromScheme/toScheme on a mapping that belongs to a concordance.")
726
+ }
727
+ }
728
+ // Merge mappings
729
+ const newMapping = Object.assign({}, existing, mapping)
730
+ // Set mapping identifier
731
+ newMapping.identifier = jskos.addMappingIdentifiers(newMapping).identifier
732
+ // Set mapping type to mappingRelation if not set
733
+ if (!mapping.type || !mapping.type.length) {
734
+ mapping.type = ["http://www.w3.org/2004/02/skos/core#mappingRelation"]
735
+ }
736
+ removeNullProperties(newMapping)
737
+
738
+ // Validate mapping after merge
739
+ if (!validateMapping(newMapping)) {
740
+ throw new InvalidBodyError()
741
+ }
742
+ if (this.config.cardinality == "1-to-1" && jskos.conceptsOfMapping(mapping, "to").length > 1) {
743
+ throw new InvalidBodyError("Only 1-to-1 mappings are supported.")
744
+ }
745
+ this.checkWhitelists(mapping)
746
+
747
+ const result = await this.model.replaceOne({ _id: newMapping._id }, newMapping)
748
+ if (result.acknowledged) {
749
+ // Update concordances if necessary
750
+ if (existing.partOf && existing.partOf[0]) {
751
+ await this.concordanceService.postAdjustmentForConcordance(existing.partOf[0].uri)
752
+ }
753
+ if (body.partOf && body.partOf[0]) {
754
+ await this.concordanceService.postAdjustmentForConcordance(body.partOf[0].uri)
755
+ }
756
+ return newMapping
757
+ } else {
758
+ throw new DatabaseAccessError()
759
+ }
760
+ }
761
+
762
+ async deleteItem({ existing }) {
763
+ super.deleteItem({ existing })
764
+
765
+ // Update concordance if necessary
766
+ if (existing.partOf && existing.partOf[0]) {
767
+ await this.concordanceService.postAdjustmentForConcordance(existing.partOf[0].uri)
768
+ }
769
+ }
770
+
771
+ /**
772
+ * Returns a promise with an array of concept schemes.
773
+ */
774
+ async getMappingSchemes(query) {
775
+ // TODO: Optimize MongoDB queries
776
+ let match = []
777
+ if (query.from) {
778
+ match.push({
779
+ $or: [{
780
+ "from.memberSet.uri": query.from,
781
+ }, {
782
+ "from.memberSet.notation": query.from,
783
+ }],
784
+ })
785
+ }
786
+ if (query.to) {
787
+ match.push({
788
+ $or: [{
789
+ "to.memberSet.uri": query.to,
790
+ }, {
791
+ "to.memberSet.notation": query.to,
792
+ }, {
793
+ "to.memberChoice.uri": query.to,
794
+ }, {
795
+ "to.memberChoice.notation": query.to,
796
+ }],
797
+ })
798
+ }
799
+ if (!match.length) {
800
+ match = [{}]
801
+ }
802
+ let mode = query.mode
803
+ if (!["and", "or"].includes(mode)) {
804
+ // default: $or
805
+ mode = "or"
806
+ }
807
+ match = { $match: { [`$${mode}`]: match } }
808
+ let promises = [
809
+ this.model.aggregate([
810
+ match,
811
+ {
812
+ $group: {
813
+ _id: "$fromScheme",
814
+ count: { $sum: 1 },
815
+ },
816
+ },
817
+ ]).exec(),
818
+ this.model.aggregate([
819
+ match,
820
+ {
821
+ $group: {
822
+ _id: "$toScheme",
823
+ count: { $sum: 1 },
824
+ },
825
+ },
826
+ ]).exec(),
827
+ ]
828
+ let schemes = {}
829
+ let results = await Promise.all(promises)
830
+ for (let result of results[0]) {
831
+ // fromScheme counts
832
+ schemes[result._id.uri] = result._id
833
+ schemes[result._id.uri].fromCount = parseInt(result.count) || 0
834
+ }
835
+ for (let result of results[1]) {
836
+ // toScheme counts
837
+ if (!result._id || !result._id.uri) {
838
+ continue
839
+ }
840
+ schemes[result._id.uri] = schemes[result._id.uri] || result._id
841
+ schemes[result._id.uri].toCount = parseInt(result.count) || 0
842
+ }
843
+ const toBeReturned = Object.values(schemes).slice(query.offset, query.offset + query.limit)
844
+ toBeReturned.totalCount = Object.values(schemes).length
845
+ return toBeReturned
846
+ }
847
+
848
+ /**
849
+ * Returns a promise with suggestions in OpenSearch Suggest Format
850
+ */
851
+ async getNotationSuggestions(query) {
852
+ let search = query.search
853
+ if (!search || search.length == 0) {
854
+ return ["", [], [], []]
855
+ }
856
+ let vocs = query.voc && query.voc.split("|")
857
+ let mongoQuery, and = [], or = []
858
+ // Scheme restrictions
859
+ // FIXME: Currently, this allows the scheme to be anywhere in the mapping.
860
+ // Restrict it to the side where the notation is searched.
861
+ if (vocs && vocs.length) {
862
+ let vocOr = []
863
+ for (let voc of vocs) {
864
+ vocOr.push({ "fromScheme.uri": voc })
865
+ vocOr.push({ "fromScheme.notation": voc })
866
+ vocOr.push({ "toScheme.uri": voc })
867
+ vocOr.push({ "toScheme.notation": voc })
868
+ }
869
+ and.push({ $or: vocOr })
870
+ }
871
+ // Notations
872
+ // TODO: - Implement mode.
873
+ let paths = ["from.memberSet", "to.memberSet", "to.memberList", "to.memberChoice"]
874
+ for (let path of paths) {
875
+ or.push({ [path + ".notation"]: { $regex: `^${_.escapeRegExp(search)}` } })
876
+ }
877
+ and.push({ $or: or })
878
+ mongoQuery = { $and: and }
879
+ let mappings = await this.model.find(mongoQuery).lean().exec()
880
+ let results = []
881
+ let descriptions = []
882
+ for (let mapping of mappings) {
883
+ for (let path of paths) {
884
+ let concepts = _.get(mapping, path, null)
885
+ if (!concepts) {
886
+ continue
887
+ }
888
+ let notations = []
889
+ for (let concept of concepts) {
890
+ notations = notations.concat(concept.notation)
891
+ }
892
+ for (let notation of notations) {
893
+ if (_.lowerCase(notation).startsWith(_.lowerCase(search))) {
894
+ let index = results.indexOf(notation)
895
+ if (index == -1) {
896
+ results.push(notation)
897
+ descriptions.push(1)
898
+ } else {
899
+ descriptions[index] += 1
900
+ }
901
+ }
902
+ }
903
+ }
904
+ }
905
+ let zippedResults = _.zip(results, descriptions)
906
+ zippedResults.sort((a, b) => {
907
+ return b[1] - a[1] || a[0] > b[0]
908
+ })
909
+ let unzippedResults = _.unzip(zippedResults.slice(query.offset, query.offset + query.limit))
910
+ const toBeReturned = [
911
+ search,
912
+ unzippedResults[0] || [],
913
+ unzippedResults[1] || [],
914
+ [],
915
+ ]
916
+ toBeReturned.totalCount = zippedResults.length
917
+ return toBeReturned
918
+ }
919
+
920
+ async createIndexes() {
921
+ const indexes = []
922
+ for (let path of ["from.memberSet", "from.memberList", "from.memberChoice", "to.memberSet", "to.memberList", "to.memberChoice", "fromScheme", "toScheme"]) {
923
+ for (let type of ["notation", "uri"]) {
924
+ indexes.push([{ [`${path}.${type}`]: 1 }, {}])
925
+ }
926
+ }
927
+ // Separately create multi-indexes for fromScheme/toScheme
928
+ indexes.push([{
929
+ "fromScheme.uri": 1,
930
+ "toScheme.uri": 1,
931
+ }, {}])
932
+ indexes.push([{
933
+ "fromScheme.uri": 1,
934
+ modified: -1,
935
+ }, {}])
936
+ indexes.push([{
937
+ "fromScheme.notation": 1,
938
+ modified: -1,
939
+ }, {}])
940
+ indexes.push([{
941
+ "toScheme.uri": 1,
942
+ modified: -1,
943
+ }, {}])
944
+ indexes.push([{
945
+ "toScheme.notation": 1,
946
+ modified: -1,
947
+ }, {}])
948
+ indexes.push([{
949
+ "partOf.uri": 1,
950
+ modified: -1,
951
+ }, {}])
952
+ indexes.push([{ uri: 1 }, {}])
953
+ indexes.push([{ identifier: 1 }, {}])
954
+ indexes.push([{ type: 1 }, {}])
955
+ indexes.push([{ created: 1, _id: 1 }, {}])
956
+ indexes.push([{ modified: 1, _id: 1 }, {}])
957
+ indexes.push([{ mappingRelevance: 1, _id: 1 }, {}])
958
+ indexes.push([{ created: 1, "from.memberSet.uri": 1, _id: 1 }, {}])
959
+ indexes.push([{ modified: 1, "from.memberSet.uri": 1, _id: 1 }, {}])
960
+ indexes.push([{ mappingRelevance: 1, "from.memberSet.uri": 1, _id: 1 }, {}])
961
+ indexes.push([{ "partOf.uri": 1 }, {}])
962
+ indexes.push([{ "creator.uri": 1 }, {}])
963
+ indexes.push([{ "creator.prefLabel.de": 1 }, {}])
964
+ indexes.push([{ "creator.prefLabel.en": 1 }, {}])
965
+ // Create collection if necessary
966
+ try {
967
+ await this.model.createCollection()
968
+ } catch (error) {
969
+ // Ignore error
970
+ }
971
+ // Drop existing indexes
972
+ await this.model.collection.dropIndexes()
973
+ for (let [index, options] of indexes) {
974
+ await this.model.collection.createIndex(index, options)
975
+ }
976
+ }
977
+
978
+ }