@paulhectork/aiiinotate 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/README.md +206 -0
- package/cli/export.js +118 -0
- package/cli/import.js +111 -0
- package/cli/index.js +45 -0
- package/cli/migrate.js +131 -0
- package/cli/serve.js +29 -0
- package/cli/utils/fastifyClient.js +86 -0
- package/cli/utils/io.js +225 -0
- package/cli/utils/mongoClient.js +21 -0
- package/cli/utils/progressbar.js +86 -0
- package/cli/xywhToInt.js +99 -0
- package/config/.env.template +48 -0
- package/docker/Dockerfile +39 -0
- package/docker/README.md +5 -0
- package/docker/docker-compose.yaml +49 -0
- package/docker/docker.sh +43 -0
- package/docker/docker_aiiinotate_import.sh +62 -0
- package/docs/api.md +381 -0
- package/docs/cli.md +132 -0
- package/docs/dev_documentation/dev_architecture.md +88 -0
- package/docs/dev_documentation/dev_db.md +58 -0
- package/docs/dev_documentation/dev_iiif_compatibility.md +43 -0
- package/docs/dev_documentation/dev_notes_quirks_and_troubleshooting.md +143 -0
- package/docs/docker.md +98 -0
- package/docs/includes/report_benchmark_aiiinotate_2026-05-28-02:50:48_7steps.png +0 -0
- package/docs/scalability.md +34 -0
- package/docs/specifications/0_w3c_open_annotations.md +332 -0
- package/docs/specifications/1_w3c_web_annotations.md +577 -0
- package/docs/specifications/2_iiif_apis.md +428 -0
- package/docs/specifications/3_iiif_annotations.md +103 -0
- package/docs/specifications/4_search_api.md +135 -0
- package/docs/specifications/5_sas.md +119 -0
- package/docs/specifications/6_mirador.md +119 -0
- package/docs/specifications/7_aikon.md +137 -0
- package/docs/specifications/include/presentation_2.0.webp +0 -0
- package/docs/specifications/include/presentation_2.0_white.png +0 -0
- package/docs/specifications/include/presentation_3.0.png +0 -0
- package/docs/specifications/include/presentation_3.0_resize.png +0 -0
- package/eslint.config.js +30 -0
- package/migrations/baseConfig.js +57 -0
- package/migrations/manageIndex.js +55 -0
- package/migrations/migrate-mongo-config-main.js +8 -0
- package/migrations/migrate-mongo-config-test.js +8 -0
- package/migrations/migrationScripts/20250825185706-collections.js +48 -0
- package/migrations/migrationScripts/20250826194832-annotations2-schema.js +42 -0
- package/migrations/migrationScripts/20250904080710-annotations2-indexes.js +69 -0
- package/migrations/migrationScripts/20251002141951-manifests2-schema.js +43 -0
- package/migrations/migrationScripts/20251006212110-manifests2-indexes.js +35 -0
- package/migrations/migrationTemplate.js +25 -0
- package/package.json +82 -0
- package/scripts/get_version.py +12 -0
- package/scripts/run.sh +36 -0
- package/scripts/setup_mongodb.sh +70 -0
- package/scripts/setup_node.sh +15 -0
- package/scripts/update_version.py +30 -0
- package/scripts/utils.sh +65 -0
- package/src/app.js +116 -0
- package/src/constants.js +73 -0
- package/src/data/annotations/annotations2.js +681 -0
- package/src/data/annotations/annotations3.js +28 -0
- package/src/data/annotations/routes.js +335 -0
- package/src/data/annotations/routes.test.js +271 -0
- package/src/data/collectionAbstract.js +283 -0
- package/src/data/index.js +29 -0
- package/src/data/manifests/manifests2.js +378 -0
- package/src/data/manifests/manifests2.test.js +53 -0
- package/src/data/manifests/manifests3.js +23 -0
- package/src/data/manifests/routes.js +122 -0
- package/src/data/manifests/routes.test.js +70 -0
- package/src/data/routes.js +181 -0
- package/src/data/routes.test.js +166 -0
- package/src/db/index.js +50 -0
- package/src/fixtures/annotations.js +41 -0
- package/src/fixtures/data/annotationList_aikon_wit9_man11_anno165_all.jsonld +827 -0
- package/src/fixtures/data/annotationList_vhs_wit250_man250_anno250_all.jsonld +37514 -0
- package/src/fixtures/data/annotationList_vhs_wit253_man253_anno253_all.jsonld +20111 -0
- package/src/fixtures/data/annotations2Invalid.jsonld +39 -0
- package/src/fixtures/data/annotations2SvgValid.jsonld +81 -0
- package/src/fixtures/data/annotations2Valid.jsonld +39 -0
- package/src/fixtures/data/bnf_invalid_manifest.json +2806 -0
- package/src/fixtures/data/bnf_valid_manifest.json +2817 -0
- package/src/fixtures/data/vhs_wit253_man253_anno253_anno-24.json +1 -0
- package/src/fixtures/generate.js +181 -0
- package/src/fixtures/index.js +69 -0
- package/src/fixtures/manifests.js +14 -0
- package/src/fixtures/utils.js +37 -0
- package/src/schemas/index.js +100 -0
- package/src/schemas/schemasBase.js +19 -0
- package/src/schemas/schemasPresentation2.js +410 -0
- package/src/schemas/schemasPresentation3.js +33 -0
- package/src/schemas/schemasResolver.js +71 -0
- package/src/schemas/schemasRoutes.js +318 -0
- package/src/server.js +25 -0
- package/src/types.js +97 -0
- package/src/utils/iiif2Utils.js +332 -0
- package/src/utils/iiif2Utils.test.js +146 -0
- package/src/utils/iiif3Utils.js +0 -0
- package/src/utils/iiifUtils.js +18 -0
- package/src/utils/logger.js +119 -0
- package/src/utils/routeUtils.js +137 -0
- package/src/utils/svg.js +417 -0
- package/src/utils/testUtils.js +289 -0
- package/src/utils/utils.js +403 -0
|
@@ -0,0 +1,681 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* IIIF presentation 2.1 annotation internals: convert incoming data, interct with the database, return data.
|
|
3
|
+
* exposes an `Annotations2` class that should contain everything you need to interact with the annotations2 collection.
|
|
4
|
+
*/
|
|
5
|
+
import fastifyPlugin from "fastify-plugin";
|
|
6
|
+
|
|
7
|
+
import CollectionAbstract from "#data/collectionAbstract.js";
|
|
8
|
+
import { STRICT_MODE } from "#constants";
|
|
9
|
+
import { IIIF_PRESENTATION_2_CONTEXT } from "#utils/iiifUtils.js";
|
|
10
|
+
import { ajvCompile, objectHasKey, isNullish, maybeToArray, visibleLog, memoize, getFirstNonEmptyPair } from "#utils/utils.js";
|
|
11
|
+
import { getManifestShortId, makeTarget, makeAnnotationId, toAnnotationList, canvasUriToManifestUri } from "#utils/iiif2Utils.js";
|
|
12
|
+
import { PAGE_SIZE } from "#constants";
|
|
13
|
+
|
|
14
|
+
/** @typedef {import("mongodb").FindCursor} FindCursor */
|
|
15
|
+
/** @typedef {import("#types").FastifyInstanceType} FastifyInstanceType */
|
|
16
|
+
/** @typedef {import("#types").MongoObjectId} MongoObjectId */
|
|
17
|
+
/** @typedef {import("#types").MongoInsertResultType} MongoInsertResultType */
|
|
18
|
+
/** @typedef {import("#types").MongoUpdateResultType} MongoUpdateResultType */
|
|
19
|
+
/** @typedef {import("#types").MongoDeleteResultType} MongoDeleteResultType */
|
|
20
|
+
/** @typedef {import("#types").InsertResponseType} InsertResponseType */
|
|
21
|
+
/** @typedef {import("#types").UpdateResponseType} UpdateResponseType */
|
|
22
|
+
/** @typedef {import("#types").DeleteResponseType} DeleteResponseType */
|
|
23
|
+
/** @typedef {import("#types").DataOperationsType } DataOperationsType */
|
|
24
|
+
/** @typedef {import("#types").AnnotationsDeleteKeyType } AnnotationsDeleteKeyType */
|
|
25
|
+
/** @typedef {import("#types").Manifests2InstanceType} Manifests2InstanceType */
|
|
26
|
+
/** @typedef {import("#types").AjvValidateFunctionType} AjvValidateFunctionType */
|
|
27
|
+
|
|
28
|
+
/** @typedef {Annotations2} Annotations2InstanceType */
|
|
29
|
+
|
|
30
|
+
// RECOMMENDED URI PATTERNS https://iiif.io/api/presentation/2.1/#a-summary-of-recommended-uri-patterns
|
|
31
|
+
//
|
|
32
|
+
// Collection {scheme}://{host}/{prefix}/collection/{name}
|
|
33
|
+
// Manifest {scheme}://{host}/{prefix}/{identifier}/manifest
|
|
34
|
+
// Sequence {scheme}://{host}/{prefix}/{identifier}/sequence/{name}
|
|
35
|
+
// Canvas {scheme}://{host}/{prefix}/{identifier}/canvas/{name}
|
|
36
|
+
// Annotation (incl images) {scheme}://{host}/{prefix}/{identifier}/annotation/{name}
|
|
37
|
+
// AnnotationList {scheme}://{host}/{prefix}/{identifier}/list/{name}
|
|
38
|
+
// Range {scheme}://{host}/{prefix}/{identifier}/range/{name}
|
|
39
|
+
// Layer {scheme}://{host}/{prefix}/{identifier}/layer/{name}
|
|
40
|
+
// Content {scheme}://{host}/{prefix}/{identifier}/res/{name}.{format}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* @extends {CollectionAbstract}
|
|
44
|
+
*/
|
|
45
|
+
class Annotations2 extends CollectionAbstract {
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* @param {FastifyInstanceType} fastify
|
|
49
|
+
*/
|
|
50
|
+
constructor(fastify) {
|
|
51
|
+
super(fastify, "annotations2");
|
|
52
|
+
/** @type {Manifests2InstanceType} */
|
|
53
|
+
this.manifestsPlugin = this.fastify.manifests2;
|
|
54
|
+
/** @type {AjvValidateFunctionType} */
|
|
55
|
+
this.validatorAnnotationList = ajvCompile(fastify.schemasResolver(
|
|
56
|
+
fastify.schemasPresentation2.getSchema("annotationList")
|
|
57
|
+
));
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
////////////////////////////////////////////////////////////////
|
|
61
|
+
// utils
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* @type {(object) => Promise<number>}
|
|
66
|
+
* cache the number of documents corresponding to a paginated query in a JS cache
|
|
67
|
+
* a simple cache avoids rerunning a count to get the total number of documents for each page of a paginated query
|
|
68
|
+
* see: https://dev.to/codewithjohnson/the-power-of-a-simple-cache-system-with-javascript-map-3j01
|
|
69
|
+
*/
|
|
70
|
+
#memoizePaginationTotalCount = memoize((x) => this.collection.countDocuments(x), 2000);
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* expand a pair of `filterKey`, `filterVal` following the schema `routeAnnotationFilter` into a proper filter for the `annotations2` collection.
|
|
74
|
+
* @param {string} filterKey
|
|
75
|
+
* @param {string} filterVal
|
|
76
|
+
* @returns
|
|
77
|
+
*/
|
|
78
|
+
#expandRouteAnnotationFilter(filterKey, filterVal) {
|
|
79
|
+
const allowedFilterKeys = [ "uri", "manifestShortId", "canvasUri", "tag" ];
|
|
80
|
+
if (!allowedFilterKeys.includes(filterKey)) {
|
|
81
|
+
throw new Error(`${this.funcname(this.#expandRouteAnnotationFilter)}: expected one of ${allowedFilterKeys} for param 'deleteKey', got '${filterKey}'`)
|
|
82
|
+
}
|
|
83
|
+
const map = {
|
|
84
|
+
uri: { "@id": filterVal },
|
|
85
|
+
canvasUri: { "on.full": filterVal },
|
|
86
|
+
manifestShortId: { "on.manifestShortId": filterVal },
|
|
87
|
+
tag: {
|
|
88
|
+
$and: [
|
|
89
|
+
{
|
|
90
|
+
// schema accepts both oa:Tag and Tag
|
|
91
|
+
$or: [
|
|
92
|
+
{ "resource.@type": "oa:Tag" },
|
|
93
|
+
{ "resource.@type": "Tag" }
|
|
94
|
+
]
|
|
95
|
+
},
|
|
96
|
+
{ "resource.chars": filterVal }
|
|
97
|
+
]
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return map[filterKey];
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* clean the body of an annotation (annotation.resource).
|
|
105
|
+
* if `annotation.resource` is an array (there are several bodies associated to that annotation), this function must be called on each item of the array
|
|
106
|
+
* @param {object} resource
|
|
107
|
+
* @returns {object | null} - the resource, or `null` if the resource is either an empty Embedded Textual Body or has no `@id`
|
|
108
|
+
*/
|
|
109
|
+
#cleanAnnotationResource(resource) {
|
|
110
|
+
if (resource) {
|
|
111
|
+
// 1) uniformize embedded textual body keys
|
|
112
|
+
// OA allows `cnt:ContentAsText` or `dctypes:Text` for Embedded Textual Bodies, IIIF only uses `dctypes:Text`
|
|
113
|
+
resource["@type"] =
|
|
114
|
+
resource["@type"] === "cnt:ContentAsText"
|
|
115
|
+
? "dctypes:Text"
|
|
116
|
+
: resource["@type"];
|
|
117
|
+
|
|
118
|
+
// OA stores Textual Body content in `cnt:chars`, IIIF uses `chars`. `value` is sometimes also used
|
|
119
|
+
resource.chars = resource.value || resource["cnt:chars"] || resource.chars; // may be undefined
|
|
120
|
+
[ "value", "cnt:chars" ].map((k) => {
|
|
121
|
+
if (Object.keys(resource).includes(k)) {
|
|
122
|
+
delete resource[k];
|
|
123
|
+
}
|
|
124
|
+
})
|
|
125
|
+
|
|
126
|
+
// 2) return `null` if resource is empty. a body is empty if
|
|
127
|
+
// - it's got no `@id` (=> it's not a referenced textaul body)
|
|
128
|
+
// - it's not an Embedded Textual Body, or it's an empty Embedded Textual Body.
|
|
129
|
+
// see: https://github.com/Aikon-platform/aiiinotate/blob/dev/docs/specifications/0_w3c_open_annotations.md#embedded-textual-body-etb
|
|
130
|
+
const
|
|
131
|
+
hasTextualBody = objectHasKey(resource, "chars"),
|
|
132
|
+
emptyBody = isNullish(resource.chars) || resource.chars === "<p></p>";
|
|
133
|
+
if (isNullish(resource["@id"]) && (emptyBody || !hasTextualBody)) {
|
|
134
|
+
return null
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
return resource;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* clean an annotation before saving it to database.
|
|
142
|
+
* some of the work consists of translating what is defined by the OpenAnnotations standard to what is actually used by IIIF annotations.
|
|
143
|
+
* if `update`, some cleaning will be skipped (especially the redefinition of "@id"), otherwise updates would fail.
|
|
144
|
+
*
|
|
145
|
+
* @param {{ annotation: object, update: boolean, throwOnXywhError: boolean }} annotation
|
|
146
|
+
* @returns {object}
|
|
147
|
+
*/
|
|
148
|
+
async #cleanAnnotation({
|
|
149
|
+
annotation,
|
|
150
|
+
update=false,
|
|
151
|
+
throwOnXywhError=STRICT_MODE
|
|
152
|
+
}) {
|
|
153
|
+
// 1) extract ids and targets. convert the target to an array.
|
|
154
|
+
// we assume that all values of `annotationTargetArray` point to the same manifest => `manifestShortId` is extracted from the 1st target
|
|
155
|
+
const
|
|
156
|
+
annotationTargetArray = await makeTarget(annotation),
|
|
157
|
+
manifestShortId = annotationTargetArray[0].manifestShortId;
|
|
158
|
+
|
|
159
|
+
if (throwOnXywhError && annotationTargetArray[0]?.xywh.some(x => isNaN(x))) {
|
|
160
|
+
throw this.insertError("annotations2.#cleanAnnotation: could not extract bounding box for annotation target", annotation.on);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
// in updates, "@id" has aldready been extracted
|
|
164
|
+
if (!update) {
|
|
165
|
+
annotation["@id"] = makeAnnotationId(annotation, manifestShortId);
|
|
166
|
+
}
|
|
167
|
+
annotation["@context"] = IIIF_PRESENTATION_2_CONTEXT["@context"];
|
|
168
|
+
annotation.on = annotationTargetArray;
|
|
169
|
+
|
|
170
|
+
// 2) process motivations.
|
|
171
|
+
// - motivations are an array of strings
|
|
172
|
+
// - open annotation specifies that motivations should be described by the `oa:Motivation`, while IIIF 2.1 examples uses the `motivation` field => uniformizwe
|
|
173
|
+
// - all values must be `sc:painting` or prefixed by `oa:`: IIIF presentation API indicates that the only allowed values are open annotation values (prefixed by `oa:`) or `sc:painting`.
|
|
174
|
+
if (objectHasKey(annotation, "oa:Motivation")) {
|
|
175
|
+
annotation.motivation = annotation["oa:Motivation"];
|
|
176
|
+
delete annotation["oa:motivation"];
|
|
177
|
+
}
|
|
178
|
+
annotation.motivation =
|
|
179
|
+
maybeToArray(annotation.motivation || [])
|
|
180
|
+
.map(String)
|
|
181
|
+
.map((motiv) =>
|
|
182
|
+
motiv.startsWith("oa:") || motiv.startsWith("sc:")
|
|
183
|
+
? motiv
|
|
184
|
+
: `oa:${motiv}`
|
|
185
|
+
);
|
|
186
|
+
|
|
187
|
+
// 3) process the resource. Resource can be either undefined, an array of objects or a single object. process all objects and, if there's no resource content, delete `annotation.resource`.
|
|
188
|
+
let resource = annotation.resource || undefined;
|
|
189
|
+
if (resource) {
|
|
190
|
+
resource =
|
|
191
|
+
Array.isArray(resource)
|
|
192
|
+
? resource.map((r) => this.#cleanAnnotationResource(r)).filter((r) => r !== null)
|
|
193
|
+
: this.#cleanAnnotationResource(resource);
|
|
194
|
+
}
|
|
195
|
+
if (resource === null || resource === undefined || (Array.isArray(resource) && !resource.length)) {
|
|
196
|
+
delete annotation.resource;
|
|
197
|
+
} else {
|
|
198
|
+
annotation.resource = resource;
|
|
199
|
+
}
|
|
200
|
+
return annotation;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* take an annotationList, clean it and return it as a array of annotations.
|
|
205
|
+
* see: https://iiif.io/api/presentation/2.1/#annotation-list
|
|
206
|
+
* @param {object} annotationList
|
|
207
|
+
* @param {boolean} throwOnXywhError
|
|
208
|
+
* @returns {Promise<object[]>}
|
|
209
|
+
*/
|
|
210
|
+
async #cleanAnnotationList(annotationList, throwOnXywhError=STRICT_MODE) {
|
|
211
|
+
// NOTE: if `this.#cleanAnnotationList` can only be accessed from annotations routes, then this check is useless (has aldready been performed).
|
|
212
|
+
if (this.validatorAnnotationList(annotationList)) {
|
|
213
|
+
this.errorNoAction("Annotations2.#cleanAnnotationList: could not recognize AnnotationList. see: https://iiif.io/api/presentation/2.1/#annotation-list.", annotationList)
|
|
214
|
+
}
|
|
215
|
+
return await Promise.all(
|
|
216
|
+
annotationList.resources.map(async (ressource) =>
|
|
217
|
+
await this.#cleanAnnotation({ annotation: ressource, throwOnXywhError })
|
|
218
|
+
)
|
|
219
|
+
)
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* after attempting to insert a manifests, update a single target of an annotation :
|
|
224
|
+
* 1. set manifestUri to `undefined` if manifest insertion failed
|
|
225
|
+
* 2. fetch target canvas index to populate canvasIdx
|
|
226
|
+
* @param {object} target: a single value of the annotation.on array
|
|
227
|
+
* @param {string[]} insertedManifestIds
|
|
228
|
+
* @returns {Promise<object>} the updated target
|
|
229
|
+
*/
|
|
230
|
+
async #updateTargetPostManifestInsert(target, insertedManifestIds) {
|
|
231
|
+
// set manifestUri to `undefined` if manifest insertion failed.
|
|
232
|
+
const updateManifestUri = (manifestUri) =>
|
|
233
|
+
insertedManifestIds.find((x) => x === manifestUri)
|
|
234
|
+
? manifestUri
|
|
235
|
+
: undefined;
|
|
236
|
+
|
|
237
|
+
// if manifest insertion worked (manifestUri) is not undefined, fetch the canvas index based on the canvas' URI
|
|
238
|
+
const setCanvasIdx = async (manifestUri, canvasUri) =>
|
|
239
|
+
manifestUri
|
|
240
|
+
? await this.manifestsPlugin.getCanvasIdx(manifestUri, canvasUri)
|
|
241
|
+
: undefined;
|
|
242
|
+
|
|
243
|
+
target.manifestUri = updateManifestUri(target.manifestUri);
|
|
244
|
+
target.canvasIdx = await setCanvasIdx(target.manifestUri, target.full);
|
|
245
|
+
return target;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* after attempting to insert manifests, update a single annotation
|
|
250
|
+
* @param {object} annotation
|
|
251
|
+
* @param {string[]} insertedManifestIds
|
|
252
|
+
* @param {boolean} throwOnCanvasIndexError
|
|
253
|
+
* @returns {Promise<object>} the updated annotation
|
|
254
|
+
*/
|
|
255
|
+
async #updateAnnotationPostManifestInsert(annotation, insertedManifestIds, throwOnCanvasIndexError) {
|
|
256
|
+
/** @type {object[]} */
|
|
257
|
+
let targetArray = annotation.on;
|
|
258
|
+
targetArray = await Promise.all(
|
|
259
|
+
targetArray.map(async (target) =>
|
|
260
|
+
await this.#updateTargetPostManifestInsert(target, insertedManifestIds))
|
|
261
|
+
)
|
|
262
|
+
if (
|
|
263
|
+
throwOnCanvasIndexError
|
|
264
|
+
&& targetArray.some((target) => target.canvasIdx === undefined)
|
|
265
|
+
) {
|
|
266
|
+
const canvasUris = targetArray.map((target) => target.full)
|
|
267
|
+
throw this.insertError(`${this.funcName(this.#insertManifestsAndGetCanvasIdx)}: could not get canvasIdx for annotation (canvas URI: ${canvasUris})`);
|
|
268
|
+
}
|
|
269
|
+
annotation.on = targetArray;
|
|
270
|
+
return annotation;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* handle all side effects on the `manifests2` collection. this does 2 things:
|
|
275
|
+
* - insert all manifests referenced by `annotationData`, and set a key `on.manifestId` on all annotations.
|
|
276
|
+
* - set a key `canvasIdx` in all values of `annotation.on`, containing the position of the annotation's target canvas in the manifest,
|
|
277
|
+
* (or undefined if the manifest or canvas were not found).
|
|
278
|
+
* @param {object|object[]} annotationData - an annotation, or array of annotations.
|
|
279
|
+
* @param {boolean} throwOnCanvasIndexError - if canvasIdx can't be found, raise an error.
|
|
280
|
+
*/
|
|
281
|
+
async #insertManifestsAndGetCanvasIdx(annotationData, throwOnCanvasIndexError=STRICT_MODE) {
|
|
282
|
+
// NOTE: instead of propagating `throwOnCanvasIndexError` to `insertManifestsFromUriArray`, we could just check if `insertResponse.fetchErrorIds.length > 0` and return an error then.
|
|
283
|
+
// convert objects to array to get a uniform interface.
|
|
284
|
+
let converted;
|
|
285
|
+
[ annotationData, converted ] = maybeToArray(annotationData, true);
|
|
286
|
+
|
|
287
|
+
// 1. get all distinct manifest URIs
|
|
288
|
+
const manifestUris = [];
|
|
289
|
+
annotationData.map((ann) => ann.on.map((target) => {
|
|
290
|
+
if (target.manifestUri != null && !manifestUris.includes(target.manifestUri)) {
|
|
291
|
+
manifestUris.push(target.manifestUri);
|
|
292
|
+
}
|
|
293
|
+
}));
|
|
294
|
+
|
|
295
|
+
// 2. insert the manifests
|
|
296
|
+
const
|
|
297
|
+
insertResponse = await this.manifestsPlugin.insertManifestsFromUriArray(manifestUris, throwOnCanvasIndexError),
|
|
298
|
+
/** @type {string[]} concatenation of ids of newly inserted manifests and previously inserted manifests. */
|
|
299
|
+
insertedManifestIds = insertResponse.insertedIds.concat(insertResponse.preExistingIds || []);
|
|
300
|
+
|
|
301
|
+
// 3. update annotations with info on manifest and canvas.
|
|
302
|
+
annotationData = await Promise.all(
|
|
303
|
+
annotationData.map(async (annotation) =>
|
|
304
|
+
await this.#updateAnnotationPostManifestInsert(annotation, insertedManifestIds, throwOnCanvasIndexError)
|
|
305
|
+
)
|
|
306
|
+
);
|
|
307
|
+
|
|
308
|
+
// retroconvert array to single object, if single object was converted.
|
|
309
|
+
return converted
|
|
310
|
+
? annotationData[0]
|
|
311
|
+
: annotationData;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
/**
|
|
315
|
+
* taking a filter document `queryFilter`, return an annotationList with paginated results
|
|
316
|
+
*
|
|
317
|
+
* params:
|
|
318
|
+
* - queryUrl: the `@id` of the annotationList.
|
|
319
|
+
* MUST be an URL to a route that sjupports pagination, in order to set `prev` and `next` in the annotationList.
|
|
320
|
+
* - queryFilter: the filter to apply to the collection
|
|
321
|
+
* - page: current page number
|
|
322
|
+
* - pageSize: number of annotations per page
|
|
323
|
+
* - label: title of the AnnotationList.
|
|
324
|
+
*
|
|
325
|
+
* returns: the paginated AnnotationList as a promise
|
|
326
|
+
*
|
|
327
|
+
* NOTE: other/more performant forms of pagination than offset: https://medium.com/mongodb/mongodb-pagination-offset-based-vs-keyset-vs-pre-generated-result-pages-4177e05d88ec
|
|
328
|
+
*
|
|
329
|
+
* @param {{
|
|
330
|
+
* queryUrl: string,
|
|
331
|
+
* queryFilter: object,
|
|
332
|
+
* page: number,
|
|
333
|
+
* pageSize: number,
|
|
334
|
+
* label: string?
|
|
335
|
+
* }}
|
|
336
|
+
* @returns {Promise<object>} - paginated annotation list
|
|
337
|
+
*/
|
|
338
|
+
async #paginate({
|
|
339
|
+
queryUrl,
|
|
340
|
+
queryFilter,
|
|
341
|
+
page=1,
|
|
342
|
+
pageSize=PAGE_SIZE,
|
|
343
|
+
label=undefined
|
|
344
|
+
}) {
|
|
345
|
+
const totalCount = await this.#memoizePaginationTotalCount(queryFilter);
|
|
346
|
+
|
|
347
|
+
const skip = Math.max((page-1) * pageSize, 0); // number of queried items up until the previous page included.
|
|
348
|
+
const cursor = await this.find(queryFilter, {}, true);
|
|
349
|
+
const annotations = await cursor
|
|
350
|
+
.sort({ "@id": 1 })
|
|
351
|
+
.skip(skip)
|
|
352
|
+
.limit(pageSize)
|
|
353
|
+
.toArray();
|
|
354
|
+
|
|
355
|
+
const hasNext = page * pageSize <= totalCount;
|
|
356
|
+
|
|
357
|
+
return toAnnotationList({
|
|
358
|
+
resources: annotations,
|
|
359
|
+
annotationListId: queryUrl,
|
|
360
|
+
page: page,
|
|
361
|
+
hasNext: hasNext,
|
|
362
|
+
label: label
|
|
363
|
+
});
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
////////////////////////////////////////////////////////////////
|
|
367
|
+
// insert / updates
|
|
368
|
+
|
|
369
|
+
/**
|
|
370
|
+
* validate and insert a single annotation.
|
|
371
|
+
*
|
|
372
|
+
* about `throwOnCanvasIndexError`:
|
|
373
|
+
* when inserting, aiiinotate attempts to fetch the target manifest of an annotation and to add the canvas number of the annotation to `annotation.on`.
|
|
374
|
+
* this may fail for a number of reasons (manifest URL and JSON structure, server storing the manifest is inaccessible...). if `throwOnCanvasIndexError`, it will raise.
|
|
375
|
+
*
|
|
376
|
+
* about `throwOnXywhError`: XYWH bounding-box extraction of an annotation is not supported for all selectors (see: `selectorToXywh()`).
|
|
377
|
+
* by default, no error is raised if a bounding box can't be extracted.
|
|
378
|
+
* if `throwOnXywhError`, an error will be thrown. this is for controlled environments where you know exactly what you'll be sending aiiinotate and must rely on `xywh`
|
|
379
|
+
*
|
|
380
|
+
* @param {object} annotation
|
|
381
|
+
* @param {boolean} throwOnCanvasIndexError
|
|
382
|
+
* @param {boolean} throwOnXywhError
|
|
383
|
+
* @returns {Promise<InsertResponseType>}
|
|
384
|
+
*/
|
|
385
|
+
async insertAnnotation(annotation, throwOnCanvasIndexError=STRICT_MODE, throwOnXywhError=STRICT_MODE) {
|
|
386
|
+
annotation = await this.#cleanAnnotation({ annotation, update: false, throwOnXywhError });
|
|
387
|
+
annotation = await this.#insertManifestsAndGetCanvasIdx(annotation, throwOnCanvasIndexError);
|
|
388
|
+
return this.insertOne(annotation);
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
/**
|
|
392
|
+
* TODO: handle side effects when changing `annotation.on`: changes that can affect `manifestShortId`, `manifestUri` and `canvasIdx`
|
|
393
|
+
* (for example, updating `annotation.on.full` would ask to change `canvasIdx`).
|
|
394
|
+
* @param {object} annotation
|
|
395
|
+
* @param {boolean?} throwOnCanvasIndexError
|
|
396
|
+
* @param {boolean?} throwOnXywhError
|
|
397
|
+
* @returns {Promise<UpdateResponseType>}
|
|
398
|
+
*/
|
|
399
|
+
async updateAnnotation(annotation, throwOnCanvasIndexError=STRICT_MODE, throwOnXywhError=STRICT_MODE) {
|
|
400
|
+
// necessary: on insert, the `@id` received is modified by `this.#cleanAnnotationList`.
|
|
401
|
+
annotation = await this.#cleanAnnotation({ annotation, update: true, throwOnXywhError });
|
|
402
|
+
// necessary:
|
|
403
|
+
// 1. handle changes to target canvas' manifest
|
|
404
|
+
// 2. when updating an annotation in MAE, the non-standard fields in the annotation are lost:
|
|
405
|
+
// `canvasIdx`, `manifestUri`, `manifestShortId` are not serialized/deserialized in MAE.
|
|
406
|
+
annotation = await this.#insertManifestsAndGetCanvasIdx(annotation, throwOnCanvasIndexError);
|
|
407
|
+
const
|
|
408
|
+
query = { "@id": annotation["@id"] },
|
|
409
|
+
update = { $set: annotation };
|
|
410
|
+
return this.updateOne(query, update);
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
/**
|
|
414
|
+
* validate and insert annotations from an annotation list.
|
|
415
|
+
*
|
|
416
|
+
* about `throwOnCanvasIndexError`:
|
|
417
|
+
* when inserting, aiiinotate attempts to fetch the target manifest of an annotation and to add the canvas number of the annotation to `annotation.on`.
|
|
418
|
+
* this may fail for a number of reasons (manifest URL and JSON structure, server storing the manifest is inaccessible...). if `throwOnCanvasIndexError`, it will raise.
|
|
419
|
+
*
|
|
420
|
+
* about `throwOnXywhError`:
|
|
421
|
+
* we try to calculate the bounding box of an annotation. this is only supported for FragmentSelectors and SvgSelectors, and will fail otherwise
|
|
422
|
+
* => if `throwOnXywhError`, throw if there is an error calculating the bounding box.
|
|
423
|
+
*
|
|
424
|
+
* @param {object} annotationList
|
|
425
|
+
* @param {boolean?} throwOnCanvasIndexError
|
|
426
|
+
* @param {boolean?} throwOnXywhError
|
|
427
|
+
* @returns {Promise<InsertResponseType>}
|
|
428
|
+
*/
|
|
429
|
+
async insertAnnotationList(annotationList, throwOnCanvasIndexError=STRICT_MODE, throwOnXywhError=STRICT_MODE) {
|
|
430
|
+
let annotationArray;
|
|
431
|
+
annotationArray = await this.#cleanAnnotationList(annotationList, throwOnXywhError);
|
|
432
|
+
annotationArray = await this.#insertManifestsAndGetCanvasIdx(annotationArray, throwOnCanvasIndexError);
|
|
433
|
+
return this.insertMany(annotationArray);
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
////////////////////////////////////////////////////////////////
|
|
437
|
+
// delete
|
|
438
|
+
|
|
439
|
+
/**
|
|
440
|
+
* @param {Object<string,string>} deleteFilter - filter for the annotations to delete
|
|
441
|
+
* @returns {Promise<DeleteResponseType>}
|
|
442
|
+
*/
|
|
443
|
+
async deleteAnnotations(deleteFilter) {
|
|
444
|
+
const err = (message) => this.deleteError(`${this.funcName(this.deleteAnnotations)}: ${message}`);
|
|
445
|
+
try {
|
|
446
|
+
let expandedDeleteFilter;
|
|
447
|
+
if (Object.keys(deleteFilter).includes("tag")) {
|
|
448
|
+
// should be validated by the route's JSONSchema, but just in case.
|
|
449
|
+
if (! Object.keys(deleteFilter).includes("manifestShortId")) {
|
|
450
|
+
throw err("Cannot delete by \"tag\" without also filtering by \"manifestShortId\" !")
|
|
451
|
+
}
|
|
452
|
+
const expand = (k) => this.#expandRouteAnnotationFilter(k, deleteFilter[k]);
|
|
453
|
+
expandedDeleteFilter = {
|
|
454
|
+
...expand("tag"),
|
|
455
|
+
...expand("manifestShortId")
|
|
456
|
+
}
|
|
457
|
+
} else {
|
|
458
|
+
const [ deleteKey, deleteVal ] = getFirstNonEmptyPair(deleteFilter);
|
|
459
|
+
expandedDeleteFilter = this.#expandRouteAnnotationFilter(deleteKey, deleteVal);
|
|
460
|
+
}
|
|
461
|
+
return this.delete(expandedDeleteFilter);
|
|
462
|
+
} catch (err) {
|
|
463
|
+
throw err(err.message);
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
////////////////////////////////////////////////////////////////
|
|
468
|
+
// get
|
|
469
|
+
|
|
470
|
+
/**
|
|
471
|
+
* find documents based on a `queryObj` and project them to `projectionObj`.
|
|
472
|
+
*
|
|
473
|
+
* about projection: 0 removes the fields from the response, 1 incldes it (but exclude all others)
|
|
474
|
+
* see: https://www.mongodb.com/docs/drivers/node/current/crud/query/project/#std-label-node-project
|
|
475
|
+
* https://stackoverflow.com/questions/74447979/mongoservererror-cannot-do-exclusion-on-field-date-in-inclusion-projection
|
|
476
|
+
* @param {object} queryObj - the filter document
|
|
477
|
+
* @param {object?} projectionObj - extra projection fields to tailor the reponse format
|
|
478
|
+
* @param {boolean} asCursor - return a cursor instead of an array of results
|
|
479
|
+
* @returns {Promise<object[] | FindCursor>}
|
|
480
|
+
*/
|
|
481
|
+
async find(queryObj, projectionObj={}, asCursor=false) {
|
|
482
|
+
// 1. construct the final projection object, knowing that we can't mix exclusive and inclusive projectin.
|
|
483
|
+
// presence of `_id` will not cause projections to fail => remove it from values.
|
|
484
|
+
const projectionValues =
|
|
485
|
+
Object.entries(projectionObj)
|
|
486
|
+
.filter(([ k,v ]) => k !== "_id")
|
|
487
|
+
.map(([ k,v ]) => v);
|
|
488
|
+
|
|
489
|
+
// if there are projection values defined and if they're not 0 or 1, then they're invalid => throw
|
|
490
|
+
if (projectionValues.length && projectionValues.find((x) => ![ 0,1 ].includes(x))) {
|
|
491
|
+
throw this.readError(`Annotations2.find: only allowed values for projection are 0 and 1. got: ${[ ...new Set(projectionValues) ]}`)
|
|
492
|
+
}
|
|
493
|
+
// mongo projection can be either inclusive (define only fields that will be included) or negative (define only fields that will be excluded), but not a mix of the 2. if you have more than 1 distinct values, you mixed inclusion and exclusion => throw
|
|
494
|
+
const distinctProjectionValues = [ ...new Set(projectionValues) ]
|
|
495
|
+
if (distinctProjectionValues.length > 1) {
|
|
496
|
+
throw this.readError(`Annotations2.find: can't mix insertion and exclusion projection in 'projectionObj'. all values must be either 0 or 1. got: ${distinctProjectionValues}`, projectionObj)
|
|
497
|
+
}
|
|
498
|
+
// negative projection: all fields will be included except for those specified.
|
|
499
|
+
// in this case, negate other fields that we don't ant exposed.
|
|
500
|
+
// in case of positive projection, no specific processing is required: only the explicitly required fields are included.
|
|
501
|
+
// in all cases, `_id` should not be included unless we explicitly ask for it.
|
|
502
|
+
projectionObj._id = projectionObj._id || 0;
|
|
503
|
+
if (distinctProjectionValues[0] === 0) {
|
|
504
|
+
projectionObj["on.manifestShortId"] = projectionObj["on.manifestShortId"] || 0;
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
// 2. find, project and return
|
|
508
|
+
const cursor = this.collection.find(queryObj, { projection: projectionObj });
|
|
509
|
+
if (!asCursor) {
|
|
510
|
+
return cursor.toArray()
|
|
511
|
+
} else {
|
|
512
|
+
return cursor;
|
|
513
|
+
};
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
/**
|
|
517
|
+
* implementation of the IIIF Search API 1.0.
|
|
518
|
+
* function arguments have been validated by JSONSchemas at route-level so they're clean.
|
|
519
|
+
*
|
|
520
|
+
* parameters:
|
|
521
|
+
* - queryUrl - the request URL (/search-api...)
|
|
522
|
+
* - manifestShortId - the manifest's identifier
|
|
523
|
+
* - q: query string (content to search for in annotations)
|
|
524
|
+
* - motivation: filter by annotation motivation
|
|
525
|
+
* - canvasMin - minimum value of `on.canvasIdx`, inclusive
|
|
526
|
+
* - canvasMax - maximum value of `on.canvasIdx`, inclusive
|
|
527
|
+
* - onlyIds - return only the @ids of matched annotations instead of the entire annotations
|
|
528
|
+
*
|
|
529
|
+
* NOTE:
|
|
530
|
+
* - only `motivation` and `q` search params are implemented
|
|
531
|
+
* - to increase search execution speed, ONLY EXACT STRING MACHES are
|
|
532
|
+
* implemented for `q` and `motivation` (in the IIIF specs, you can supply
|
|
533
|
+
* multiple space-separated values and the server should return all partial
|
|
534
|
+
* matches to any of those strings.)
|
|
535
|
+
* - non-standard `canvasMin`, `canvasMax` and `onlyIds` parameters are implemented
|
|
536
|
+
*
|
|
537
|
+
* see:
|
|
538
|
+
* https://iiif.io/api/search/1.0/
|
|
539
|
+
* https://github.com/Aikon-platform/aiiinotate/blob/dev/docs/specifications/4_search_api.md
|
|
540
|
+
*
|
|
541
|
+
* @param {{
|
|
542
|
+
* queryUrl: string,
|
|
543
|
+
* manifestShortId: string,
|
|
544
|
+
* q: string?,
|
|
545
|
+
* motivation: ("painting"|"non-painting"|"commenting"|"describing"|"tagging"|"linking")?,
|
|
546
|
+
* canvasMin: number?,
|
|
547
|
+
* canvasMax: number?,
|
|
548
|
+
* onlyIds: boolean
|
|
549
|
+
* page: number
|
|
550
|
+
* pageSize: number
|
|
551
|
+
* }}
|
|
552
|
+
* @returns {object} annotationList containing results
|
|
553
|
+
*/
|
|
554
|
+
async search({
|
|
555
|
+
queryUrl,
|
|
556
|
+
manifestShortId,
|
|
557
|
+
q,
|
|
558
|
+
motivation=undefined,
|
|
559
|
+
canvasMin=undefined,
|
|
560
|
+
canvasMax=undefined,
|
|
561
|
+
onlyIds=false,
|
|
562
|
+
page=1,
|
|
563
|
+
pageSize=PAGE_SIZE
|
|
564
|
+
}) {
|
|
565
|
+
const
|
|
566
|
+
filtersBase = { "on.manifestShortId": manifestShortId },
|
|
567
|
+
filtersExtra = { $and: [] };
|
|
568
|
+
|
|
569
|
+
// expand query parameters
|
|
570
|
+
if (q) {
|
|
571
|
+
filtersExtra.$and.push({
|
|
572
|
+
$or: [
|
|
573
|
+
{ "@id": q },
|
|
574
|
+
{ "resource.@id": q },
|
|
575
|
+
{ "resource.chars": q }
|
|
576
|
+
]
|
|
577
|
+
});
|
|
578
|
+
};
|
|
579
|
+
if (motivation) {
|
|
580
|
+
filtersExtra.$and.push(
|
|
581
|
+
motivation === "non-painting"
|
|
582
|
+
? { motivation: { $ne: "sc:painting" } }
|
|
583
|
+
: motivation === "painting"
|
|
584
|
+
? { motivation: "sc:painting" }
|
|
585
|
+
: { motivation: `oa:${motivation}` }
|
|
586
|
+
);
|
|
587
|
+
};
|
|
588
|
+
if (!isNaN(canvasMin)) {
|
|
589
|
+
// if canvasMax is undefined, then search for canvasIdx===canvasMin
|
|
590
|
+
if (!canvasMax) {
|
|
591
|
+
filtersExtra.$and.push({ "on.canvasIdx": canvasMin })
|
|
592
|
+
// if canvasMin and canvasMax, canvasIdx must be in [canvasMin, canvasMax] (inclusive).
|
|
593
|
+
} else {
|
|
594
|
+
filtersExtra.$and.push({
|
|
595
|
+
$and: [
|
|
596
|
+
{ "on.canvasIdx": { $gte: canvasMin } },
|
|
597
|
+
{ "on.canvasIdx": { $lte: canvasMax } }
|
|
598
|
+
]
|
|
599
|
+
})
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
const queryFilter =
|
|
603
|
+
filtersExtra.$and.length
|
|
604
|
+
? { ...filtersBase, ...filtersExtra }
|
|
605
|
+
: filtersBase;
|
|
606
|
+
|
|
607
|
+
if (!onlyIds) {
|
|
608
|
+
return await this.#paginate({
|
|
609
|
+
queryUrl,
|
|
610
|
+
queryFilter,
|
|
611
|
+
page,
|
|
612
|
+
pageSize,
|
|
613
|
+
label: `Paginated annotation List (page: ${page}, ${pageSize} items per page)`
|
|
614
|
+
})
|
|
615
|
+
} else {
|
|
616
|
+
// NOTE: there is no pagination if `onlyIds` is true
|
|
617
|
+
return (await this.find(queryFilter, { "@id": 1 }))
|
|
618
|
+
.map((ann) => ann["@id"]);
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
/**
|
|
623
|
+
* find all annotations whose target (`on.full`) is `canvasUri`.
|
|
624
|
+
* @param {{
|
|
625
|
+
* queryUrl: string,
|
|
626
|
+
* canvasUri: string,
|
|
627
|
+
* page: Number,
|
|
628
|
+
* pageSize: Number
|
|
629
|
+
* }} canvasUri
|
|
630
|
+
* @returns {AnnotationList}
|
|
631
|
+
*/
|
|
632
|
+
async findByCanvasUri({
|
|
633
|
+
queryUrl,
|
|
634
|
+
canvasUri,
|
|
635
|
+
page=1,
|
|
636
|
+
pageSize=PAGE_SIZE
|
|
637
|
+
}) {
|
|
638
|
+
return this.#paginate({
|
|
639
|
+
queryUrl,
|
|
640
|
+
queryFilter: { "on.full": canvasUri },
|
|
641
|
+
label: `annotations targeting canvas ${canvasUri}`,
|
|
642
|
+
page,
|
|
643
|
+
pageSize
|
|
644
|
+
});
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
/**
|
|
648
|
+
* find an annotation by its "@id"
|
|
649
|
+
* @param {string} annotationUri
|
|
650
|
+
* @returns {Promise<object>} the annotation, or `{}` if none was found
|
|
651
|
+
*/
|
|
652
|
+
async findById(annotationUri) {
|
|
653
|
+
return this.collection.findOne({ "@id": annotationUri })
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
/**
|
|
657
|
+
* count number of annotations.
|
|
658
|
+
* @param {string} filterKey
|
|
659
|
+
* @param {string} filterVal
|
|
660
|
+
* @returns
|
|
661
|
+
*/
|
|
662
|
+
async count(filterKey, filterVal) {
|
|
663
|
+
try {
|
|
664
|
+
const
|
|
665
|
+
countFilter = this.#expandRouteAnnotationFilter(filterKey, filterVal),
|
|
666
|
+
count = await this.collection.countDocuments(countFilter);
|
|
667
|
+
return { count: count }
|
|
668
|
+
} catch (err) {
|
|
669
|
+
throw this.readError(`${this.funcName(this.count)}: ${err.message}`)
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
}
|
|
674
|
+
|
|
675
|
+
export default fastifyPlugin((fastify, options, done) => {
|
|
676
|
+
fastify.decorate("annotations2", new Annotations2(fastify));
|
|
677
|
+
done();
|
|
678
|
+
}, {
|
|
679
|
+
name: "annotations2",
|
|
680
|
+
dependencies: [ "manifests2" ]
|
|
681
|
+
})
|