@paulhectork/aiiinotate 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/LICENSE +661 -0
  2. package/README.md +206 -0
  3. package/cli/export.js +118 -0
  4. package/cli/import.js +111 -0
  5. package/cli/index.js +45 -0
  6. package/cli/migrate.js +131 -0
  7. package/cli/serve.js +29 -0
  8. package/cli/utils/fastifyClient.js +86 -0
  9. package/cli/utils/io.js +225 -0
  10. package/cli/utils/mongoClient.js +21 -0
  11. package/cli/utils/progressbar.js +86 -0
  12. package/cli/xywhToInt.js +99 -0
  13. package/config/.env.template +48 -0
  14. package/docker/Dockerfile +39 -0
  15. package/docker/README.md +5 -0
  16. package/docker/docker-compose.yaml +49 -0
  17. package/docker/docker.sh +43 -0
  18. package/docker/docker_aiiinotate_import.sh +62 -0
  19. package/docs/api.md +381 -0
  20. package/docs/cli.md +132 -0
  21. package/docs/dev_documentation/dev_architecture.md +88 -0
  22. package/docs/dev_documentation/dev_db.md +58 -0
  23. package/docs/dev_documentation/dev_iiif_compatibility.md +43 -0
  24. package/docs/dev_documentation/dev_notes_quirks_and_troubleshooting.md +143 -0
  25. package/docs/docker.md +98 -0
  26. package/docs/includes/report_benchmark_aiiinotate_2026-05-28-02:50:48_7steps.png +0 -0
  27. package/docs/scalability.md +34 -0
  28. package/docs/specifications/0_w3c_open_annotations.md +332 -0
  29. package/docs/specifications/1_w3c_web_annotations.md +577 -0
  30. package/docs/specifications/2_iiif_apis.md +428 -0
  31. package/docs/specifications/3_iiif_annotations.md +103 -0
  32. package/docs/specifications/4_search_api.md +135 -0
  33. package/docs/specifications/5_sas.md +119 -0
  34. package/docs/specifications/6_mirador.md +119 -0
  35. package/docs/specifications/7_aikon.md +137 -0
  36. package/docs/specifications/include/presentation_2.0.webp +0 -0
  37. package/docs/specifications/include/presentation_2.0_white.png +0 -0
  38. package/docs/specifications/include/presentation_3.0.png +0 -0
  39. package/docs/specifications/include/presentation_3.0_resize.png +0 -0
  40. package/eslint.config.js +30 -0
  41. package/migrations/baseConfig.js +57 -0
  42. package/migrations/manageIndex.js +55 -0
  43. package/migrations/migrate-mongo-config-main.js +8 -0
  44. package/migrations/migrate-mongo-config-test.js +8 -0
  45. package/migrations/migrationScripts/20250825185706-collections.js +48 -0
  46. package/migrations/migrationScripts/20250826194832-annotations2-schema.js +42 -0
  47. package/migrations/migrationScripts/20250904080710-annotations2-indexes.js +69 -0
  48. package/migrations/migrationScripts/20251002141951-manifests2-schema.js +43 -0
  49. package/migrations/migrationScripts/20251006212110-manifests2-indexes.js +35 -0
  50. package/migrations/migrationTemplate.js +25 -0
  51. package/package.json +82 -0
  52. package/scripts/get_version.py +12 -0
  53. package/scripts/run.sh +36 -0
  54. package/scripts/setup_mongodb.sh +70 -0
  55. package/scripts/setup_node.sh +15 -0
  56. package/scripts/update_version.py +30 -0
  57. package/scripts/utils.sh +65 -0
  58. package/src/app.js +116 -0
  59. package/src/constants.js +73 -0
  60. package/src/data/annotations/annotations2.js +681 -0
  61. package/src/data/annotations/annotations3.js +28 -0
  62. package/src/data/annotations/routes.js +335 -0
  63. package/src/data/annotations/routes.test.js +271 -0
  64. package/src/data/collectionAbstract.js +283 -0
  65. package/src/data/index.js +29 -0
  66. package/src/data/manifests/manifests2.js +378 -0
  67. package/src/data/manifests/manifests2.test.js +53 -0
  68. package/src/data/manifests/manifests3.js +23 -0
  69. package/src/data/manifests/routes.js +122 -0
  70. package/src/data/manifests/routes.test.js +70 -0
  71. package/src/data/routes.js +181 -0
  72. package/src/data/routes.test.js +166 -0
  73. package/src/db/index.js +50 -0
  74. package/src/fixtures/annotations.js +41 -0
  75. package/src/fixtures/data/annotationList_aikon_wit9_man11_anno165_all.jsonld +827 -0
  76. package/src/fixtures/data/annotationList_vhs_wit250_man250_anno250_all.jsonld +37514 -0
  77. package/src/fixtures/data/annotationList_vhs_wit253_man253_anno253_all.jsonld +20111 -0
  78. package/src/fixtures/data/annotations2Invalid.jsonld +39 -0
  79. package/src/fixtures/data/annotations2SvgValid.jsonld +81 -0
  80. package/src/fixtures/data/annotations2Valid.jsonld +39 -0
  81. package/src/fixtures/data/bnf_invalid_manifest.json +2806 -0
  82. package/src/fixtures/data/bnf_valid_manifest.json +2817 -0
  83. package/src/fixtures/data/vhs_wit253_man253_anno253_anno-24.json +1 -0
  84. package/src/fixtures/generate.js +181 -0
  85. package/src/fixtures/index.js +69 -0
  86. package/src/fixtures/manifests.js +14 -0
  87. package/src/fixtures/utils.js +37 -0
  88. package/src/schemas/index.js +100 -0
  89. package/src/schemas/schemasBase.js +19 -0
  90. package/src/schemas/schemasPresentation2.js +410 -0
  91. package/src/schemas/schemasPresentation3.js +33 -0
  92. package/src/schemas/schemasResolver.js +71 -0
  93. package/src/schemas/schemasRoutes.js +318 -0
  94. package/src/server.js +25 -0
  95. package/src/types.js +97 -0
  96. package/src/utils/iiif2Utils.js +332 -0
  97. package/src/utils/iiif2Utils.test.js +146 -0
  98. package/src/utils/iiif3Utils.js +0 -0
  99. package/src/utils/iiifUtils.js +18 -0
  100. package/src/utils/logger.js +119 -0
  101. package/src/utils/routeUtils.js +137 -0
  102. package/src/utils/svg.js +417 -0
  103. package/src/utils/testUtils.js +289 -0
  104. package/src/utils/utils.js +403 -0
@@ -0,0 +1,283 @@
1
+ import { maybeToArray, inspectObj } from "#utils/utils.js"
2
+ import { formatInsertResponse, formatDeleteResponse, formatUpdateResponse } from "#utils/routeUtils.js";
3
+
4
+ /** @typedef {import("#types").MongoDbType} MongoDbType */
5
+ /** @typedef {import("#types").MongoClientType} MongoClientType */
6
+ /** @typedef {import("#types").IiifPresentationVersionType} IiifPresentationVersionType */
7
+ /** @typedef {import("#types").MongoCollectionType} MongoCollectionType */
8
+ /** @typedef {import("#types").CollectionNamesType} CollectionNamesType */
9
+ /** @typedef {import("#types").FastifyInstanceType} FastifyInstanceType */
10
+ /** @typedef {import("#types").DataOperationsType } DataOperationsType */
11
+
12
+ const allowedCollectionNames = [ "manifests2", "manifests3", "annotations2", "annotations3" ];
13
+
14
+ class CollectionAbstractError extends Error {
15
+ /**
16
+ * @param {string?} collectionName: name of the collection we're working on
17
+ * @param {string} message: error message
18
+ * @param {DataOperationsType} operation: to describe the type of database interaction
19
+ * @param {object} errInfo: extra info to display for the error
20
+ */
21
+ constructor(collectionName, operation, message, errInfo) {
22
+ const
23
+ collInfo = collectionName ? `on collection '${collectionName}'` : "",
24
+ operationInfo = operation ? `, when performing operation '${operation.toLocaleLowerCase()}'`: "";
25
+ super(`CollectionAbstractError: ${collInfo} ${operationInfo}: ${message}`);
26
+ this.info = errInfo;
27
+ }
28
+ }
29
+
30
+ /**
31
+ * @param {string?} collectionName
32
+ * @returns {Function}
33
+ */
34
+ const errorConstructor = (collectionName) =>
35
+ /**
36
+ * @param {string?} operation
37
+ * @returns {Function}
38
+ */
39
+ (operation) =>
40
+ /**
41
+ *
42
+ * @param {string} message
43
+ * @param {object} errInfo
44
+ * @returns {CollectionAbstractError}
45
+ */
46
+ (message, errInfo) =>
47
+ new CollectionAbstractError(collectionName, operation, message, errInfo);
48
+
49
+ // an error with no collection name or info.
50
+ const abstractError = errorConstructor(undefined)(undefined);
51
+
52
+ /**
53
+ * abstract class defining common processes to interact with a mongo collectios: inserts, updates, errors...
54
+ * this class contains agnostic methods and data that can be applied to any collection.
55
+ * @class
56
+ * @constructor
57
+ * @public
58
+ */
59
+ class CollectionAbstract {
60
+ /**
61
+ * @param {FastifyInstanceType} fastify
62
+ * @param {CollectionNamesType} collectionName
63
+ */
64
+ constructor(fastify, collectionName) {
65
+
66
+ if (!allowedCollectionNames.includes(collectionName)) {
67
+ throw new abstractError(`invalid 'collectionName': expected one of ${allowedCollectionNames}, got '${collectionName}'`);
68
+ }
69
+
70
+ const collectionOptions =
71
+ collectionName === "annotations2"
72
+ ? { validator: { $jsonSchema: fastify.schemasPresentation2.getSchema("annotation") } }
73
+ : collectionName === "annotations3"
74
+ ? { validator: { /** TODO */ } }
75
+ : collectionName === "manifests2"
76
+ ? { validator: { $jsonSchema: fastify.schemasPresentation2.getSchema("manifestMongo") } }
77
+ // else: manifets3.
78
+ : { validator: { /** TODO */ } };
79
+
80
+ const iiifPresentationVersion = collectionName.endsWith("2") ? 2 : 3;
81
+
82
+ /** @type {FastifyInstanceType} */
83
+ this.fastify = fastify;
84
+ /** @type {MongoClientType} */
85
+ this.client = fastify.mongo.client;
86
+ /** @type {MongoDbType} */
87
+ this.db = fastify.mongo.db;
88
+ /** @type {MongoCollectionType} */
89
+ this.collection = this.db.collection(collectionName, collectionOptions);
90
+ /** @type {IiifPresentationVersionType} */
91
+ this.iiifPresentationVersion = iiifPresentationVersion;
92
+
93
+ /** @type {Function(string?) => Function(string, object) => Error} */
94
+ this.errorConstructor = errorConstructor(collectionName);
95
+ /** @type {Function(string, object?) => Error} */
96
+ this.errorNoAction = this.errorConstructor(undefined);
97
+ // create this.(read|insert|update|delete)Error, properties that will be used to throw the proper error.
98
+ [ "read", "insert", "update", "delete" ].forEach((op) =>
99
+ /** @type {Function(string,object?) => Error} */
100
+ this[`${op}Error`] = errorConstructor(collectionName)(op)
101
+ )
102
+ }
103
+
104
+ //////////////////////////////////////
105
+ // UTILS
106
+
107
+ /** @returns {string} */
108
+ className() {
109
+ return this.constructor.name;
110
+ }
111
+
112
+ /** @param {Function} func */
113
+ funcName(func) {
114
+ if (typeof func !== "function") {
115
+ throw new Error(`${this.className()}.${this.funcName.name} : expected 'func' to be a function, got '${typeof func}' (func = ${func})`);
116
+ }
117
+ return `${this.className()}.${func.name}`
118
+ }
119
+
120
+ /**
121
+ * resolve internal mongo '_id' fields to iiif '@id' fields
122
+ * @param {MongoCollectionType} collection
123
+ * @param {string | string[]} mongoIds
124
+ * @returns {Promise<string[]>}
125
+ */
126
+ async getIiifIdsFromMongoIds(mongoIds) {
127
+ mongoIds = maybeToArray(mongoIds);
128
+ const key = this.iiifPresentationVersion === 2 ? "@id" : "id";
129
+ const collectionIds = await this.collection.find(
130
+ { _id: { $in: mongoIds } },
131
+ { projection: { [key]: 1 } }
132
+ ).toArray();
133
+ return collectionIds.map(a => a[key]);
134
+ }
135
+
136
+ //////////////////////////////////////
137
+ // RESPONSES: what is sent from collection classes
138
+ // to routes and other consumers of the class after an insert/update/delete.
139
+
140
+ /**
141
+ * make a uniform response format for insertOne and insertMany
142
+ * @param {MongoInsertResultType} mongoRes
143
+ * @returns {Promise<InsertResponseType>}
144
+ */
145
+ async makeInsertResponse(mongoRes) {
146
+ // retrieve the "@id"s
147
+ const insertedIds = await this.getIiifIdsFromMongoIds(
148
+ // MongoInsertOneResultType and MongoInsertManyResultType have a different structureex
149
+ mongoRes.insertedId || Object.values(mongoRes.insertedIds)
150
+ );
151
+ return formatInsertResponse({ insertedIds });
152
+ }
153
+
154
+ /**
155
+ * @param {MongoUpdateResultType} mongoRes
156
+ * @returns {Promise<UpdateResponseType>}
157
+ */
158
+ async makeUpdateResponse(mongoRes) {
159
+ if (mongoRes.upsertedId) {
160
+ // only 1 entry can be upserted by a mongo query => extract the 1st upserted @id from the mongo @id.
161
+ const upsertedIds = await this.getIiifIdsFromMongoIds(
162
+ mongoRes.upsertedId
163
+ );
164
+ mongoRes.upsertedId = upsertedIds.length ? upsertedIds[0] : mongoRes.upsertedId;
165
+ }
166
+ return formatUpdateResponse(mongoRes);
167
+ }
168
+
169
+ /**
170
+ * throw an error with just the object describing the error data (and not the stack or anything else).
171
+ * used to propagate write errors to routes.
172
+ * @param {DataOperationsType} operation: describes the database operation
173
+ * @param {import("mongodb").MongoServerError} err: the mongo error
174
+ */
175
+ throwMongoError(operation, err) {
176
+ throw this.errorConstructor(operation)(err.message, err.errorResponse);
177
+ }
178
+
179
+ notImplementedError() {
180
+ throw this.errorNoAction("not implemented");
181
+ }
182
+
183
+ //////////////////////////////////////
184
+ // INSERT/UPDATE
185
+
186
+ /**
187
+ * insert a single document `doc` into `this.collection`.
188
+ * no validation or checking is done here. obviously, `doc` must fit the JsonSchema defined for `this.collection`.
189
+ * @private
190
+ * @param {object} doc
191
+ * @returns {Promise<InsertResponseType>}
192
+ */
193
+ async insertOne(doc) {
194
+ try {
195
+ const result = await this.collection.insertOne(doc);
196
+ return this.makeInsertResponse(result);
197
+ } catch (err) {
198
+ this.throwMongoError("insert", err);
199
+ }
200
+ }
201
+
202
+ /**
203
+ * insert documents from an array of documents.
204
+ * no validation or checking is done here. obviously, documents in `docArr` must fit the JsonSchema defined for `this.collection`.
205
+ * @param {object[]} docArr
206
+ * @returns {Promise<InsertResponseType>}
207
+ */
208
+ async insertMany(docArr) {
209
+ try {
210
+ // mongo insertMany throws an error if we try to insert an empty array of documetns
211
+ // => if docArr is empty, return an empty insert response.
212
+ if (docArr.length) {
213
+ const result = await this.collection.insertMany(docArr);
214
+ return this.makeInsertResponse(result);
215
+ } else {
216
+ return formatInsertResponse({
217
+ insertedIds: [],
218
+ rejectedIds: []
219
+ })
220
+ }
221
+ } catch (err) {
222
+ this.throwMongoError("insert", err);
223
+ }
224
+ }
225
+
226
+ /**
227
+ * update a single document, targeted by an unique identifier (should be "@id" for iiif 3, "id" otherwise).
228
+ * @param {object} query: query targeting a document
229
+ * @param {object} update: the updated document.
230
+ * @returns {Promise<UpdateResponseType>}
231
+ */
232
+ async updateOne(query, update){
233
+ try {
234
+ const result = await this.collection.updateOne(query, update);
235
+ return this.makeUpdateResponse(result);
236
+ } catch (err) {
237
+ this.throwMongoError("update", err)
238
+ }
239
+ }
240
+
241
+ //////////////////////////////////////
242
+ // DELETE
243
+
244
+ /**
245
+ * delete all objects that match `queryObj` from `this.collection`
246
+ * NOTE: if nothing is deleted, it's not an error, we return: { deletedCount: 0 }
247
+ * @param {object} queryObj
248
+ * @returns {Promise<DeleteResponseType>}
249
+ */
250
+ async delete(queryObj) {
251
+ try {
252
+ const deleteResult = await this.collection.deleteMany(queryObj);
253
+ return formatDeleteResponse(deleteResult);
254
+ } catch (err) {
255
+ this.throwMongoError("delete", err);
256
+ }
257
+ }
258
+
259
+ //////////////////////////////////////
260
+ // READ
261
+
262
+ /**
263
+ * true if `queryObj` matches at least 1 document, false otherwise.
264
+ * @param {object} queryObj
265
+ * @returns {Promise<boolean>}
266
+ */
267
+ async exists(queryObj) {
268
+ const r = await this.collection.countDocuments(queryObj, { limit: 1 });
269
+ return r === 1;
270
+ }
271
+
272
+ /**
273
+ * count the number of documents that match `queryObj`.
274
+ * @param {object} queryObj
275
+ * @returns {Promise<number>}
276
+ */
277
+ count(queryObj) {
278
+ return this.collection.countDocuments(queryObj);
279
+ }
280
+
281
+ }
282
+
283
+ export default CollectionAbstract;
@@ -0,0 +1,29 @@
1
+ import fastifyPlugin from "fastify-plugin"
2
+
3
+ import Annotations2 from "#annotations/annotations2.js";
4
+ import Annotations3 from "#annotations/annotations3.js";
5
+ import Manifests2 from "#manifests/manifests2.js";
6
+ import Manifests3 from "#manifests/manifests3.js";
7
+ import annotationsRoutes from "#annotations/routes.js";
8
+ import manifestsRoutes from "#manifests/routes.js";
9
+ import commonRoutes from "#data/routes.js";
10
+
11
+ /** @typedef {import("#types").FastifyInstanceType} FastifyInstanceType */
12
+
13
+ /**
14
+ * @param {FastifyInstanceType} fastify
15
+ * @param {object} options
16
+ */
17
+ function data(fastify, options, done) {
18
+
19
+ fastify.register(Manifests2);
20
+ fastify.register(Manifests3);
21
+ fastify.register(Annotations2);
22
+ fastify.register(Annotations3);
23
+ fastify.register(annotationsRoutes);
24
+ fastify.register(manifestsRoutes);
25
+ fastify.register(commonRoutes);
26
+ done();
27
+ }
28
+
29
+ export default fastifyPlugin(data);
@@ -0,0 +1,378 @@
1
+ import fastifyPlugin from "fastify-plugin";
2
+
3
+ import CollectionAbstract from "#data/collectionAbstract.js";
4
+ import { getManifestShortId } from "#utils/iiif2Utils.js";
5
+ import { formatInsertResponse } from "#utils/routeUtils.js";
6
+ import { inspectObj, visibleLog, ajvCompile, memoize, objectHasKey } from "#utils/utils.js";
7
+ import { IIIF_PRESENTATION_2_CONTEXT } from "#utils/iiifUtils.js";
8
+ import { PUBLIC_URL } from "#constants";
9
+
10
+ /** @typedef {import("#types").FastifyInstanceType} FastifyInstanceType */
11
+ /** @typedef {import("#types").MongoObjectId} MongoObjectId */
12
+ /** @typedef {import("#types").MongoInsertResultType} MongoInsertResultType */
13
+ /** @typedef {import("#types").MongoUpdateResultType} MongoUpdateResultType */
14
+ /** @typedef {import("#types").MongoDeleteResultType} MongoDeleteResultType */
15
+ /** @typedef {import("#types").InsertResponseType} InsertResponseType */
16
+ /** @typedef {import("#types").UpdateResponseType} UpdateResponseType */
17
+ /** @typedef {import("#types").DeleteResponseType} DeleteResponseType */
18
+ /** @typedef {import("#types").DataOperationsType } DataOperationsType */
19
+ /** @typedef {import("#types").AnnotationsDeleteKeyType } AnnotationsDeleteKeyType */
20
+ /** @typedef {import("#types").Manifest2InternalType } Manifest2InternalType */
21
+ /** @typedef {import("#types").AjvValidateFunctionType} AjvValidateFunctionType */
22
+ /** @typedef {import("#types").IiifCollection2Type} IiifCollection2Type */
23
+
24
+ /** @typedef {Manifests2} Manifests2InstanceType */
25
+
26
+
27
+ /**
28
+ * @class
29
+ * @constructor
30
+ * @public
31
+ * @extends {CollectionAbstract}
32
+ */
33
+ class Manifests2 extends CollectionAbstract {
34
+ /**
35
+ * @param {FastifyInstanceType} fastify
36
+ */
37
+ constructor(fastify) {
38
+ super(fastify, "manifests2");
39
+
40
+ /** @type {AjvValidateFunctionType} */
41
+ this.validatorManifest = ajvCompile(fastify.schemasResolver(
42
+ fastify.schemasPresentation2.getSchema("manifestPublic")
43
+ ));
44
+ }
45
+
46
+ /////////////////////////////////////////////
47
+ // utils
48
+
49
+ /**
50
+ * fetch the array of canvasIds for a single manifest.
51
+ * since, in AIKON after a RegionExtraction, an annotation insert
52
+ * is done once per canvas, and for each annotation insert, we
53
+ * fetch the canvas index, we memoize the canvas list for each manifest URI
54
+ * to avoid multiplying database calls.
55
+ * @type {(string) => Promise<string[]>}
56
+ */
57
+ #memoizeGetManifestCanvasIds = memoize(async (manifestUri) => {
58
+ const doc = await this.collection
59
+ .findOne(
60
+ { "@id": manifestUri },
61
+ { projection: { canvasIds: 1, _id: 0 } } // findOne is enough, there's only one manifest per URI
62
+ );
63
+ return doc?.canvasIds ?? [];
64
+ }, 60_000);
65
+
66
+ /**
67
+ * NOTE: PERFORMANCE: using AJV validation is MUCH FASTER than doing manual verifications (-25% execution time for the test suite)
68
+ * @param {object} manifest
69
+ * @returns {void}
70
+ */
71
+ #validateManifest(manifest) {
72
+ if (!this.validatorManifest(manifest)) {
73
+ let info = {};
74
+ if (objectHasKey(manifest, "@id")) {
75
+ info = { "@id": manifest["@id"] };
76
+ }
77
+ throw this.insertError("validateManifest: invalid manifest structure", info);
78
+ }
79
+ }
80
+
81
+ /**
82
+ * convert a manifest to internal data model
83
+ *
84
+ * NOTE: there is no need to reprocess "@id"s: the whole manifest is not stored in the database, so we need to keep all urls and references to external data intact.
85
+ * NOTE: only the 1st (default) sequence is processed. other optional sequences MUST be referenced from the manifest (not embedded). in practice, they are rare, so we don´t process them.
86
+ * see: https://iiif.io/api/presentation/2.1/#sequence
87
+ *
88
+ * @param {object} manifest
89
+ * @returns {Manifest2InternalType}
90
+ */
91
+ #cleanManifest(manifest) {
92
+ return {
93
+ "@id": manifest["@id"],
94
+ "@type": "sc:Manifest",
95
+ manifestShortId: getManifestShortId(manifest["@id"]),
96
+ canvasIds: manifest.sequences[0].canvases.map((canvas) => canvas["@id"])
97
+ };
98
+ }
99
+
100
+ /**
101
+ * validation + cleaning pipeline
102
+ * @param {object} manifest
103
+ * @returns {object}
104
+ */
105
+ #validateAndCleanManifest(manifest) {
106
+ this.#validateManifest(manifest);
107
+ return this.#cleanManifest(manifest);
108
+ }
109
+
110
+ /**
111
+ * fetch a manifest and return it as an object
112
+ * @param {string} manifestUri
113
+ * @returns
114
+ */
115
+ async #fetchManifestFromUri(manifestUri) {
116
+ console.log(">>>> MANIFEST URI", manifestUri);
117
+ try {
118
+ const r = await fetch(manifestUri);
119
+ return await r.json();
120
+ } catch (err) {
121
+ throw this.insertError(`error fetching manifest with URI '${manifestUri}'`);
122
+ }
123
+ }
124
+
125
+ /////////////////////////////////////////////
126
+ // write
127
+
128
+ /**
129
+ * save a single manifest to database.
130
+ * @param {object} manifest - a IIIF manifest
131
+ * @returns {Promise<InsertResponseType>}
132
+ */
133
+ async insertManifest(manifest) {
134
+ manifest = this.#validateAndCleanManifest(manifest);
135
+ const manifestExists = await this.exists({ "@id": manifest["@id"] });
136
+ if (!manifestExists) {
137
+ return this.insertOne(manifest);
138
+ } else {
139
+ return formatInsertResponse({ preExistingIds: [ manifest["@id"] ] });
140
+ }
141
+ }
142
+
143
+ /**
144
+ * insert several manifests to database
145
+ * if `throwOnError===false`, don't raise if there is an error: instead, insert as many documents as possible.
146
+ * see the docs of `insertManifestsFromUriArray` for more info.
147
+ *
148
+ * @param {object[]} manifestArray - array of manifests
149
+ * @returns {Promise<InsertResponseType>}
150
+ */
151
+ async insertManifestArray(manifestArray, throwOnError=true) {
152
+ // build 2 arrays, one of the manifests that pass validation, one of the @ids of the manifests with errors, mapped to an error message.
153
+ let
154
+ cleanManifestArray = [],
155
+ invalidManifestArray = [],
156
+ preExistingIds = [];
157
+ manifestArray.map((manifest) => {
158
+ try {
159
+ cleanManifestArray.push(this.#validateAndCleanManifest(manifest));
160
+ } catch (err) {
161
+ if (throwOnError) {
162
+ throw err;
163
+ }
164
+ // returns a mapping of `{manifestUri: errorMessage}`
165
+ invalidManifestArray.push({ [manifest["@id"]]: err.message });
166
+ }
167
+ });
168
+
169
+ // filter out the manifests that are aldready in our collection
170
+ // NOTE: i have attempted to move this to `insertManifestsFromUriArray` but it leads to what I suspect is a data race causing unique constaints fails.
171
+ // TLDR: don't move or disable this check.
172
+ let cleanIds = cleanManifestArray.map((manifest) => manifest["@id"]);
173
+ [ cleanIds, preExistingIds ] = await this.#filterManifestIdsInCollection(cleanIds);
174
+
175
+ // remove pre-inserted IDs of manifests from cleanManifestArray
176
+ cleanManifestArray = cleanManifestArray.filter((manifest) => cleanIds.includes(manifest["@id"]));
177
+
178
+ // insert. if there has been an error but throwOnError === "false", complete the response object with description of the errors
179
+ // no need for try..except, no errors should happen here.
180
+ if (cleanManifestArray.length) {
181
+ const result = await this.insertMany(cleanManifestArray);
182
+ result.preExistingIds = preExistingIds;
183
+ result.rejectedIds = invalidManifestArray;
184
+ return result;
185
+
186
+ } else {
187
+ return formatInsertResponse({
188
+ preExistingIds: preExistingIds,
189
+ rejectedIds: invalidManifestArray
190
+ });
191
+ }
192
+ }
193
+
194
+ /**
195
+ * insert a manifest from an URI
196
+ * @param {string} manifestUri
197
+ * @returns {Promise<InsertResponseType>}
198
+ */
199
+ async insertManifestFromUri(manifestUri) {
200
+ try {
201
+ const manifest = await this.#fetchManifestFromUri(manifestUri);
202
+ return this.insertManifest(manifest);
203
+ } catch (err) {
204
+ throw this.insertError(`error inserting manifest with URI '${manifestUri}' because of error: ${err.message}`);
205
+ }
206
+ }
207
+
208
+ /**
209
+ * from an array of URIs, insert many manifests.
210
+ *
211
+ * if `throwOnError===false`, the function will not throw. instead, it will try to insert as much as possible, and the response will give info on the failure cases.
212
+ * the first use case for this behaviour is to index all manifests related to an array of annotations. given that we reconstruct
213
+ * manifest URIs from canvas URIs manually, there may always be an error. we want to insert a manifest when possible, and return an error othersise.
214
+ *
215
+ * @param {string[]} manifestUriArray
216
+ * @param {boolean} throwOnError
217
+ * @returns {Promise<InsertResponseType>}
218
+ */
219
+ async insertManifestsFromUriArray(manifestUriArray, throwOnError=true) {
220
+ // PERFORMANCE ~2850ms
221
+ const
222
+ fetchErrorIds = [],
223
+ manifestArray = [];
224
+
225
+ // fetch the manifests. if there's a fetch error, they won't be inserted.
226
+ await Promise.all(
227
+ manifestUriArray.map(async (manifestUri) => {
228
+ try {
229
+ const r = await this.#fetchManifestFromUri(manifestUri);
230
+ if (! r.error) {
231
+ manifestArray.push(r);
232
+ } else {
233
+ fetchErrorIds.push(manifestUri);
234
+ }
235
+ } catch (err) {
236
+ if (throwOnError) {
237
+ throw err;
238
+ }
239
+ fetchErrorIds.push(manifestUri);
240
+ }
241
+ })
242
+ );
243
+
244
+ if (fetchErrorIds.length){
245
+ const errMsg = `error inserting ${fetchErrorIds.length} manifests: ${fetchErrorIds}`
246
+ if (throwOnError) {
247
+ throw this.insertError(errMsg)
248
+ } else if (fetchErrorIds.length) {
249
+ this.fastify.log.error(errMsg, fetchErrorIds);
250
+ }
251
+ }
252
+
253
+ // insert and format response
254
+ const result = await this.insertManifestArray(manifestArray, throwOnError, true);
255
+ result.fetchErrorIds = fetchErrorIds;
256
+ return result;
257
+ }
258
+
259
+ /////////////////////////////////////////////
260
+ // delete
261
+
262
+ /**
263
+ * @param {"manifestShortId"|"uri"} deleteKey = what deleteId describes: a manifest URI or its short ID
264
+ * @param {string} deleteVal - data to delete
265
+ * @returns {Promise<DeleteResponseType>}
266
+ */
267
+ // NOTE: could be refactored with `annotations2.delete`: both functions are the same, only the filter changes
268
+ async deleteManifest(deleteKey, deleteVal) {
269
+ const allowedDeleteKey = [ "uri", "manifestShortId" ];
270
+ if (!allowedDeleteKey.includes(deleteKey)) {
271
+ throw this.deleteError(`${this.funcName(this.deleteManifest)}: expected one of ${allowedDeleteKey}, got '${deleteKey}'`);
272
+ }
273
+
274
+ const deleteFilter =
275
+ deleteKey==="uri"
276
+ ? { "@id": deleteVal }
277
+ : { manifestShortId: deleteVal };
278
+
279
+ return this.delete(deleteFilter);
280
+ }
281
+
282
+ /////////////////////////////////////////////
283
+ // read
284
+
285
+ /**
286
+ * find which manifests in an array of manifest IDs are aldready in the manifests collection
287
+ * @param {string[]} manifestUriArray - array of manifest IDs
288
+ * @returns {Promise<Array<Array<string?>>>} 2 arrays:
289
+ * - array of manifest IDs that are not aldready in this collection
290
+ * - array of manifest IDs that are not in this collection
291
+ */
292
+ async #filterManifestIdsInCollection(manifestUriArray) {
293
+ const
294
+ preExistingIds = [],
295
+ toInsertIds = [];
296
+ // all manifest IDs from manifestUriArray that are aldready in our manifest collection
297
+ const inCollectionArray = (
298
+ await this.collection.find(
299
+ { "@id": { $in: manifestUriArray } },
300
+ { projection: { "@id": 1 } }
301
+ ).toArray()
302
+ ).map((r) => r["@id"]);
303
+ // split manifestUriArray in 2 lists.
304
+ manifestUriArray.map((manifestUri) =>
305
+ inCollectionArray.includes(manifestUri)
306
+ ? preExistingIds.push(manifestUri)
307
+ : toInsertIds.push(manifestUri)
308
+ );
309
+ return [ toInsertIds, preExistingIds ];
310
+ }
311
+
312
+ /**
313
+ * return the position of `canvasUri` within the manifest with ID `manifestUri`,
314
+ * or return `undefined` if the canvas is not found in the manifest, or the manifest is not indexed.
315
+ * @param {string} manifestUri
316
+ * @param {string} canvasUri
317
+ * @returns {Promise<number?>}
318
+ */
319
+ async getCanvasIdx(manifestUri, canvasUri) {
320
+ // old method without memoization.
321
+ // - with `aggregate`, ~2800ms for the whole test suite to run.
322
+ // - with a native `coll.findOne()` and then getting the canvas ID manually (`arr.indexOf`), ~4000ms for the whole test suite to run.
323
+ // https://www.mongodb.com/docs/manual/aggregation/
324
+ // https://www.mongodb.com/docs/manual/reference/operator/aggregation/indexOfArray/
325
+ // const r = await this.collection.aggregate([
326
+ // { $match: { "@id": manifestUri } },
327
+ // { $project: { index: { $indexOfArray: ["$canvasIds", canvasUri] } } }
328
+ // ]).next();
329
+ const r = await this.#memoizeGetManifestCanvasIds(manifestUri);
330
+ const index = r.indexOf(canvasUri);
331
+ return index !== -1 ? index : undefined
332
+ }
333
+
334
+ /**
335
+ * return a collection of all manifests in the database.
336
+ * @returns {Promise<IiifCollection2Type>}
337
+ */
338
+ async getManifests() {
339
+
340
+ const manifestIndex = await this.collection.find(
341
+ {},
342
+ { projection: { "@id": 1, "@type": 1, _id: 0 } }
343
+ ).toArray();
344
+ return {
345
+ ...IIIF_PRESENTATION_2_CONTEXT,
346
+ "@type": "sc:Collection",
347
+ "@id": `${PUBLIC_URL}/manifests/2`,
348
+ label: "Collection of all manifests indexed in the annotation server",
349
+ members: manifestIndex
350
+ }
351
+ }
352
+
353
+ /**
354
+ * @param {string} manifestShortId
355
+ * @returns {object}
356
+ */
357
+ async findByManifestShortId(manifestShortId) {
358
+ try {
359
+ const manifestIndexArray = await this.collection
360
+ .find(
361
+ { manifestShortId: manifestShortId },
362
+ { limit: 1, project: { _id: 0 } }
363
+ )
364
+ .limit(1);
365
+ return manifestIndexArray.length ? manifestIndexArray[1] : {}
366
+ } catch (err) {
367
+ throw this.readError(`${this.funcName(this.findByManifestShortId)}: error fetching manifest with manifestShortId: '${manifestShortId}'`)
368
+ }
369
+ }
370
+ }
371
+
372
+ export default fastifyPlugin((fastify, options, done) => {
373
+ fastify.decorate("manifests2", new Manifests2(fastify));
374
+ done();
375
+ }, {
376
+ name: "manifests2",
377
+ })
378
+