arocapi 5.0.0-beta.2 → 5.0.0-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -824
- package/dist/app.d.ts +20 -7
- package/dist/app.js +42 -28
- package/dist/app.js.map +1 -1
- package/dist/app.test.js +41 -56
- package/dist/app.test.js.map +1 -1
- package/dist/errorHandling.test.d.ts +1 -0
- package/dist/errorHandling.test.js +222 -0
- package/dist/errorHandling.test.js.map +1 -0
- package/dist/index.dev.js +4 -3
- package/dist/index.dev.js.map +1 -1
- package/dist/routes/capabilities.test.js +8 -8
- package/dist/routes/capabilities.test.js.map +1 -1
- package/dist/routes/crate.test.js +67 -19
- package/dist/routes/crate.test.js.map +1 -1
- package/dist/routes/entities.d.ts +5 -2
- package/dist/routes/entities.js +31 -49
- package/dist/routes/entities.js.map +1 -1
- package/dist/routes/entities.test.js +16 -12
- package/dist/routes/entities.test.js.map +1 -1
- package/dist/routes/entity.d.ts +7 -2
- package/dist/routes/entity.js +14 -29
- package/dist/routes/entity.js.map +1 -1
- package/dist/routes/entity.test.js +19 -13
- package/dist/routes/entity.test.js.map +1 -1
- package/dist/routes/file.d.ts +6 -0
- package/dist/routes/file.js +67 -59
- package/dist/routes/file.js.map +1 -1
- package/dist/routes/file.test.js +99 -22
- package/dist/routes/file.test.js.map +1 -1
- package/dist/routes/files.d.ts +5 -2
- package/dist/routes/files.js +24 -39
- package/dist/routes/files.js.map +1 -1
- package/dist/routes/files.test.js +44 -21
- package/dist/routes/files.test.js.map +1 -1
- package/dist/routes/metadata.d.ts +6 -0
- package/dist/routes/metadata.js +49 -54
- package/dist/routes/metadata.js.map +1 -1
- package/dist/routes/search.d.ts +6 -2
- package/dist/routes/search.js +95 -99
- package/dist/routes/search.js.map +1 -1
- package/dist/routes/search.test.js +85 -39
- package/dist/routes/search.test.js.map +1 -1
- package/dist/test/helpers/fastify.js +2 -3
- package/dist/test/helpers/fastify.js.map +1 -1
- package/dist/test/integration.setup.d.ts +18 -0
- package/dist/test/integration.setup.js +15 -7
- package/dist/test/integration.setup.js.map +1 -1
- package/dist/test/integration.test.js +368 -2
- package/dist/test/integration.test.js.map +1 -1
- package/dist/transformers/default.d.ts +10 -18
- package/dist/transformers/default.js +41 -15
- package/dist/transformers/default.js.map +1 -1
- package/dist/transformers/default.test.js +109 -47
- package/dist/transformers/default.test.js.map +1 -1
- package/dist/transformers/transformer.test.js +12 -34
- package/dist/transformers/transformer.test.js.map +1 -1
- package/dist/types/access.d.ts +9 -0
- package/dist/types/access.js +2 -0
- package/dist/types/access.js.map +1 -0
- package/dist/types/context.d.ts +6 -0
- package/dist/types/context.js +2 -0
- package/dist/types/context.js.map +1 -0
- package/dist/types/fileHandlers.d.ts +5 -9
- package/dist/types/transformers.d.ts +4 -10
- package/dist/types/visibility.d.ts +5 -0
- package/dist/types/visibility.js +2 -0
- package/dist/types/visibility.js.map +1 -0
- package/dist/utils/access.d.ts +17 -0
- package/dist/utils/access.js +41 -0
- package/dist/utils/access.js.map +1 -0
- package/dist/utils/errors.d.ts +13 -4
- package/dist/utils/errors.js +44 -8
- package/dist/utils/errors.js.map +1 -1
- package/dist/utils/errors.test.js +14 -1
- package/dist/utils/errors.test.js.map +1 -1
- package/dist/utils/validation.d.ts +2 -0
- package/dist/utils/validation.js +8 -0
- package/dist/utils/validation.js.map +1 -0
- package/dist/utils/visibility.d.ts +18 -0
- package/dist/utils/visibility.js +16 -0
- package/dist/utils/visibility.js.map +1 -0
- package/package.json +5 -4
- package/src/app.test.ts +45 -66
- package/src/app.ts +85 -54
- package/src/errorHandling.test.ts +288 -0
- package/src/index.dev.ts +6 -4
- package/src/routes/__snapshots__/search.test.ts.snap +2 -2
- package/src/routes/capabilities.test.ts +8 -8
- package/src/routes/crate.test.ts +80 -19
- package/src/routes/entities.test.ts +16 -12
- package/src/routes/entities.ts +40 -59
- package/src/routes/entity.test.ts +19 -13
- package/src/routes/entity.ts +22 -35
- package/src/routes/file.test.ts +117 -22
- package/src/routes/file.ts +84 -62
- package/src/routes/files.test.ts +52 -26
- package/src/routes/files.ts +31 -47
- package/src/routes/metadata.ts +64 -58
- package/src/routes/search.test.ts +88 -39
- package/src/routes/search.ts +123 -119
- package/src/test/helpers/fastify.ts +2 -3
- package/src/test/integration.setup.ts +18 -7
- package/src/test/integration.test.ts +472 -1
- package/src/transformers/default.test.ts +139 -47
- package/src/transformers/default.ts +103 -83
- package/src/transformers/transformer.test.ts +18 -44
- package/src/types/access.ts +20 -0
- package/src/types/context.ts +15 -0
- package/src/types/fileHandlers.ts +5 -13
- package/src/types/transformers.ts +4 -32
- package/src/types/visibility.ts +18 -0
- package/src/utils/access.ts +80 -0
- package/src/utils/errors.test.ts +16 -1
- package/src/utils/errors.ts +67 -10
- package/src/utils/validation.ts +11 -0
- package/src/utils/visibility.ts +56 -0
package/README.md
CHANGED
|
@@ -1,840 +1,53 @@
|
|
|
1
|
-
#
|
|
1
|
+
# arocapi
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
[PILARS](https://w3id.org/ldac/pilars).
|
|
3
|
+
arocapi implements the [RO-Crate API](https://ro-crate-api.crate-works.org) for data collections, packaged as a
|
|
4
|
+
Fastify plugin. You mount it into your own Fastify (or Express) app and bring your storage, search index and access
|
|
5
|
+
rules; arocapi serves the routes for entities, files, search and RO-Crate metadata, following the
|
|
6
|
+
[PILARS](https://w3id.org/ldac/pilars) protocols for long-term archival repositories.
|
|
8
7
|
|
|
9
|
-
|
|
10
|
-
Fastify app, or into an Express app via Fastify (see
|
|
11
|
-
[Using with Express](#using-with-express)).
|
|
8
|
+
**Documentation: <https://arocapi.crate-works.org>**
|
|
12
9
|
|
|
13
|
-
|
|
14
|
-
|
|
10
|
+
- [What is arocapi](https://arocapi.crate-works.org/introduction/what-is-arocapi/)
|
|
11
|
+
- [Getting started](https://arocapi.crate-works.org/getting-started/install/)
|
|
12
|
+
- [Configuration](https://arocapi.crate-works.org/configuration/data-sources/)
|
|
13
|
+
- [Routes](https://arocapi.crate-works.org/reference/routes/)
|
|
15
14
|
|
|
16
|
-
|
|
17
|
-
- [OCFL] - the Oxford Common File Layout specification on disk or cloud-based
|
|
18
|
-
object storage
|
|
19
|
-
- Object Storage as used in [PARADISEC]
|
|
20
|
-
- A simple RO-Crate file layout (TODO - likely a super-simple protocol for
|
|
21
|
-
directory hierarchy with the presence on RO-Crate used to indicate that
|
|
22
|
-
everything under that directory is a single RO-Crate)
|
|
23
|
-
- Authorisation based on the concept of Access (and Deposit, TODO) Licenses
|
|
24
|
-
using an arms-length process with an external authority such as a REMS
|
|
25
|
-
instance or other license/group management system
|
|
26
|
-
- Based on the principle that all data MUST have a implementation-neutral
|
|
27
|
-
natural-language description of its access conditions stored with it
|
|
28
|
-
- Licenses may be based on 'traditional' access control lists maintained in
|
|
29
|
-
software such as PARADISEC or a license manager such as REMS, (TODO: an
|
|
30
|
-
example implementation using GitHub groups to manage license access, a
|
|
31
|
-
simple ACL manager?)).
|
|
15
|
+
## Install
|
|
32
16
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
- Node.js (version 22 or higher)
|
|
36
|
-
- pnpm package manager
|
|
37
|
-
- Database: MySQL or PostgreSQL
|
|
38
|
-
- OpenSearch for search functionality
|
|
39
|
-
- Docker and Docker Compose (for development)
|
|
40
|
-
|
|
41
|
-
## Quick Start
|
|
42
|
-
|
|
43
|
-
### 1. Installation
|
|
44
|
-
|
|
45
|
-
```bash
|
|
46
|
-
# Install the package
|
|
17
|
+
```sh
|
|
47
18
|
pnpm add arocapi
|
|
48
|
-
|
|
49
|
-
# Install required peer dependencies
|
|
50
|
-
pnpm add @prisma/client @opensearch-project/opensearch
|
|
51
|
-
pnpm add -D prisma
|
|
52
|
-
```
|
|
53
|
-
|
|
54
|
-
Changes merged to `main` are published as prereleases under the `beta` tag: `pnpm add arocapi@beta`.
|
|
55
|
-
|
|
56
|
-
### 2. Environment Setup
|
|
57
|
-
|
|
58
|
-
Create a `.env` file in your project root:
|
|
59
|
-
|
|
60
|
-
```env
|
|
61
|
-
# Database Configuration (You can use an DB that Prisma supports)
|
|
62
|
-
DATABASE_URL="mysql://root:password@localhost:3306/catalog"
|
|
63
|
-
|
|
64
|
-
# OpenSearch Configuration
|
|
65
|
-
OPENSEARCH_URL="http://localhost:9200"
|
|
66
|
-
```
|
|
67
|
-
|
|
68
|
-
## Creating Your API
|
|
69
|
-
|
|
70
|
-
Create your Fastify application with Typescript support:
|
|
71
|
-
|
|
72
|
-
```typescript
|
|
73
|
-
// src/index.ts
|
|
74
|
-
import { Client } from '@opensearch-project/opensearch';
|
|
75
|
-
import arocapi, { AllPublicAccessTransformer, AllPublicFileAccessTransformer } from 'arocapi';
|
|
76
|
-
import Fastify from 'fastify';
|
|
77
|
-
import { PrismaClient } from './generated/prisma/client.js';
|
|
78
|
-
import { Readable } from 'stream';
|
|
79
|
-
|
|
80
|
-
const prisma = new PrismaClient();
|
|
81
|
-
|
|
82
|
-
if (!process.env.OPENSEARCH_URL) {
|
|
83
|
-
throw new Error('OPENSEARCH_URL environment variable is not set');
|
|
84
|
-
}
|
|
85
|
-
const opensearchUrl = process.env.OPENSEARCH_URL;
|
|
86
|
-
const opensearch = new Client({ node: opensearchUrl });
|
|
87
|
-
|
|
88
|
-
const fastify = Fastify({
|
|
89
|
-
logger: true,
|
|
90
|
-
});
|
|
91
|
-
|
|
92
|
-
// For fully public datasets
|
|
93
|
-
fastify.register(arocapi, {
|
|
94
|
-
prisma,
|
|
95
|
-
opensearch,
|
|
96
|
-
accessTransformer: AllPublicAccessTransformer,
|
|
97
|
-
fileAccessTransformer: AllPublicFileAccessTransformer,
|
|
98
|
-
// Required: File handler for serving File entity content
|
|
99
|
-
fileHandler: {
|
|
100
|
-
get: async (file) => {
|
|
101
|
-
const fileUrl = `https://storage.example.com/${file.meta.storagePath}`;
|
|
102
|
-
return { type: 'redirect', url: fileUrl };
|
|
103
|
-
},
|
|
104
|
-
head: async (file) => ({
|
|
105
|
-
contentType: file.mediaType,
|
|
106
|
-
contentLength: file.size,
|
|
107
|
-
}),
|
|
108
|
-
},
|
|
109
|
-
// Required: RO-Crate handler for serving RO-Crate metadata
|
|
110
|
-
roCrateHandler: {
|
|
111
|
-
get: async (entity) => {
|
|
112
|
-
const jsonString = JSON.stringify(entity.rocrate, null, 2);
|
|
113
|
-
return {
|
|
114
|
-
type: 'stream',
|
|
115
|
-
stream: Readable.from([jsonString]),
|
|
116
|
-
metadata: {
|
|
117
|
-
contentType: 'application/ld+json',
|
|
118
|
-
contentLength: Buffer.byteLength(jsonString),
|
|
119
|
-
},
|
|
120
|
-
};
|
|
121
|
-
},
|
|
122
|
-
head: async (entity) => ({
|
|
123
|
-
contentType: 'application/ld+json',
|
|
124
|
-
contentLength: Buffer.byteLength(JSON.stringify(entity.rocrate)),
|
|
125
|
-
}),
|
|
126
|
-
},
|
|
127
|
-
});
|
|
128
|
-
|
|
129
|
-
try {
|
|
130
|
-
await fastify.listen({ port: 3000 });
|
|
131
|
-
} catch (err) {
|
|
132
|
-
fastify.log.error(err);
|
|
133
|
-
process.exit(1);
|
|
134
|
-
}
|
|
135
|
-
```
|
|
136
|
-
|
|
137
|
-
### CORS
|
|
138
|
-
|
|
139
|
-
arocapi doesn't set CORS headers; CORS is up to your application. To let
|
|
140
|
-
browsers on other origins call the API, register
|
|
141
|
-
[`@fastify/cors`](https://github.com/fastify/fastify-cors) on the root
|
|
142
|
-
instance:
|
|
143
|
-
|
|
144
|
-
```bash
|
|
145
|
-
pnpm add @fastify/cors
|
|
146
|
-
```
|
|
147
|
-
|
|
148
|
-
```typescript
|
|
149
|
-
import cors from '@fastify/cors';
|
|
150
|
-
|
|
151
|
-
fastify.register(cors);
|
|
152
|
-
fastify.register(arocapi, { /* ... */ });
|
|
153
|
-
```
|
|
154
|
-
|
|
155
|
-
The defaults allow all origins with `GET`, `HEAD` and `POST`, matching what
|
|
156
|
-
arocapi registered before 5.0.0. See the `@fastify/cors` docs to restrict them.
|
|
157
|
-
|
|
158
|
-
## Database Setup with Prisma
|
|
159
|
-
|
|
160
|
-
### 1. Initialize Prisma
|
|
161
|
-
|
|
162
|
-
```bash
|
|
163
|
-
# Initialize prisma with your preferred database provider
|
|
164
|
-
npx prisma init --datasource-provider mysql
|
|
165
|
-
# or for PostgreSQL: npx prisma init --datasource-provider postgresql
|
|
166
|
-
```
|
|
167
|
-
|
|
168
|
-
### 2. Configure Prisma Multi-Model Setup
|
|
169
|
-
|
|
170
|
-
Modify the `prisma.config.ts` file in your project root to support folder based schemas:
|
|
171
|
-
|
|
172
|
-
```typescript
|
|
173
|
-
// prisma.config.ts
|
|
174
|
-
import 'dotenv/config';
|
|
175
|
-
import { defineConfig } from 'prisma/config';
|
|
176
|
-
|
|
177
|
-
export default defineConfig({
|
|
178
|
-
// Adds support for folder based schemas
|
|
179
|
-
schema: 'prisma',
|
|
180
|
-
});
|
|
181
|
-
```
|
|
182
|
-
|
|
183
|
-
### 3. Set Up Model Structure
|
|
184
|
-
|
|
185
|
-
Create the directory structure and link to arocapi models:
|
|
186
|
-
|
|
187
|
-
```bash
|
|
188
|
-
# Create prisma models directory
|
|
189
|
-
mkdir -p prisma/models
|
|
190
|
-
|
|
191
|
-
# Link to the upstream arocapi models
|
|
192
|
-
ln -s ../../node_modules/arocapi/prisma/models prisma/arocapi
|
|
193
|
-
```
|
|
194
|
-
|
|
195
|
-
### 4. Generate Prisma Client and Run Migrations
|
|
196
|
-
|
|
197
|
-
```bash
|
|
198
|
-
# Create database migrations for this version of arocapi
|
|
199
|
-
npx prisma migrate dev --name="arocapi-$(pnpm list | grep arocapi | awk '{ print $2 }')"
|
|
200
|
-
|
|
201
|
-
# Generate the Prisma client
|
|
202
|
-
npx prisma generate
|
|
203
|
-
|
|
204
|
-
```
|
|
205
|
-
|
|
206
|
-
## Available API Routes
|
|
207
|
-
|
|
208
|
-
The arocapi provides the following endpoints:
|
|
209
|
-
|
|
210
|
-
- `GET /capabilities` - Declare the spec version and supported search filters and facets (always public)
|
|
211
|
-
- `GET /entities` - List all entities with pagination and filtering
|
|
212
|
-
- `GET /entity/:id` - Get a specific entity by ID
|
|
213
|
-
- `GET /entity/:id/metadata` - Download RO-Crate metadata
|
|
214
|
-
- `GET /files` - List files with pagination and filtering
|
|
215
|
-
- `GET /file/:id` - Download or access file content
|
|
216
|
-
- `POST /search` - Search entities using OpenSearch
|
|
217
|
-
|
|
218
|
-
## Customising OpenSearch query
|
|
219
|
-
|
|
220
|
-
The OpenSearch query builder used in the `/search` endpoint can be customised to match any
|
|
221
|
-
structure of the actual OpenSearch data. The default OpensearchQueryBuilder class is used to
|
|
222
|
-
build all the queries for the search and an optional config can be passed on
|
|
223
|
-
to customise the class behaviour.
|
|
224
|
-
|
|
225
|
-
### Query Builder (Optional)
|
|
226
|
-
Use the `queryBuilderClass` optional parameter to specify a custom builder class.
|
|
227
|
-
If not specified, the default `OpensearchQueryBuilder` is used.
|
|
228
|
-
The custom class should extend the `OpensearchQueryBuilder` class and
|
|
229
|
-
may override any or all of the methods as required:
|
|
230
|
-
|
|
231
|
-
- `buildQuery`: For generating the base search query.
|
|
232
|
-
- `buildAggregations`: For generating the facets or aggregations related query.
|
|
233
|
-
- `buildSort`: For generating the ordering/sorting part of the query.
|
|
234
|
-
- `searchCapabilities`: For declaring the search filters and facets in `GET /capabilities`.
|
|
235
|
-
|
|
236
|
-
By default `searchCapabilities` declares each `terms` aggregation as a `string`
|
|
237
|
-
filter and a facet. Other aggregation types are not declared. Override it to add
|
|
238
|
-
labels or declare filter-only fields:
|
|
239
|
-
|
|
240
|
-
```typescript
|
|
241
|
-
import { OpensearchQueryBuilder, type SearchCapability } from 'arocapi';
|
|
242
|
-
|
|
243
|
-
class MyQueryBuilder extends OpensearchQueryBuilder {
|
|
244
|
-
searchCapabilities(): SearchCapability {
|
|
245
|
-
return {
|
|
246
|
-
filters: {
|
|
247
|
-
inLanguage: { type: 'string', label: 'Language' },
|
|
248
|
-
dateCreated: { type: 'date', label: 'Date created' },
|
|
249
|
-
},
|
|
250
|
-
facets: { inLanguage: { label: 'Language' } },
|
|
251
|
-
};
|
|
252
|
-
}
|
|
253
|
-
}
|
|
254
|
-
```
|
|
255
|
-
|
|
256
|
-
### Query Builder Options (Optional)
|
|
257
|
-
The `queryBuilderOptions` optional parameter is passed as a constructor argument
|
|
258
|
-
when instantiating the queryBuilderClass. The default `OpensearchQueryBuilder` can be
|
|
259
|
-
configured with:
|
|
260
|
-
|
|
261
|
-
- `aggregations`: This field controls the aggregation and will be passed verbatim as the `aggs` field in the generated search query.
|
|
262
|
-
|
|
263
|
-
## Customising Entity Responses
|
|
264
|
-
|
|
265
|
-
The API provides a flexible transformer system for customising entity responses
|
|
266
|
-
through two types of transformers:
|
|
267
|
-
|
|
268
|
-
### Access Transformer (Required)
|
|
269
|
-
|
|
270
|
-
The `accessTransformer` parameter is **required** for security. You must explicitly
|
|
271
|
-
choose how access control is handled for your repository.
|
|
272
|
-
|
|
273
|
-
**For fully public datasets**, use `AllPublicAccessTransformer`:
|
|
274
|
-
|
|
275
|
-
```typescript
|
|
276
|
-
import arocapi, { AllPublicAccessTransformer } from 'arocapi';
|
|
277
|
-
|
|
278
|
-
await server.register(arocapi, {
|
|
279
|
-
prisma,
|
|
280
|
-
opensearch,
|
|
281
|
-
accessTransformer: AllPublicAccessTransformer,
|
|
282
|
-
});
|
|
283
|
-
```
|
|
284
|
-
|
|
285
|
-
**For restricted content**, implement a custom access transformer:
|
|
286
|
-
|
|
287
|
-
```typescript
|
|
288
|
-
const accessTransformer = async (entity, { request }) => {
|
|
289
|
-
// Custom logic to determine access
|
|
290
|
-
const user = await authenticateUser(request);
|
|
291
|
-
const canAccessContent = await checkLicense(entity.contentLicenseId, user);
|
|
292
|
-
|
|
293
|
-
return {
|
|
294
|
-
...entity,
|
|
295
|
-
access: {
|
|
296
|
-
metadata: true, // Metadata always visible
|
|
297
|
-
content: canAccessContent,
|
|
298
|
-
contentAuthorisationUrl: canAccessContent
|
|
299
|
-
? undefined
|
|
300
|
-
: 'https://rems.example.com/request-access',
|
|
301
|
-
},
|
|
302
|
-
};
|
|
303
|
-
};
|
|
304
|
-
```
|
|
305
|
-
|
|
306
|
-
> [!WARNING]
|
|
307
|
-
> The `accessTransformer` is required to prevent accidental exposure of
|
|
308
|
-
> restricted content. You must make an explicit choice about access control for
|
|
309
|
-
> your repository.
|
|
310
|
-
|
|
311
|
-
### Entity Transformers
|
|
312
|
-
|
|
313
|
-
Optional transformations for enriching or modifying response data. Multiple
|
|
314
|
-
transformers can be chained together.
|
|
315
|
-
|
|
316
|
-
```typescript
|
|
317
|
-
await server.register(arocapi, {
|
|
318
|
-
prisma,
|
|
319
|
-
opensearch,
|
|
320
|
-
accessTransformer: AllPublicAccessTransformer,
|
|
321
|
-
entityTransformers: [
|
|
322
|
-
// Add computed fields
|
|
323
|
-
(entity) => ({
|
|
324
|
-
...entity,
|
|
325
|
-
displayName: `${entity.name} [${entity.entityType.split('/').pop()}]`,
|
|
326
|
-
}),
|
|
327
|
-
// Add counts
|
|
328
|
-
async (entity) => {
|
|
329
|
-
const objectCount = entity.memberOf
|
|
330
|
-
? await prisma.entity.count({
|
|
331
|
-
where: { memberOf: entity.id },
|
|
332
|
-
})
|
|
333
|
-
: 0;
|
|
334
|
-
|
|
335
|
-
return {
|
|
336
|
-
...entity,
|
|
337
|
-
counts: {
|
|
338
|
-
objects: objectCount,
|
|
339
|
-
}
|
|
340
|
-
};
|
|
341
|
-
},
|
|
342
|
-
],
|
|
343
|
-
});
|
|
344
|
-
```
|
|
345
|
-
|
|
346
|
-
### Transformation Pipeline
|
|
347
|
-
|
|
348
|
-
Every entity response flows through this three-stage pipeline:
|
|
349
|
-
|
|
350
|
-
1. **Base transformer** - Converts database entities to standard format
|
|
351
|
-
2. **Access transformer** - Adds access control information
|
|
352
|
-
3. **Entity transformers** - Optional additional transformations
|
|
353
|
-
|
|
354
|
-
### Transformer Context
|
|
355
|
-
|
|
356
|
-
Access and entity transformers receive a `TransformerContext` as their second argument with the shape `{ request, fastify }`. The `request` is the Fastify request for per-call data (headers, auth, query). The `fastify` instance is provided as an extension point — arocapi does not decorate it with anything itself, but consumers who decorate `fastify` with their own helpers (e.g. via `fastify.decorate('userService', ...)`) can read those decorations through `context.fastify` from inside their transformers. For database access, prefer closing over your own client reference as shown in the examples below.
|
|
357
|
-
|
|
358
|
-
### Common Use Cases
|
|
359
|
-
|
|
360
|
-
**Access control for restricted content:**
|
|
361
|
-
|
|
362
|
-
```typescript
|
|
363
|
-
accessTransformer: async (entity, { request }) => {
|
|
364
|
-
const hasAccess = await checkUserPermissions(request, entity.contentLicenseId);
|
|
365
|
-
return {
|
|
366
|
-
...entity,
|
|
367
|
-
access: {
|
|
368
|
-
metadata: true,
|
|
369
|
-
content: hasAccess,
|
|
370
|
-
},
|
|
371
|
-
};
|
|
372
|
-
}
|
|
373
|
-
```
|
|
374
|
-
|
|
375
|
-
**Adding computed or derived fields:**
|
|
376
|
-
|
|
377
|
-
```typescript
|
|
378
|
-
entityTransformers: [
|
|
379
|
-
(entity) => ({
|
|
380
|
-
...entity,
|
|
381
|
-
shortId: entity.id.split('/').pop(),
|
|
382
|
-
year: extractYear(entity.description),
|
|
383
|
-
}),
|
|
384
|
-
]
|
|
385
|
-
```
|
|
386
|
-
|
|
387
|
-
**Fetching related data asynchronously:**
|
|
388
|
-
|
|
389
|
-
```typescript
|
|
390
|
-
entityTransformers: [
|
|
391
|
-
async (entity) => ({
|
|
392
|
-
...entity,
|
|
393
|
-
stats: await fetchEntityStats(entity.id, prisma),
|
|
394
|
-
}),
|
|
395
|
-
]
|
|
396
|
-
```
|
|
397
|
-
|
|
398
|
-
## Customising File Responses
|
|
399
|
-
|
|
400
|
-
The API provides a separate transformer system for file responses, similar to entity transformers but specifically designed for files.
|
|
401
|
-
|
|
402
|
-
### File Access Transformer (Required)
|
|
403
|
-
|
|
404
|
-
The `fileAccessTransformer` parameter is **required** for security. You must explicitly choose how access control is handled for file content.
|
|
405
|
-
|
|
406
|
-
**Key Difference from Entity Transformers**: File metadata (filename, size, mediaType, etc.) is always accessible. Only content access is controlled via the `access.content` field.
|
|
407
|
-
|
|
408
|
-
**For fully public datasets**, use `AllPublicFileAccessTransformer`:
|
|
409
|
-
|
|
410
|
-
```typescript
|
|
411
|
-
import arocapi, { AllPublicAccessTransformer, AllPublicFileAccessTransformer } from 'arocapi';
|
|
412
|
-
|
|
413
|
-
await server.register(arocapi, {
|
|
414
|
-
prisma,
|
|
415
|
-
opensearch,
|
|
416
|
-
accessTransformer: AllPublicAccessTransformer,
|
|
417
|
-
fileAccessTransformer: AllPublicFileAccessTransformer,
|
|
418
|
-
});
|
|
419
19
|
```
|
|
420
20
|
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
```typescript
|
|
424
|
-
const fileAccessTransformer = async (file, { request }) => {
|
|
425
|
-
// Custom logic to determine file content access
|
|
426
|
-
const user = await authenticateUser(request);
|
|
427
|
-
const canAccessContent = await checkLicense(file.contentLicenseId, user);
|
|
21
|
+
v5 is in beta, and the documentation site describes it. Until it is released, install it with
|
|
22
|
+
`pnpm add arocapi@beta`. Changes merged to `main` are published under the `beta` tag.
|
|
428
23
|
|
|
429
|
-
|
|
430
|
-
...file,
|
|
431
|
-
access: {
|
|
432
|
-
content: canAccessContent,
|
|
433
|
-
contentAuthorizationUrl: canAccessContent
|
|
434
|
-
? undefined
|
|
435
|
-
: 'https://rems.example.com/request-access',
|
|
436
|
-
},
|
|
437
|
-
};
|
|
438
|
-
};
|
|
24
|
+
The v4 documentation is the [README at the v4.0.1 tag](https://github.com/crate-works/arocapi/blob/v4.0.1/README.md).
|
|
439
25
|
|
|
440
|
-
|
|
441
|
-
prisma,
|
|
442
|
-
opensearch,
|
|
443
|
-
accessTransformer: AllPublicAccessTransformer,
|
|
444
|
-
fileAccessTransformer,
|
|
445
|
-
});
|
|
446
|
-
```
|
|
26
|
+
## Development
|
|
447
27
|
|
|
448
|
-
|
|
449
|
-
> The `fileAccessTransformer` is required to prevent accidental exposure of
|
|
450
|
-
> restricted file content. You must make an explicit choice about access control
|
|
451
|
-
> for your files.
|
|
28
|
+
You need Node.js 22.18 or later, pnpm, and Docker with Docker Compose.
|
|
452
29
|
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
prisma,
|
|
461
|
-
opensearch,
|
|
462
|
-
accessTransformer: AllPublicAccessTransformer,
|
|
463
|
-
fileAccessTransformer: AllPublicFileAccessTransformer,
|
|
464
|
-
fileTransformers: [
|
|
465
|
-
// Add computed fields
|
|
466
|
-
(file) => ({
|
|
467
|
-
...file,
|
|
468
|
-
displayFilename: file.filename.toUpperCase(),
|
|
469
|
-
sizeInKB: Math.round(file.size / 1024),
|
|
470
|
-
extension: file.filename.split('.').pop(),
|
|
471
|
-
}),
|
|
472
|
-
// Fetch parent entity information
|
|
473
|
-
async (file) => {
|
|
474
|
-
const parent = await prisma.entity.findFirst({
|
|
475
|
-
where: { rocrateId: file.memberOf },
|
|
476
|
-
});
|
|
477
|
-
|
|
478
|
-
return {
|
|
479
|
-
...file,
|
|
480
|
-
parentEntity: parent ? {
|
|
481
|
-
id: parent.rocrateId,
|
|
482
|
-
name: parent.name,
|
|
483
|
-
type: parent.entityType,
|
|
484
|
-
} : null,
|
|
485
|
-
};
|
|
486
|
-
},
|
|
487
|
-
],
|
|
488
|
-
});
|
|
30
|
+
```sh
|
|
31
|
+
pnpm install
|
|
32
|
+
docker compose up -d # MySQL, OpenSearch and the Oni UI
|
|
33
|
+
pnpm generate # generate the Prisma client
|
|
34
|
+
pnpm db:migrate # apply migrations
|
|
35
|
+
pnpm seed # load the sample data in example/data
|
|
36
|
+
pnpm dev # build and serve src/index.dev.ts on http://localhost:9000
|
|
489
37
|
```
|
|
490
38
|
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
Every file response flows through this three-stage pipeline:
|
|
494
|
-
|
|
495
|
-
1. **Base file transformer** - Converts database File records to standard format
|
|
496
|
-
2. **File access transformer** - Adds access control information (content only)
|
|
497
|
-
3. **File transformers** - Optional additional transformations
|
|
498
|
-
|
|
499
|
-
### Applied Routes
|
|
500
|
-
|
|
501
|
-
File transformers are applied to the files listing route:
|
|
502
|
-
|
|
503
|
-
- `GET /files` - File list (each file transformed)
|
|
504
|
-
|
|
505
|
-
**Note**: The `/file/:id` endpoint (for downloading file content) uses the `fileHandler` system, not file transformers. See the File Handler System section below for details.
|
|
506
|
-
|
|
507
|
-
## File Handler System
|
|
508
|
-
|
|
509
|
-
The API provides two separate handler systems for serving different types of content. These are distinct from the file transformer system described above.
|
|
510
|
-
|
|
511
|
-
**Important Distinction:**
|
|
512
|
-
|
|
513
|
-
- **File Transformers** (above) - Transform file metadata in the `/files` listing endpoint response
|
|
514
|
-
- **File Handler** (this section) - Serves actual file content via the `/file/:id` download endpoint
|
|
515
|
-
- **RO-Crate Handler** (this section) - Serves RO-Crate metadata via the `/entity/:id/rocrate` endpoint
|
|
516
|
-
|
|
517
|
-
### File Handler (Required)
|
|
518
|
-
|
|
519
|
-
The `fileHandler` parameter is **required**. It serves actual file content for the `/file/:id` endpoint.
|
|
520
|
-
|
|
521
|
-
This handler is responsible for:
|
|
522
|
-
|
|
523
|
-
- Streaming file content from your storage backend
|
|
524
|
-
- Generating redirect URLs to external storage (S3, CDN, etc.)
|
|
525
|
-
- Implementing access control for file downloads
|
|
526
|
-
- Supporting HTTP range requests for media streaming
|
|
527
|
-
|
|
528
|
-
### RO-Crate Handler (Required)
|
|
529
|
-
|
|
530
|
-
The `roCrateHandler` parameter is **required**. It serves RO-Crate metadata as JSON-LD
|
|
531
|
-
for any entity type (Collection, Object, or File) via the `/entity/:id/rocrate` endpoint.
|
|
532
|
-
|
|
533
|
-
**File Handler Example** (S3 with redirect):
|
|
534
|
-
|
|
535
|
-
```typescript
|
|
536
|
-
fileHandler: {
|
|
537
|
-
get: async (file) => {
|
|
538
|
-
const command = new GetObjectCommand({
|
|
539
|
-
Bucket: file.meta.bucket,
|
|
540
|
-
Key: file.meta.s3Key,
|
|
541
|
-
});
|
|
542
|
-
const url = await getSignedUrl(s3, command, { expiresIn: 3600 });
|
|
543
|
-
|
|
544
|
-
return { type: 'redirect', url };
|
|
545
|
-
},
|
|
546
|
-
head: async (file) => {
|
|
547
|
-
return {
|
|
548
|
-
contentType: file.mediaType,
|
|
549
|
-
contentLength: file.size,
|
|
550
|
-
};
|
|
551
|
-
},
|
|
552
|
-
}
|
|
553
|
-
```
|
|
554
|
-
|
|
555
|
-
**File Handler Example** (local filesystem):
|
|
556
|
-
|
|
557
|
-
```typescript
|
|
558
|
-
fileHandler: {
|
|
559
|
-
get: async (file) => {
|
|
560
|
-
const filePath = `/data/files/${file.meta.storagePath}`;
|
|
561
|
-
const stats = await stat(filePath);
|
|
562
|
-
|
|
563
|
-
return {
|
|
564
|
-
type: 'stream',
|
|
565
|
-
stream: createReadStream(filePath),
|
|
566
|
-
metadata: {
|
|
567
|
-
contentType: file.mediaType,
|
|
568
|
-
contentLength: stats.size,
|
|
569
|
-
lastModified: stats.mtime,
|
|
570
|
-
},
|
|
571
|
-
};
|
|
572
|
-
},
|
|
573
|
-
head: async (file) => {
|
|
574
|
-
const filePath = `/data/files/${file.meta.storagePath}`;
|
|
575
|
-
const stats = await stat(filePath);
|
|
576
|
-
|
|
577
|
-
return {
|
|
578
|
-
contentType: file.mediaType,
|
|
579
|
-
contentLength: stats.size,
|
|
580
|
-
lastModified: stats.mtime,
|
|
581
|
-
};
|
|
582
|
-
},
|
|
583
|
-
}
|
|
584
|
-
```
|
|
585
|
-
|
|
586
|
-
**RO-Crate Handler Example** (stream from database):
|
|
587
|
-
|
|
588
|
-
```typescript
|
|
589
|
-
roCrateHandler: {
|
|
590
|
-
get: async (entity) => {
|
|
591
|
-
const jsonString = JSON.stringify(entity.rocrate, null, 2);
|
|
592
|
-
|
|
593
|
-
return {
|
|
594
|
-
type: 'stream',
|
|
595
|
-
stream: Readable.from([jsonString]),
|
|
596
|
-
metadata: {
|
|
597
|
-
contentType: 'application/ld+json',
|
|
598
|
-
contentLength: Buffer.byteLength(jsonString),
|
|
599
|
-
},
|
|
600
|
-
};
|
|
601
|
-
},
|
|
602
|
-
head: async (entity) => {
|
|
603
|
-
const jsonString = JSON.stringify(entity.rocrate);
|
|
604
|
-
return {
|
|
605
|
-
contentType: 'application/ld+json',
|
|
606
|
-
contentLength: Buffer.byteLength(jsonString),
|
|
607
|
-
};
|
|
608
|
-
},
|
|
609
|
-
}
|
|
610
|
-
```
|
|
611
|
-
|
|
612
|
-
### File Handler Response Types
|
|
613
|
-
|
|
614
|
-
The file handler must return one of two response types:
|
|
615
|
-
|
|
616
|
-
**Redirect Response** - Redirect to external file location:
|
|
617
|
-
|
|
618
|
-
```typescript
|
|
619
|
-
{ type: 'redirect', url: 'https://storage.example.com/file.wav' }
|
|
620
|
-
```
|
|
621
|
-
|
|
622
|
-
**Stream Response** - Serve file content directly:
|
|
623
|
-
|
|
624
|
-
```typescript
|
|
625
|
-
{
|
|
626
|
-
type: 'stream',
|
|
627
|
-
stream: Readable, // Node.js readable stream
|
|
628
|
-
metadata: {
|
|
629
|
-
contentType: 'audio/wav',
|
|
630
|
-
contentLength: 1024,
|
|
631
|
-
etag?: '"abc123"', // Optional
|
|
632
|
-
lastModified?: new Date(), // Optional
|
|
633
|
-
},
|
|
634
|
-
}
|
|
635
|
-
```
|
|
636
|
-
|
|
637
|
-
### Query Parameters
|
|
638
|
-
|
|
639
|
-
The `/file/:id` endpoint supports these query parameters:
|
|
640
|
-
|
|
641
|
-
- `disposition` - 'inline' (default) or 'attachment' for download prompts
|
|
642
|
-
- `filename` - Custom filename for Content-Disposition header (defaults to entity.name)
|
|
643
|
-
- `noRedirect` - If true with redirect response, returns JSON `{location: url}` instead of HTTP 302
|
|
644
|
-
|
|
645
|
-
### HTTP Range Support
|
|
646
|
-
|
|
647
|
-
Range request support for media streaming can be implemented in your file
|
|
648
|
-
handler by parsing the `Range` header from the request context and returning
|
|
649
|
-
a partial stream:
|
|
650
|
-
|
|
651
|
-
```typescript
|
|
652
|
-
fileHandler: {
|
|
653
|
-
get: async (file, { request }) => {
|
|
654
|
-
const range = request.headers.range;
|
|
655
|
-
// Parse range header and return partial content stream
|
|
656
|
-
// Your storage backend (S3, filesystem, etc.) likely supports range reads natively
|
|
657
|
-
},
|
|
658
|
-
}
|
|
659
|
-
```
|
|
660
|
-
|
|
661
|
-
### Authentication and Authorisation
|
|
662
|
-
|
|
663
|
-
The library delegates authentication and authorisation to your transformers.
|
|
664
|
-
The access transformer receives the full Fastify request context, so you can
|
|
665
|
-
verify credentials and set access flags accordingly:
|
|
666
|
-
|
|
667
|
-
```typescript
|
|
668
|
-
accessTransformer: async (entity, { request }) => {
|
|
669
|
-
const token = request.headers.authorization;
|
|
670
|
-
const user = await verifyToken(token);
|
|
671
|
-
|
|
672
|
-
if (!user) {
|
|
673
|
-
// Unauthenticated — metadata visible but content restricted
|
|
674
|
-
return {
|
|
675
|
-
...entity,
|
|
676
|
-
access: {
|
|
677
|
-
metadata: true,
|
|
678
|
-
content: false,
|
|
679
|
-
contentAuthorizationUrl: 'https://auth.example.com/login',
|
|
680
|
-
},
|
|
681
|
-
};
|
|
682
|
-
}
|
|
683
|
-
|
|
684
|
-
const canAccess = await checkUserLicense(user, entity.contentLicenseId);
|
|
685
|
-
|
|
686
|
-
return {
|
|
687
|
-
...entity,
|
|
688
|
-
access: {
|
|
689
|
-
metadata: true,
|
|
690
|
-
content: canAccess,
|
|
691
|
-
contentAuthorizationUrl: canAccess
|
|
692
|
-
? undefined
|
|
693
|
-
: 'https://rems.example.com/request-access',
|
|
694
|
-
},
|
|
695
|
-
};
|
|
696
|
-
}
|
|
697
|
-
```
|
|
698
|
-
|
|
699
|
-
The same pattern applies to `fileAccessTransformer` for controlling file
|
|
700
|
-
content downloads.
|
|
701
|
-
|
|
702
|
-
### Entity Meta Field
|
|
703
|
-
|
|
704
|
-
The `meta` JSON field in the Entity model stores implementation-specific
|
|
705
|
-
metadata for your file handler:
|
|
706
|
-
|
|
707
|
-
```typescript
|
|
708
|
-
await prisma.entity.create({
|
|
709
|
-
data: {
|
|
710
|
-
id: 'http://example.com/file/123',
|
|
711
|
-
name: 'audio.wav',
|
|
712
|
-
entityType: 'http://schema.org/MediaObject',
|
|
713
|
-
// ... other required fields (memberOf, rootCollection, metadataLicenseId, contentLicenseId)
|
|
714
|
-
meta: {
|
|
715
|
-
bucket: 's3://my-bucket',
|
|
716
|
-
storagePath: 'collections/col-01',
|
|
717
|
-
checksum: 'sha256:abc123...',
|
|
718
|
-
},
|
|
719
|
-
},
|
|
720
|
-
});
|
|
721
|
-
```
|
|
722
|
-
|
|
723
|
-
Your file handler can use this metadata to locate files in your storage system.
|
|
724
|
-
|
|
725
|
-
## Using with Express
|
|
726
|
-
|
|
727
|
-
arocapi doesn't ship an Express adapter, but you can register the plugin on a
|
|
728
|
-
Fastify instance and forward requests to it. Install `express` in your app
|
|
729
|
-
first.
|
|
730
|
-
|
|
731
|
-
```typescript
|
|
732
|
-
// src/index.ts
|
|
733
|
-
import arocapi, { type Options } from 'arocapi';
|
|
734
|
-
import express from 'express';
|
|
735
|
-
import Fastify from 'fastify';
|
|
736
|
-
|
|
737
|
-
const options: Options = {
|
|
738
|
-
// Same options as passed to fastify.register in Creating Your API
|
|
739
|
-
};
|
|
740
|
-
|
|
741
|
-
const fastify = Fastify({
|
|
742
|
-
logger: true,
|
|
743
|
-
});
|
|
744
|
-
fastify.register(arocapi, options);
|
|
745
|
-
await fastify.ready();
|
|
746
|
-
|
|
747
|
-
const app = express();
|
|
748
|
-
app.use('/api', (req, res) => {
|
|
749
|
-
fastify.routing(req, res);
|
|
750
|
-
});
|
|
751
|
-
|
|
752
|
-
const port = process.env.PORT || 3000;
|
|
753
|
-
app.listen(port, () => {
|
|
754
|
-
console.log(`API server listening on port ${port}`);
|
|
755
|
-
});
|
|
756
|
-
```
|
|
757
|
-
|
|
758
|
-
## Development Workflow
|
|
759
|
-
|
|
760
|
-
### Local Development Setup
|
|
761
|
-
|
|
762
|
-
1. **Start the services:**
|
|
763
|
-
|
|
764
|
-
```bash
|
|
765
|
-
# Start MySQL and OpenSearch using docker-compose
|
|
766
|
-
docker compose up -d
|
|
767
|
-
```
|
|
768
|
-
|
|
769
|
-
2. **Set up the database:**
|
|
770
|
-
|
|
771
|
-
```bash
|
|
772
|
-
# Generate Prisma client
|
|
773
|
-
npx prisma generate
|
|
774
|
-
|
|
775
|
-
# Apply database migrations
|
|
776
|
-
npx prisma migrate deploy
|
|
777
|
-
```
|
|
778
|
-
|
|
779
|
-
3. **Run your application:**
|
|
780
|
-
|
|
781
|
-
```bash
|
|
782
|
-
# For development mode
|
|
783
|
-
pnpm run dev
|
|
784
|
-
|
|
785
|
-
# Or start directly
|
|
786
|
-
node dist/app.js
|
|
787
|
-
```
|
|
788
|
-
|
|
789
|
-
4. **Access your API:**
|
|
790
|
-
- API: <http://localhost:3000>
|
|
791
|
-
- Database: localhost:3306 (MySQL)
|
|
792
|
-
- OpenSearch: <http://localhost:9200>
|
|
793
|
-
|
|
794
|
-
### Available Scripts
|
|
795
|
-
|
|
796
|
-
When using arocapi in your project, you can use these scripts:
|
|
797
|
-
|
|
798
|
-
```bash
|
|
799
|
-
# Development
|
|
800
|
-
pnpm run dev # Start in development mode with hot reload
|
|
801
|
-
pnpm run start # Start in production mode
|
|
802
|
-
|
|
803
|
-
# Database Management
|
|
804
|
-
npx prisma generate # Generate Prisma client
|
|
805
|
-
npx prisma migrate dev # Create and apply new migration
|
|
806
|
-
npx prisma migrate deploy # Apply existing migrations
|
|
807
|
-
npx prisma studio # Open Prisma Studio (database GUI)
|
|
808
|
-
|
|
809
|
-
# Database Console
|
|
810
|
-
docker compose exec db mysql -u root -p catalog # Connect to MySQL console
|
|
811
|
-
|
|
812
|
-
# Testing
|
|
813
|
-
pnpm run test # Run test cases (if configured)
|
|
814
|
-
```
|
|
815
|
-
|
|
816
|
-
### Project Structure
|
|
817
|
-
|
|
818
|
-
When implementing arocapi in your project, follow this structure:
|
|
819
|
-
|
|
820
|
-
```text
|
|
821
|
-
your-project/
|
|
822
|
-
├── src/
|
|
823
|
-
│ ├── app.ts # Your main application file
|
|
824
|
-
│ └── generated/
|
|
825
|
-
│ └── prisma/ # Generated Prisma client
|
|
826
|
-
├── prisma/
|
|
827
|
-
│ ├── schema.prisma # Prisma schema
|
|
828
|
-
│ ├── models/ # Your custom models
|
|
829
|
-
│ └── arocapi/ # Symlink to arocapi models
|
|
830
|
-
├── prisma.config.ts # Prisma configuration
|
|
831
|
-
├── .env # Environment variables
|
|
832
|
-
└── package.json
|
|
833
|
-
```
|
|
39
|
+
`.env` holds the local connection URLs and is committed, so don't put secrets in it.
|
|
834
40
|
|
|
835
|
-
|
|
41
|
+
| Command | What it does |
|
|
42
|
+
| ----------------------------- | -------------------------------------------------------------------- |
|
|
43
|
+
| `pnpm lint` | Biome, type checking, the docs examples type check and knip |
|
|
44
|
+
| `pnpm test` | Start the test containers, apply migrations and run every test |
|
|
45
|
+
| `pnpm test src/app.test.ts` | Run one test file |
|
|
46
|
+
| `pnpm test:watch` | Run the tests in watch mode |
|
|
47
|
+
| `pnpm test:coverage` | Run the tests with an HTML coverage report |
|
|
48
|
+
| `pnpm dbconsole` | Open a MySQL console on the development database |
|
|
49
|
+
| `pnpm --filter website dev` | Run the documentation site locally |
|
|
50
|
+
| `pnpm --filter website build` | Build the documentation site, checking its diagrams and every link |
|
|
836
51
|
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
- [PILARS](https://w3id.org/ldac/pilars)
|
|
840
|
-
- [REMS](https://trust.aaf.edu.au/rems/)
|
|
52
|
+
Commit messages follow [Conventional Commits](https://www.conventionalcommits.org/); they decide the next version.
|
|
53
|
+
See [TESTING.md](TESTING.md) for the test setup and [RELEASING.md](RELEASING.md) for releases.
|