@bedrock/vc-delivery 7.15.0 → 7.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/ExchangeProcessor.js +1 -1
- package/lib/config.js +34 -1
- package/lib/constants.js +10 -1
- package/lib/http.js +1 -13
- package/lib/storage/exchanges.js +63 -67
- package/lib/storage/variables.js +225 -0
- package/lib/storage/variablesGarbageCollector.js +83 -0
- package/lib/vcapi.js +4 -8
- package/package.json +3 -2
package/lib/ExchangeProcessor.js
CHANGED
|
@@ -168,7 +168,7 @@ export class ExchangeProcessor {
|
|
|
168
168
|
if(exchange.state === 'complete') {
|
|
169
169
|
await exchanges.complete({workflowId: workflow.id, exchange});
|
|
170
170
|
} else {
|
|
171
|
-
await exchanges.update({workflowId: workflow.id, exchange});
|
|
171
|
+
await exchanges.update({workflowId: workflow.id, exchange, meta});
|
|
172
172
|
}
|
|
173
173
|
meta.updated = Date.now();
|
|
174
174
|
await emitExchangeUpdated({workflow, exchange, step});
|
package/lib/config.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/*!
|
|
2
|
-
* Copyright (c) 2022-
|
|
2
|
+
* Copyright (c) 2022-2026 Digital Bazaar, Inc.
|
|
3
3
|
*/
|
|
4
4
|
import * as bedrock from '@bedrock/core';
|
|
5
|
+
import '@bedrock/express';
|
|
5
6
|
|
|
6
7
|
const c = bedrock.util.config.main;
|
|
7
8
|
const cc = c.computer();
|
|
@@ -11,6 +12,15 @@ const {config} = bedrock;
|
|
|
11
12
|
const namespace = 'vc-workflow';
|
|
12
13
|
config[namespace] = {};
|
|
13
14
|
|
|
15
|
+
config[namespace].exchanges = {
|
|
16
|
+
variablesGarbageCollector: {
|
|
17
|
+
// collect expired externalized exchange "variables" every 5 minutes; may
|
|
18
|
+
// be slightly randomized
|
|
19
|
+
// default: 5 minutes
|
|
20
|
+
interval: 5 * 60 * 1000
|
|
21
|
+
}
|
|
22
|
+
};
|
|
23
|
+
|
|
14
24
|
// create dev application identity for vc-workflow (must be overridden in
|
|
15
25
|
// deployments) ...and `ensureConfigOverride` has already been set via
|
|
16
26
|
// `bedrock-app-identity` so it doesn't have to be set here
|
|
@@ -29,3 +39,26 @@ cc('app-identity.seeds.services.vc-exchanger.id', () =>
|
|
|
29
39
|
config['app-identity'].seeds.services['vc-workflow'].id);
|
|
30
40
|
cc('app-identity.seeds.services.vc-exchanger.seedMultibase', () =>
|
|
31
41
|
config['app-identity'].seeds.services['vc-workflow'].seedMultibase);
|
|
42
|
+
|
|
43
|
+
// set body parser limits for workflow endpoints (and deprecated `/exchangers`)
|
|
44
|
+
const routePrefixes = ['/workflows', '/exchangers'];
|
|
45
|
+
const createBodyParserOptions = ({limit}) => ({
|
|
46
|
+
json: {
|
|
47
|
+
strict: false,
|
|
48
|
+
limit,
|
|
49
|
+
type: ['json', '+json']
|
|
50
|
+
}
|
|
51
|
+
});
|
|
52
|
+
const bodyParserRoutes = config.express.bodyParser.routes;
|
|
53
|
+
for(const routePrefix of routePrefixes) {
|
|
54
|
+
// exchange clients POST to this route to execute exchanges; limit indicates
|
|
55
|
+
// how large submitted VPs can be
|
|
56
|
+
bodyParserRoutes[
|
|
57
|
+
`${routePrefix}/:localWorkflowId/exchanges/:localExchangeId`
|
|
58
|
+
] = createBodyParserOptions({limit: '10MB'});
|
|
59
|
+
// exchanges are created using this route; limit indicates how large
|
|
60
|
+
// variables can be (in total)
|
|
61
|
+
bodyParserRoutes[
|
|
62
|
+
`${routePrefix}/:localWorkflowId/exchanges`
|
|
63
|
+
] = createBodyParserOptions({limit: '10MB'});
|
|
64
|
+
}
|
package/lib/constants.js
CHANGED
|
@@ -1,6 +1,15 @@
|
|
|
1
1
|
/*!
|
|
2
|
-
* Copyright (c) 2024-
|
|
2
|
+
* Copyright (c) 2024-2026 Digital Bazaar, Inc.
|
|
3
3
|
*/
|
|
4
|
+
// allow up to 3 days to resolve invalid exchange issues (which also more than
|
|
5
|
+
// covers large exchange `variables` download and decoding times, etc.)
|
|
6
|
+
// (86400 seconds in 24 hours)
|
|
7
|
+
export const EXCHANGE_EXPIRY_GRACE_PERIOD = 86400 * 3 * 1000;
|
|
8
|
+
// TTL is measured in minutes, default is 15 minutes
|
|
9
|
+
export const EXCHANGE_TTL_DEFAULT = 60 * 15;
|
|
10
|
+
// 48 hours
|
|
11
|
+
export const EXCHANGE_TTL_MAX_IN_MS = 1000 * 60 * 60 * 24 * 2;
|
|
12
|
+
|
|
4
13
|
// maximum # of issuer instances that can be associated with a workflow
|
|
5
14
|
export const MAX_ISSUER_INSTANCES = 10;
|
|
6
15
|
// maximum # of OID4VP client profiles that can be associated with a workflow
|
package/lib/http.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/*!
|
|
2
|
-
* Copyright (c) 2018-2026 Digital Bazaar, Inc.
|
|
2
|
+
* Copyright (c) 2018-2026 Digital Bazaar, Inc.
|
|
3
3
|
*/
|
|
4
4
|
import * as bedrock from '@bedrock/core';
|
|
5
5
|
import * as exchanges from './storage/exchanges.js';
|
|
@@ -11,7 +11,6 @@ import {
|
|
|
11
11
|
} from '../schemas/bedrock-vc-workflow.js';
|
|
12
12
|
import {metering, middleware} from '@bedrock/service-core';
|
|
13
13
|
import {asyncHandler} from '@bedrock/express';
|
|
14
|
-
import bodyParser from 'body-parser';
|
|
15
14
|
import cors from 'cors';
|
|
16
15
|
import {getWorkflowId} from './helpers.js';
|
|
17
16
|
import {logger} from './logger.js';
|
|
@@ -19,17 +18,6 @@ import {createValidateMiddleware as validate} from '@bedrock/validation';
|
|
|
19
18
|
|
|
20
19
|
const {util: {BedrockError}} = bedrock;
|
|
21
20
|
|
|
22
|
-
// FIXME: remove and apply to specific routes via
|
|
23
|
-
// `bedrock.express.bodyParser.routes` + `@bedrock/express@8.4`
|
|
24
|
-
bedrock.events.on('bedrock-express.configure.bodyParser', app => {
|
|
25
|
-
app.use(bodyParser.json({
|
|
26
|
-
// allow json values that are not just objects or arrays
|
|
27
|
-
strict: false,
|
|
28
|
-
limit: '10MB',
|
|
29
|
-
type: ['json', '+json']
|
|
30
|
-
}));
|
|
31
|
-
});
|
|
32
|
-
|
|
33
21
|
export async function addRoutes({app, service} = {}) {
|
|
34
22
|
const {routePrefix} = service;
|
|
35
23
|
|
package/lib/storage/exchanges.js
CHANGED
|
@@ -1,13 +1,20 @@
|
|
|
1
1
|
/*!
|
|
2
|
-
* Copyright (c) 2022-2026 Digital Bazaar, Inc.
|
|
2
|
+
* Copyright (c) 2022-2026 Digital Bazaar, Inc.
|
|
3
3
|
*/
|
|
4
4
|
import * as bedrock from '@bedrock/core';
|
|
5
5
|
import * as database from '@bedrock/mongodb';
|
|
6
|
+
import {decodeVariables, encodeVariables} from './variables.js';
|
|
7
|
+
import {
|
|
8
|
+
EXCHANGE_EXPIRY_GRACE_PERIOD, EXCHANGE_TTL_DEFAULT
|
|
9
|
+
} from '../constants.js';
|
|
6
10
|
import {parseLocalId, stripStacktrace} from '../helpers.js';
|
|
7
11
|
import assert from 'assert-plus';
|
|
8
12
|
import {logger} from '../logger.js';
|
|
9
13
|
import {serializeError} from 'serialize-error';
|
|
10
14
|
|
|
15
|
+
// ensure externalized `variables` garbage collector runs
|
|
16
|
+
import './variablesGarbageCollector.js';
|
|
17
|
+
|
|
11
18
|
const {util: {BedrockError}} = bedrock;
|
|
12
19
|
|
|
13
20
|
/* Note: Exchanges have default TTLs of 15 minutes and are always in one of
|
|
@@ -23,6 +30,7 @@ one or more steps, each that might issue, verify, or deliver VCs. Capabilities
|
|
|
23
30
|
must be provided to issue or verify VCs. */
|
|
24
31
|
|
|
25
32
|
const COLLECTION_NAME = 'vc-exchange';
|
|
33
|
+
const BUCKET_NAME = 'vc-exchange-variables';
|
|
26
34
|
|
|
27
35
|
// allow updates to the last error every 500ms
|
|
28
36
|
const LAST_ERROR_UPDATE_CONSTRAINTS = {
|
|
@@ -33,10 +41,10 @@ const LAST_ERROR_UPDATE_CONSTRAINTS = {
|
|
|
33
41
|
updateTimeLimit: 1000
|
|
34
42
|
};
|
|
35
43
|
|
|
36
|
-
const MONGODB_ILLEGAL_KEY_CHAR_REGEX = /[%$.]/;
|
|
37
|
-
|
|
38
44
|
bedrock.events.on('bedrock-mongodb.ready', async () => {
|
|
39
|
-
await database.openCollections([
|
|
45
|
+
await database.openCollections([
|
|
46
|
+
COLLECTION_NAME, `${BUCKET_NAME}.files`
|
|
47
|
+
]);
|
|
40
48
|
|
|
41
49
|
await database.createIndexes([{
|
|
42
50
|
// cover exchange queries by local workflow ID + exchange ID
|
|
@@ -84,32 +92,32 @@ export async function insert({workflowId, exchange}) {
|
|
|
84
92
|
assert.string(workflowId, 'workflowId');
|
|
85
93
|
assert.object(exchange, 'exchange');
|
|
86
94
|
assert.string(exchange.id, 'exchange.id');
|
|
95
|
+
assert.string(exchange.expires, 'exchange.expires');
|
|
87
96
|
// optional time to live in seconds
|
|
88
97
|
assert.optionalNumber(exchange.ttl, 'exchange.ttl');
|
|
89
98
|
// optional variables to use in VC templates
|
|
90
99
|
assert.optionalObject(exchange.variables, 'exchange.variables');
|
|
91
100
|
// optional current step in the exchange
|
|
92
101
|
assert.optionalString(exchange.step, 'exchange.step');
|
|
93
|
-
// optional expires in exchange
|
|
94
|
-
assert.optionalString(exchange.expires, 'exchange.expires');
|
|
95
102
|
// optional protocols in exchange
|
|
96
103
|
assert.optionalObject(exchange.protocols, 'exchange.protocols');
|
|
97
104
|
|
|
98
105
|
// build exchange record
|
|
99
106
|
const now = Date.now();
|
|
100
|
-
const meta = {
|
|
107
|
+
const meta = {
|
|
108
|
+
created: now,
|
|
109
|
+
updated: now,
|
|
110
|
+
expires: new Date(exchange.expires)
|
|
111
|
+
};
|
|
101
112
|
// possible states are: `pending`, `active`, `complete`, or `invalid`
|
|
102
113
|
exchange = {...exchange, sequence: 0, state: 'pending'};
|
|
103
|
-
if(exchange.expires !== undefined) {
|
|
104
|
-
meta.expires = new Date(exchange.expires);
|
|
105
|
-
}
|
|
106
114
|
const {localId: localWorkflowId} = parseLocalId({id: workflowId});
|
|
107
115
|
const record = {
|
|
108
116
|
localWorkflowId,
|
|
109
117
|
// backwards compatibility: enable existing systems to find record
|
|
110
118
|
localExchangerId: localWorkflowId,
|
|
111
119
|
meta,
|
|
112
|
-
exchange:
|
|
120
|
+
exchange: await encodeVariables({workflowId, exchange, meta})
|
|
113
121
|
};
|
|
114
122
|
|
|
115
123
|
// insert the exchange and get the updated record
|
|
@@ -175,11 +183,15 @@ export async function get({
|
|
|
175
183
|
}
|
|
176
184
|
|
|
177
185
|
let record = await collection.findOne(query, {projection});
|
|
178
|
-
if(
|
|
186
|
+
if(!allowExpired) {
|
|
179
187
|
// ensure `expires` is enforced programmatically even if background job
|
|
180
|
-
// has not yet removed the record
|
|
188
|
+
// has not yet removed the record; force unexpiring exchanges to be not
|
|
189
|
+
// found via this code path -- any exchanges without an expiration date
|
|
190
|
+
// are from very old software and they will need to be manually cleaned up
|
|
191
|
+
// in the database
|
|
181
192
|
const now = new Date();
|
|
182
|
-
|
|
193
|
+
// note: for undefined `expires`, this will be `NaN || now` => `now`
|
|
194
|
+
const expires = new Date(record.exchange.expires) || now;
|
|
183
195
|
if(now >= expires) {
|
|
184
196
|
record = null;
|
|
185
197
|
}
|
|
@@ -198,7 +210,7 @@ export async function get({
|
|
|
198
210
|
});
|
|
199
211
|
}
|
|
200
212
|
|
|
201
|
-
record.exchange =
|
|
213
|
+
record.exchange = await decodeVariables({workflowId, record});
|
|
202
214
|
|
|
203
215
|
// backwards compatibility; initialize `sequence`
|
|
204
216
|
if(record.exchange.sequence === undefined) {
|
|
@@ -228,21 +240,36 @@ export async function get({
|
|
|
228
240
|
* @param {string} options.workflowId - The ID of the workflow the exchange
|
|
229
241
|
* is associated with.
|
|
230
242
|
* @param {object} options.exchange - The exchange to update.
|
|
243
|
+
* @param {object} options.meta - The exchange meta to update.
|
|
231
244
|
* @param {boolean} [options.explain=false] - An optional explain boolean.
|
|
232
245
|
*
|
|
233
246
|
* @returns {Promise<boolean | ExplainObject>} Resolves with `true` on update
|
|
234
247
|
* success or an ExplainObject if `explain=true`.
|
|
235
248
|
*/
|
|
236
|
-
export async function update({
|
|
249
|
+
export async function update({
|
|
250
|
+
workflowId, exchange, meta, explain = false
|
|
251
|
+
} = {}) {
|
|
237
252
|
assert.string(workflowId, 'workflowId');
|
|
238
253
|
assert.object(exchange, 'exchange');
|
|
239
254
|
const {id} = exchange;
|
|
240
255
|
|
|
256
|
+
// force exchange to expire if this code has been called on an old exchange
|
|
257
|
+
// with no expiration date
|
|
258
|
+
let updateExpires = false;
|
|
259
|
+
if(exchange.expires === undefined) {
|
|
260
|
+
updateExpires = true;
|
|
261
|
+
const ttl = exchange.ttl ?? EXCHANGE_TTL_DEFAULT;
|
|
262
|
+
// TTL is in seconds, convert to milliseconds
|
|
263
|
+
const expires = new Date(Date.now() + ttl * 1000);
|
|
264
|
+
exchange.expires = expires.toISOString().replace(/\.\d+Z$/, 'Z');
|
|
265
|
+
}
|
|
266
|
+
|
|
241
267
|
// encode variable content for storage in mongoDB
|
|
242
|
-
|
|
268
|
+
meta = {...meta};
|
|
269
|
+
exchange = await encodeVariables({workflowId, exchange, meta});
|
|
243
270
|
|
|
244
271
|
// build update
|
|
245
|
-
const update = _buildUpdate({exchange});
|
|
272
|
+
const update = _buildUpdate({exchange, meta, updateExpires});
|
|
246
273
|
|
|
247
274
|
const {base, localId: localWorkflowId} = parseLocalId({id: workflowId});
|
|
248
275
|
|
|
@@ -516,10 +543,11 @@ async function _invalidateExchange({record}) {
|
|
|
516
543
|
`Could not mark exchange "${record.exchange.id}" invalid.`, {error});
|
|
517
544
|
}
|
|
518
545
|
|
|
519
|
-
/*
|
|
520
|
-
is specified in the exchange record). */
|
|
546
|
+
/* Consider perform auto-revocation of the VCs or notification (the action to
|
|
547
|
+
take is specified in the exchange record). */
|
|
521
548
|
// FIXME: handle auto-revocation / notification in background; do not throw
|
|
522
|
-
// errors to client
|
|
549
|
+
// errors to client; consider removing invalidation state as well and rely
|
|
550
|
+
// solely on TTL to manage exchange secrecy/abuse
|
|
523
551
|
}
|
|
524
552
|
|
|
525
553
|
async function _markExchangeInvalid({record}) {
|
|
@@ -539,9 +567,7 @@ async function _markExchangeInvalid({record}) {
|
|
|
539
567
|
$set: {
|
|
540
568
|
'exchange.state': 'invalid',
|
|
541
569
|
'meta.updated': now,
|
|
542
|
-
|
|
543
|
-
// (86400 seconds in 24 hours)
|
|
544
|
-
'meta.expires': new Date(now + 86400 * 3 * 1000)
|
|
570
|
+
'meta.expires': new Date(now + EXCHANGE_EXPIRY_GRACE_PERIOD)
|
|
545
571
|
}
|
|
546
572
|
};
|
|
547
573
|
const collection = database.collections[COLLECTION_NAME];
|
|
@@ -562,7 +588,7 @@ async function _markExchangeInvalid({record}) {
|
|
|
562
588
|
}
|
|
563
589
|
}
|
|
564
590
|
|
|
565
|
-
function _buildUpdate({exchange}) {
|
|
591
|
+
function _buildUpdate({exchange, meta, updateExpires = false}) {
|
|
566
592
|
// build update
|
|
567
593
|
const now = Date.now();
|
|
568
594
|
const update = {
|
|
@@ -579,15 +605,15 @@ function _buildUpdate({exchange}) {
|
|
|
579
605
|
if(exchange.step !== undefined) {
|
|
580
606
|
update.$set['exchange.step'] = exchange.step;
|
|
581
607
|
}
|
|
582
|
-
// only
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
608
|
+
// only update (fix) `expires` if it was not previously set (a very old
|
|
609
|
+
// exchange is being updated)
|
|
610
|
+
if(updateExpires) {
|
|
611
|
+
// TTL is in seconds, convert to milliseconds
|
|
612
|
+
const expires = new Date(exchange.expires);
|
|
613
|
+
// unset any previously set `ttl` and set `expires` instead
|
|
587
614
|
update.$unset['exchange.ttl'] = true;
|
|
588
615
|
update.$set['meta.expires'] = expires;
|
|
589
|
-
update.$set['exchange.expires'] =
|
|
590
|
-
expires.toISOString().replace(/\.\d+Z$/, 'Z');
|
|
616
|
+
update.$set['exchange.expires'] = exchange.expires;
|
|
591
617
|
}
|
|
592
618
|
if(exchange.lastError !== undefined) {
|
|
593
619
|
update.$set['exchange.lastError'] =
|
|
@@ -595,43 +621,13 @@ function _buildUpdate({exchange}) {
|
|
|
595
621
|
} else {
|
|
596
622
|
update.$unset['exchange.lastError'] = true;
|
|
597
623
|
}
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
}
|
|
601
|
-
|
|
602
|
-
function _encodeVariables({exchange}) {
|
|
603
|
-
// if any JSON object any variable uses a character that is not legal in
|
|
604
|
-
// a JSON key in mongoDB then stringify all the variables
|
|
605
|
-
if(_hasIllegalMongoDBKeyChar(exchange.variables)) {
|
|
606
|
-
return {...exchange, variables: JSON.stringify(exchange.variables)};
|
|
607
|
-
}
|
|
608
|
-
return exchange;
|
|
609
|
-
}
|
|
610
|
-
|
|
611
|
-
function _decodeVariables({exchange}) {
|
|
612
|
-
if(typeof exchange.variables === 'string') {
|
|
613
|
-
return {...exchange, variables: JSON.parse(exchange.variables)};
|
|
624
|
+
if(meta?.variablesFilename === false) {
|
|
625
|
+
update.$unset['meta.variablesFilename'] = true;
|
|
626
|
+
} else if(typeof meta?.variablesFilename === 'string') {
|
|
627
|
+
update.$set['meta.variablesFilename'] = meta.variablesFilename;
|
|
614
628
|
}
|
|
615
|
-
return exchange;
|
|
616
|
-
}
|
|
617
629
|
|
|
618
|
-
|
|
619
|
-
if(Array.isArray(value)) {
|
|
620
|
-
for(const e of value) {
|
|
621
|
-
if(_hasIllegalMongoDBKeyChar(e)) {
|
|
622
|
-
return true;
|
|
623
|
-
}
|
|
624
|
-
}
|
|
625
|
-
} else if(value && typeof value === 'object') {
|
|
626
|
-
const keys = Object.keys(value);
|
|
627
|
-
for(const key of keys) {
|
|
628
|
-
if(MONGODB_ILLEGAL_KEY_CHAR_REGEX.test(key) ||
|
|
629
|
-
_hasIllegalMongoDBKeyChar(value[key])) {
|
|
630
|
-
return true;
|
|
631
|
-
}
|
|
632
|
-
}
|
|
633
|
-
}
|
|
634
|
-
return false;
|
|
630
|
+
return update;
|
|
635
631
|
}
|
|
636
632
|
|
|
637
633
|
/**
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* Copyright (c) 2022-2026 Digital Bazaar, Inc.
|
|
3
|
+
*/
|
|
4
|
+
import * as bedrock from '@bedrock/core';
|
|
5
|
+
import * as database from '@bedrock/mongodb';
|
|
6
|
+
import {createHash} from 'node:crypto';
|
|
7
|
+
import {EXCHANGE_EXPIRY_GRACE_PERIOD} from '../constants.js';
|
|
8
|
+
import {PassThrough} from 'node:stream';
|
|
9
|
+
import {pipeline} from 'node:stream/promises';
|
|
10
|
+
import {buffer as readIntoBuffer} from 'node:stream/consumers';
|
|
11
|
+
|
|
12
|
+
const {util: {BedrockError}} = bedrock;
|
|
13
|
+
|
|
14
|
+
/* Note: If the total size for `exchange.variables` is less than
|
|
15
|
+
`VARIABLES_SIZE_LIMIT` then the variables are stored in the exchange record.
|
|
16
|
+
Otherwise, the variables must be converted to JSON and externally stored, in
|
|
17
|
+
a gridfs bucket. Additionally, if the variables contain any JSON keys that
|
|
18
|
+
contain an invalid MongoDB key character, the variables must be stored as
|
|
19
|
+
JSON either within the exchange document or externally (where the JSON is
|
|
20
|
+
stored is determined by size limit mentioned above).
|
|
21
|
+
|
|
22
|
+
Importantly, all deployed workflow systems must be upgraded to enable reading
|
|
23
|
+
externalized variables prior to writing any externalized variables. So the
|
|
24
|
+
size limit must be at least 10MiB as old software prohibited submitting any
|
|
25
|
+
payloads larger than this over HTTP. That 10MiB HTTP limit must not be raised
|
|
26
|
+
until all systems have been upgraded to enable reading externalized variables,
|
|
27
|
+
ensuring no externalized variables will be written before then.
|
|
28
|
+
|
|
29
|
+
There still needs to be a cap on the maximum payload that will be accepted
|
|
30
|
+
on various HTTP endpoints to prevent DoS, but once systems support externally
|
|
31
|
+
stored variables, the limit can be raised beyond 10MiB. Additionally, a feature
|
|
32
|
+
to enable different limits per workflow could be implemented if desired (but it
|
|
33
|
+
will require different HTTP body parser setup). */
|
|
34
|
+
|
|
35
|
+
// very generous 1 hour grace period before externalized variables are deleted
|
|
36
|
+
// after an exchange expires to allow for various asynchronous behaviors
|
|
37
|
+
const VARIABLES_EXPIRY_GRACE_PERIOD = 1000 * 60 * 60;
|
|
38
|
+
|
|
39
|
+
// for gridfs storage of large exchange `variables`
|
|
40
|
+
export const VARIABLES_STORAGE = {
|
|
41
|
+
name: 'vc-exchange-variables',
|
|
42
|
+
bucket: null
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
// limit to exchange `variables` size; at this size or larger, `variables`
|
|
46
|
+
// must be externalized and stored in a gridfs bucket
|
|
47
|
+
const VARIABLES_SIZE_LIMIT = 1024 * 1024 * 10;
|
|
48
|
+
|
|
49
|
+
// used to determine whether `variables` can be stored parsed as BSON or must
|
|
50
|
+
// be converted to JSON
|
|
51
|
+
const MONGODB_ILLEGAL_KEY_CHAR_REGEX = /[%$.]/;
|
|
52
|
+
|
|
53
|
+
bedrock.events.on('bedrock-mongodb.ready', async () => {
|
|
54
|
+
await database.openCollections([`${VARIABLES_STORAGE.name}.files`]);
|
|
55
|
+
|
|
56
|
+
VARIABLES_STORAGE.bucket = database.createGridFSBucket({
|
|
57
|
+
bucketName: VARIABLES_STORAGE.name
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
await database.createIndexes([{
|
|
61
|
+
// enables content-based ID lookups in `VARIABLES_STORAGE.name` bucket
|
|
62
|
+
collection: `${VARIABLES_STORAGE.name}.files`,
|
|
63
|
+
fields: {filename: 1},
|
|
64
|
+
options: {
|
|
65
|
+
unique: true
|
|
66
|
+
}
|
|
67
|
+
}, {
|
|
68
|
+
// enables an application worker (defined below) to find and delete expired
|
|
69
|
+
// `files` in the `VARIABLES_STORAGE.name` bucket; a TTL index is not used
|
|
70
|
+
// because `chunks` must also be removed and chunks do not contain a
|
|
71
|
+
// similar metadata field that would allow for robust atomic updates and
|
|
72
|
+
// clean up; so the GridFS file deletion API is used in the application
|
|
73
|
+
// worker
|
|
74
|
+
collection: `${VARIABLES_STORAGE.name}.files`,
|
|
75
|
+
// `metadata` is a built-in property for the `files` document schema; and
|
|
76
|
+
// `expires` will be added to it
|
|
77
|
+
fields: {'metadata.expires': 1},
|
|
78
|
+
options: {
|
|
79
|
+
partialFilterExpression: {
|
|
80
|
+
'metadata.expires': {$exists: true}
|
|
81
|
+
},
|
|
82
|
+
unique: false
|
|
83
|
+
}
|
|
84
|
+
}]);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
export async function encodeVariables({workflowId, exchange, meta}) {
|
|
88
|
+
// express variables as JSON to determine total size; a future optimization
|
|
89
|
+
// might be able to avoid converting to JSON (in some cases)
|
|
90
|
+
const {variables, ...rest} = exchange;
|
|
91
|
+
const variablesJson = JSON.stringify(variables);
|
|
92
|
+
|
|
93
|
+
// any `variables` that are under the size limit can be stored in the
|
|
94
|
+
// exchange document itself
|
|
95
|
+
// FIXME: this version of the software includes `!meta.variablesFilename` in
|
|
96
|
+
// the conditional below to ensure that live updates can occur: it ensures
|
|
97
|
+
// this version of the software can run concurrently with either older
|
|
98
|
+
// versions of the software that have no understanding of externalized
|
|
99
|
+
// `variables` storage or with newer versions that do understand it (but not
|
|
100
|
+
// with both); this works because, until a new (likely major) release enables
|
|
101
|
+
// externalized `variables` storage (by setting `meta.variablesFilename`
|
|
102
|
+
// based on `variables` size), the following conditional will always execute;
|
|
103
|
+
// however, once such a release is made and this version encounters
|
|
104
|
+
// `meta.variablesFilename`, it will use externalized `variables` storage
|
|
105
|
+
// even if `variables` would fit in an exchange document, because this
|
|
106
|
+
// version defers the storage location decision to newer software; note that
|
|
107
|
+
// the new release will remove the `!meta.variablesFilename` check and simply
|
|
108
|
+
// use the size limit
|
|
109
|
+
if(!meta.variablesFilename || variablesJson.length < VARIABLES_SIZE_LIMIT) {
|
|
110
|
+
meta.variablesFilename = false;
|
|
111
|
+
// if any object has a key that uses a character that is not legal in
|
|
112
|
+
// a JSON key in mongoDB document, then stringify all the variables
|
|
113
|
+
return !_hasIllegalMongoDBKeyChar(variables) ?
|
|
114
|
+
exchange : {...rest, variables: variablesJson};
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// generate content-based identifier for filename; noting that an local
|
|
118
|
+
// exchange ID MUST NOT (and currently does not) include a `_` character as
|
|
119
|
+
// it is used as a delimiter here
|
|
120
|
+
const buffer = Buffer.from(variablesJson, 'utf8');
|
|
121
|
+
const filename = `${exchange.id}_${_multibaseMultihashSha256(buffer)}`;
|
|
122
|
+
meta.variablesFilename = filename;
|
|
123
|
+
|
|
124
|
+
// create gridfs file metadata w/expiry; `exchange.expires` MUST be set
|
|
125
|
+
const expires = new Date(exchange.expires);
|
|
126
|
+
const metadata = {
|
|
127
|
+
// add exchange grace period to expiry to cover maximum exchange TTL and
|
|
128
|
+
// add a generous `variables` grace period as well
|
|
129
|
+
expires: new Date(
|
|
130
|
+
expires.getTime() +
|
|
131
|
+
EXCHANGE_EXPIRY_GRACE_PERIOD + VARIABLES_EXPIRY_GRACE_PERIOD)
|
|
132
|
+
};
|
|
133
|
+
|
|
134
|
+
// store variables; any duplicate content-based identifier will throw a
|
|
135
|
+
// duplicate error which can be safely ignored
|
|
136
|
+
const stream = new PassThrough();
|
|
137
|
+
stream.end(buffer);
|
|
138
|
+
try {
|
|
139
|
+
await pipeline(
|
|
140
|
+
stream,
|
|
141
|
+
VARIABLES_STORAGE.bucket.openUploadStream(filename, {metadata}));
|
|
142
|
+
} catch(e) {
|
|
143
|
+
if(!database.isDuplicateError(e)) {
|
|
144
|
+
throw new BedrockError(`Could not store exchange variables.`, {
|
|
145
|
+
name: 'OperationError',
|
|
146
|
+
details: {
|
|
147
|
+
workflow: workflowId,
|
|
148
|
+
exchange: exchange.id,
|
|
149
|
+
public: true,
|
|
150
|
+
httpStatusCode: 500
|
|
151
|
+
},
|
|
152
|
+
cause: e
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
return {...rest};
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
export async function decodeVariables({workflowId, record}) {
|
|
161
|
+
const {exchange, meta} = record;
|
|
162
|
+
|
|
163
|
+
// if `variables` are stored as a string, parse them from JSON
|
|
164
|
+
if(typeof exchange.variables === 'string') {
|
|
165
|
+
return {...exchange, variables: JSON.parse(exchange.variables)};
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// if `meta` indicates that the variables are externalized, then read them
|
|
169
|
+
// from the gridfs variables bucket
|
|
170
|
+
if(meta.variablesFilename) {
|
|
171
|
+
try {
|
|
172
|
+
const {bucket} = VARIABLES_STORAGE;
|
|
173
|
+
const buffer = await readIntoBuffer(
|
|
174
|
+
bucket.openDownloadStreamByName(meta.variablesFilename));
|
|
175
|
+
exchange.variables = JSON.parse(buffer.toString('utf8'));
|
|
176
|
+
} catch(e) {
|
|
177
|
+
throw new BedrockError(`Could not load exchange variables.`, {
|
|
178
|
+
name: 'OperationError',
|
|
179
|
+
details: {
|
|
180
|
+
workflow: workflowId,
|
|
181
|
+
exchange: exchange.id,
|
|
182
|
+
public: true,
|
|
183
|
+
httpStatusCode: 500
|
|
184
|
+
},
|
|
185
|
+
cause: e
|
|
186
|
+
});
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
return exchange;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
function _hasIllegalMongoDBKeyChar(value) {
|
|
194
|
+
if(Array.isArray(value)) {
|
|
195
|
+
for(const e of value) {
|
|
196
|
+
if(_hasIllegalMongoDBKeyChar(e)) {
|
|
197
|
+
return true;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
} else if(value && typeof value === 'object') {
|
|
201
|
+
const keys = Object.keys(value);
|
|
202
|
+
for(const key of keys) {
|
|
203
|
+
if(MONGODB_ILLEGAL_KEY_CHAR_REGEX.test(key) ||
|
|
204
|
+
_hasIllegalMongoDBKeyChar(value[key])) {
|
|
205
|
+
return true;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
return false;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function _multibaseMultihashSha256(buffer) {
|
|
213
|
+
// compute SHA-256 hash
|
|
214
|
+
const digest = createHash('sha256').update(buffer).digest();
|
|
215
|
+
|
|
216
|
+
// format as multihash digest
|
|
217
|
+
// sha2-256: 0x12, length: 32 (0x20), digest value
|
|
218
|
+
const mh = new Uint8Array(34);
|
|
219
|
+
mh[0] = 0x12;
|
|
220
|
+
mh[1] = 0x20;
|
|
221
|
+
mh.set(digest, 2);
|
|
222
|
+
|
|
223
|
+
// return as multibase-base64url-encoded value
|
|
224
|
+
return 'u' + Buffer.from(mh).toString('base64url');
|
|
225
|
+
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* Copyright (c) 2022-2026 Digital Bazaar, Inc.
|
|
3
|
+
*/
|
|
4
|
+
import * as bedrock from '@bedrock/core';
|
|
5
|
+
import {logger} from '../logger.js';
|
|
6
|
+
import {rangeDelay} from 'delay';
|
|
7
|
+
import {VARIABLES_STORAGE} from './variables.js';
|
|
8
|
+
|
|
9
|
+
// state for running a garbage collector for expired externalized `variables`
|
|
10
|
+
const VARIABLES_GARBAGE_COLLECTOR = {
|
|
11
|
+
// used to abort variables garbage collector
|
|
12
|
+
abortController: new AbortController(),
|
|
13
|
+
// a Promise that resolves after the `variables` garbage collector has
|
|
14
|
+
// shutdown cleanly after receiving an abort signal
|
|
15
|
+
shutdownPromise: null
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
bedrock.events.on('bedrock.ready', () => {
|
|
19
|
+
// start the `variables` garbage collector, which runs continuously
|
|
20
|
+
VARIABLES_GARBAGE_COLLECTOR.shutdownPromise = _startGarbageCollector();
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
bedrock.events.on('bedrock.exit', async () => {
|
|
24
|
+
try {
|
|
25
|
+
// abort variables garbage collector
|
|
26
|
+
VARIABLES_GARBAGE_COLLECTOR.abortController.abort();
|
|
27
|
+
logger.debug(
|
|
28
|
+
'Sent abort signal to "variables" garbage collector, ' +
|
|
29
|
+
'waiting for shutdown...');
|
|
30
|
+
await VARIABLES_GARBAGE_COLLECTOR.shutdownPromise;
|
|
31
|
+
logger.debug('"Variables" garbage collector shutdown was successful.');
|
|
32
|
+
} catch(error) {
|
|
33
|
+
logger.error(
|
|
34
|
+
'Error during "variables" garbage collector shutdown.', {error});
|
|
35
|
+
}
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
async function _deleteExpiredVariables({signal}) {
|
|
39
|
+
signal.throwIfAborted();
|
|
40
|
+
|
|
41
|
+
// delete all files found (limit=1000)
|
|
42
|
+
const {bucket} = VARIABLES_STORAGE;
|
|
43
|
+
const now = new Date(Date.now() + 86400 * 1000 * 365);
|
|
44
|
+
const projection = {_id: 1};
|
|
45
|
+
const cursor = bucket.find({
|
|
46
|
+
'metadata.expires': {$lte: now}
|
|
47
|
+
}, {projection}).limit(1000);
|
|
48
|
+
for await (const {_id} of cursor) {
|
|
49
|
+
try {
|
|
50
|
+
signal.throwIfAborted();
|
|
51
|
+
await bucket.delete(_id);
|
|
52
|
+
} catch(e) {
|
|
53
|
+
// ignore file not found errors and throw all others; note: there is
|
|
54
|
+
// currently not a better way to check for a `not found` error than to
|
|
55
|
+
// check the message text
|
|
56
|
+
if(!e.message.includes('not found')) {
|
|
57
|
+
throw e;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async function _startGarbageCollector() {
|
|
64
|
+
const {
|
|
65
|
+
exchanges: {
|
|
66
|
+
variablesGarbageCollector: {interval}
|
|
67
|
+
}
|
|
68
|
+
} = bedrock.config['vc-workflow'];
|
|
69
|
+
const {signal} = VARIABLES_GARBAGE_COLLECTOR.abortController;
|
|
70
|
+
while(!signal.aborted) {
|
|
71
|
+
try {
|
|
72
|
+
// collect expired externalized exchange "variables" then delay for
|
|
73
|
+
// `interval` plus some fuzzing (up to 1 minute) to spread load
|
|
74
|
+
await _deleteExpiredVariables({signal});
|
|
75
|
+
await rangeDelay(interval, interval + 60000, {signal});
|
|
76
|
+
} catch(e) {
|
|
77
|
+
if(e.name === 'AbortError') {
|
|
78
|
+
break;
|
|
79
|
+
}
|
|
80
|
+
logger.error('Error in "variables" garbage collector job.', {error: e});
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}
|
package/lib/vcapi.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
/*!
|
|
2
|
-
* Copyright (c) 2018-2026 Digital Bazaar, Inc.
|
|
2
|
+
* Copyright (c) 2018-2026 Digital Bazaar, Inc.
|
|
3
3
|
*/
|
|
4
4
|
import * as bedrock from '@bedrock/core';
|
|
5
5
|
import * as exchanges from './storage/exchanges.js';
|
|
6
6
|
import {evaluateExchangeStep, generateRandom} from './helpers.js';
|
|
7
|
+
import {EXCHANGE_TTL_DEFAULT, EXCHANGE_TTL_MAX_IN_MS} from './constants.js';
|
|
7
8
|
import {exportJWK, generateKeyPair, importJWK} from 'jose';
|
|
8
9
|
import {ExchangeProcessor} from './ExchangeProcessor.js';
|
|
9
10
|
|
|
@@ -14,15 +15,10 @@ import * as oid4vp from './oid4/oid4vp.js';
|
|
|
14
15
|
|
|
15
16
|
const {util: {BedrockError}} = bedrock;
|
|
16
17
|
|
|
17
|
-
const FIFTEEN_MINUTES = 60 * 15;
|
|
18
|
-
|
|
19
|
-
// 48 hours; make configurable
|
|
20
|
-
const MAX_TTL_IN_MS = 1000 * 60 * 60 * 24 * 2;
|
|
21
|
-
|
|
22
18
|
export async function createExchange({workflow, exchange}) {
|
|
23
19
|
const {
|
|
24
20
|
expires,
|
|
25
|
-
ttl =
|
|
21
|
+
ttl = EXCHANGE_TTL_DEFAULT,
|
|
26
22
|
variables = {},
|
|
27
23
|
// allow steps to be skipped by creator as needed
|
|
28
24
|
step: stepName = workflow.initialStep,
|
|
@@ -54,7 +50,7 @@ export async function createExchange({workflow, exchange}) {
|
|
|
54
50
|
}
|
|
55
51
|
|
|
56
52
|
// should expires isn't too far into the future
|
|
57
|
-
const maxExpires = new Date(Date.now() +
|
|
53
|
+
const maxExpires = new Date(Date.now() + EXCHANGE_TTL_MAX_IN_MS);
|
|
58
54
|
if(new Date(exchange.expires) > maxExpires) {
|
|
59
55
|
throw new BedrockError(
|
|
60
56
|
'Maximum exchange expiration date is "' +
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bedrock/vc-delivery",
|
|
3
|
-
"version": "7.
|
|
3
|
+
"version": "7.16.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Bedrock Verifiable Credential Delivery",
|
|
6
6
|
"main": "./lib/index.js",
|
|
@@ -47,6 +47,7 @@
|
|
|
47
47
|
"bnid": "^3.0.0",
|
|
48
48
|
"body-parser": "^1.20.3",
|
|
49
49
|
"cors": "^2.8.5",
|
|
50
|
+
"delay": "^7.0.0",
|
|
50
51
|
"jose": "^6.1.0",
|
|
51
52
|
"json-pointer": "^0.6.2",
|
|
52
53
|
"jsonata": "^2.0.6",
|
|
@@ -56,7 +57,7 @@
|
|
|
56
57
|
"@bedrock/app-identity": "^4.0.0",
|
|
57
58
|
"@bedrock/core": "^6.3.0",
|
|
58
59
|
"@bedrock/did-io": "^10.4.0",
|
|
59
|
-
"@bedrock/express": "^8.
|
|
60
|
+
"@bedrock/express": "^8.6.4",
|
|
60
61
|
"@bedrock/https-agent": "^4.1.0",
|
|
61
62
|
"@bedrock/mongodb": "^11.0.0",
|
|
62
63
|
"@bedrock/oauth2-verifier": "^2.3.1",
|