@flowfuse/file-server 2.7.1-7ab5d1b-202408010939.0 → 2.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5 -0
- package/forge/context-driver/sequelize.js +137 -96
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,40 @@ const path = require('path')
|
|
|
5
5
|
|
|
6
6
|
let sequelize, app
|
|
7
7
|
|
|
8
|
+
/**
|
|
9
|
+
*/
|
|
10
|
+
const activeLocks = new Map()
|
|
11
|
+
/**
|
|
12
|
+
* This is a simple instanceId-level locking mechanism that ensures we single-thread
|
|
13
|
+
* requests related to a single instance.
|
|
14
|
+
*
|
|
15
|
+
* This is not scalable, but solves an immediate issue around deadlocks caused by
|
|
16
|
+
* parallel requests to update different context scopes for the same instance.
|
|
17
|
+
*
|
|
18
|
+
* See https://github.com/FlowFuse/file-server/issues/122
|
|
19
|
+
* @param {*} instanceId the id of the instance to lock
|
|
20
|
+
* @returns a promise that resolves once the lock is held. The promise resolves with a function that must be called to release the lock
|
|
21
|
+
*/
|
|
22
|
+
async function getInstanceLock (instanceId) {
|
|
23
|
+
let lockingPromise
|
|
24
|
+
if (!activeLocks.has(instanceId)) {
|
|
25
|
+
lockingPromise = Promise.resolve()
|
|
26
|
+
activeLocks.set(instanceId, lockingPromise)
|
|
27
|
+
} else {
|
|
28
|
+
lockingPromise = activeLocks.get(instanceId)
|
|
29
|
+
}
|
|
30
|
+
let unlockNextPromise
|
|
31
|
+
const nextPromise = new Promise(resolve => {
|
|
32
|
+
unlockNextPromise = () => {
|
|
33
|
+
resolve()
|
|
34
|
+
}
|
|
35
|
+
return unlockNextPromise
|
|
36
|
+
})
|
|
37
|
+
const unlockPromise = lockingPromise.then(() => unlockNextPromise)
|
|
38
|
+
activeLocks.set(instanceId, lockingPromise.then(() => nextPromise))
|
|
39
|
+
return unlockPromise
|
|
40
|
+
}
|
|
41
|
+
|
|
8
42
|
module.exports = {
|
|
9
43
|
init: async function (_app) {
|
|
10
44
|
app = _app
|
|
@@ -44,7 +78,7 @@ module.exports = {
|
|
|
44
78
|
|
|
45
79
|
sequelize = new Sequelize(dbOptions)
|
|
46
80
|
|
|
47
|
-
app.log.info(`
|
|
81
|
+
app.log.info(`FlowFuse File Server Sequelize Context connected to ${dbOptions.dialect} on ${dbOptions.host || dbOptions.storage}`)
|
|
48
82
|
|
|
49
83
|
const Context = sequelize.define('Context', {
|
|
50
84
|
project: { type: DataTypes.STRING, allowNull: false, unique: 'context-project-scope-unique' },
|
|
@@ -56,106 +90,113 @@ module.exports = {
|
|
|
56
90
|
},
|
|
57
91
|
/**
|
|
58
92
|
* Set the context data for a given scope
|
|
59
|
-
* @param {string}
|
|
93
|
+
* @param {string} instanceId - The instance id
|
|
60
94
|
* @param {string} scope - The context scope to write to
|
|
61
95
|
* @param {[{key:string, value:any}]} input - The context data to write
|
|
62
96
|
* @param {boolean} [overwrite=false] - If true, any context data will be overwritten (i.e. for a cache dump). If false, the context data will be merged with the existing data.
|
|
63
97
|
* @param {number} quotaOverride - if set overrides the locally configured limit
|
|
64
98
|
*/
|
|
65
|
-
set: async function (
|
|
66
|
-
|
|
67
|
-
await
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
//
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
99
|
+
set: async function (instanceId, scope, input, overwrite = false, quotaOverride = 1000) {
|
|
100
|
+
// Obtain the lock for this instance
|
|
101
|
+
const unlock = await getInstanceLock(instanceId)
|
|
102
|
+
try {
|
|
103
|
+
const { path } = parseScope(scope)
|
|
104
|
+
await sequelize.transaction({
|
|
105
|
+
type: Sequelize.Transaction.TYPES.IMMEDIATE
|
|
106
|
+
},
|
|
107
|
+
async (t) => {
|
|
108
|
+
// get the existing row of context data from the database (if any)
|
|
109
|
+
let existingRow = await this.Context.findOne({
|
|
110
|
+
where: {
|
|
111
|
+
project: instanceId,
|
|
112
|
+
scope: path
|
|
113
|
+
},
|
|
114
|
+
lock: t.LOCK.UPDATE,
|
|
115
|
+
transaction: t
|
|
116
|
+
})
|
|
117
|
+
const quotaLimit = quotaOverride || app.config?.context?.quota || 0
|
|
118
|
+
// if quota is set, check if we are over quota or will be after this update
|
|
119
|
+
if (quotaLimit > 0) {
|
|
120
|
+
// Difficulties implementing this correctly
|
|
121
|
+
// - The final size of data can only be determined after the data is stored.
|
|
122
|
+
// This is due to the fact that some keys may be deleted and some may be added
|
|
123
|
+
// and the size of the data is not the same as the size of the keys.
|
|
124
|
+
// This implementation is not ideal, but it is a good approximation and will
|
|
125
|
+
// prevent the possibility of runaway storage usage.
|
|
126
|
+
let changeSize = 0
|
|
127
|
+
let hasValues = false
|
|
128
|
+
// if we are overwriting, then we need to remove the existing size to get the final size
|
|
129
|
+
if (existingRow) {
|
|
130
|
+
if (overwrite) {
|
|
131
|
+
changeSize -= getItemSize(existingRow.values || '')
|
|
132
|
+
} else {
|
|
133
|
+
hasValues = existingRow?.values && Object.keys(existingRow.values).length > 0
|
|
134
|
+
}
|
|
97
135
|
}
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
136
|
+
// calculate the change in size
|
|
137
|
+
for (const element of input) {
|
|
138
|
+
const currentItem = hasValues ? getObjectProperty(existingRow.values, element.key) : undefined
|
|
139
|
+
if (currentItem === undefined && element.value !== undefined) {
|
|
140
|
+
// this is an addition
|
|
141
|
+
changeSize += getItemSize(element.value)
|
|
142
|
+
} else if (currentItem !== undefined && element.value === undefined) {
|
|
143
|
+
// this is an deletion
|
|
144
|
+
changeSize -= getItemSize(currentItem)
|
|
145
|
+
} else {
|
|
146
|
+
// this is an update
|
|
147
|
+
changeSize -= getItemSize(currentItem)
|
|
148
|
+
changeSize += getItemSize(element.value)
|
|
149
|
+
}
|
|
112
150
|
}
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
151
|
+
// only calculate the current size if we are going to need it
|
|
152
|
+
if (changeSize >= 0) {
|
|
153
|
+
const currentSize = await this.quota(instanceId)
|
|
154
|
+
if (currentSize + changeSize > quotaLimit) {
|
|
155
|
+
const err = new Error('Over Quota')
|
|
156
|
+
err.code = 'over_quota'
|
|
157
|
+
err.error = err.message
|
|
158
|
+
err.limit = quotaLimit
|
|
159
|
+
throw err
|
|
160
|
+
}
|
|
123
161
|
}
|
|
124
162
|
}
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
// if we are overwriting, then we need to reset the values in the existing row (if any)
|
|
128
|
-
if (existingRow && overwrite) {
|
|
129
|
-
existingRow.values = {} // reset the values since this is a mem cache -> DB dump
|
|
130
|
-
}
|
|
131
163
|
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
existingRow = await this.Context.create({
|
|
136
|
-
project: projectId,
|
|
137
|
-
scope: path,
|
|
138
|
-
values: {}
|
|
139
|
-
},
|
|
140
|
-
{
|
|
141
|
-
transaction: t
|
|
142
|
-
})
|
|
164
|
+
// if we are overwriting, then we need to reset the values in the existing row (if any)
|
|
165
|
+
if (existingRow && overwrite) {
|
|
166
|
+
existingRow.values = {} // reset the values since this is a mem cache -> DB dump
|
|
143
167
|
}
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
168
|
+
|
|
169
|
+
// if there is no input, then we are probably deleting the row
|
|
170
|
+
if (input?.length > 0) {
|
|
171
|
+
if (!existingRow) {
|
|
172
|
+
existingRow = await this.Context.create({
|
|
173
|
+
project: instanceId,
|
|
174
|
+
scope: path,
|
|
175
|
+
values: {}
|
|
176
|
+
},
|
|
177
|
+
{
|
|
178
|
+
transaction: t
|
|
179
|
+
})
|
|
180
|
+
}
|
|
181
|
+
for (const i in input) {
|
|
182
|
+
const path = input[i].key
|
|
183
|
+
const value = input[i].value
|
|
184
|
+
util.setMessageProperty(existingRow.values, path, value)
|
|
185
|
+
}
|
|
148
186
|
}
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
187
|
+
if (existingRow) {
|
|
188
|
+
if (existingRow.values && Object.keys(existingRow.values).length === 0) {
|
|
189
|
+
await existingRow.destroy({ transaction: t })
|
|
190
|
+
} else {
|
|
191
|
+
existingRow.changed('values', true)
|
|
192
|
+
await existingRow.save({ transaction: t })
|
|
193
|
+
}
|
|
156
194
|
}
|
|
157
|
-
}
|
|
158
|
-
}
|
|
195
|
+
})
|
|
196
|
+
} finally {
|
|
197
|
+
// Regardless of the result, release the lock
|
|
198
|
+
await unlock()
|
|
199
|
+
}
|
|
159
200
|
},
|
|
160
201
|
/**
|
|
161
202
|
* Get the context data for a given scope
|
|
@@ -295,17 +336,17 @@ module.exports = {
|
|
|
295
336
|
}
|
|
296
337
|
},
|
|
297
338
|
quota: async function (projectId) {
|
|
298
|
-
|
|
339
|
+
// Sum the lengths in the query
|
|
340
|
+
// - note for postgres, we have to cast the values column from JSON to text
|
|
341
|
+
const sizeResult = await this.Context.findOne({
|
|
299
342
|
where: {
|
|
300
343
|
project: projectId
|
|
301
|
-
}
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
const strValues = JSON.stringify(scope.values)
|
|
306
|
-
size += strValues.length
|
|
344
|
+
},
|
|
345
|
+
attributes: [
|
|
346
|
+
[sequelize.fn('SUM', sequelize.fn('LENGTH', sequelize.cast(sequelize.col('values'), 'text'))), 'length']
|
|
347
|
+
]
|
|
307
348
|
})
|
|
308
|
-
return
|
|
349
|
+
return sizeResult.getDataValue('length') || 0
|
|
309
350
|
}
|
|
310
351
|
}
|
|
311
352
|
|