@balena/pinejs 21.3.0 → 21.4.0-build-large-file-uploads-2-917f66f4e89b38adb4b0671769e30d678c031448-1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pinejs-cache.json +1 -1
- package/.versionbot/CHANGELOG.yml +37 -1
- package/CHANGELOG.md +7 -1
- package/VERSION +1 -1
- package/out/config-loader/env.d.ts +4 -0
- package/out/config-loader/env.js +4 -0
- package/out/config-loader/env.js.map +1 -1
- package/out/server-glue/module.js +2 -0
- package/out/server-glue/module.js.map +1 -1
- package/out/webresource-handler/index.d.ts +9 -0
- package/out/webresource-handler/index.js +24 -1
- package/out/webresource-handler/index.js.map +1 -1
- package/out/webresource-handler/multipartUpload.d.ts +12 -0
- package/out/webresource-handler/multipartUpload.js +249 -0
- package/out/webresource-handler/multipartUpload.js.map +1 -0
- package/out/webresource-handler/webresource.d.ts +42 -0
- package/out/webresource-handler/webresource.js +2 -0
- package/out/webresource-handler/webresource.js.map +1 -0
- package/out/webresource-handler/webresource.sbvr +60 -0
- package/package.json +5 -5
- package/src/config-loader/env.ts +11 -0
- package/src/server-glue/module.ts +2 -0
- package/src/webresource-handler/index.ts +44 -0
- package/src/webresource-handler/multipartUpload.ts +367 -0
- package/src/webresource-handler/webresource.sbvr +60 -0
- package/src/webresource-handler/webresource.ts +48 -0
@@ -0,0 +1,42 @@
|
|
1
|
+
import type { Types } from '@balena/abstract-sql-to-typescript';
|
2
|
+
export interface MultipartUpload {
|
3
|
+
Read: {
|
4
|
+
created_at: Types['Date Time']['Read'];
|
5
|
+
modified_at: Types['Date Time']['Read'];
|
6
|
+
id: Types['Serial']['Read'];
|
7
|
+
uuid: Types['Short Text']['Read'];
|
8
|
+
resource_name: Types['Short Text']['Read'];
|
9
|
+
field_name: Types['Short Text']['Read'];
|
10
|
+
resource_id: Types['Integer']['Read'];
|
11
|
+
upload_id: Types['Short Text']['Read'];
|
12
|
+
file_key: Types['Short Text']['Read'];
|
13
|
+
status: 'pending' | 'completed' | 'cancelled';
|
14
|
+
filename: Types['Short Text']['Read'];
|
15
|
+
content_type: Types['Short Text']['Read'];
|
16
|
+
size: Types['Integer']['Read'];
|
17
|
+
chunk_size: Types['Integer']['Read'];
|
18
|
+
expiry_date: Types['Date Time']['Read'];
|
19
|
+
is_created_by__actor: Types['Integer']['Read'] | null;
|
20
|
+
};
|
21
|
+
Write: {
|
22
|
+
created_at: Types['Date Time']['Write'];
|
23
|
+
modified_at: Types['Date Time']['Write'];
|
24
|
+
id: Types['Serial']['Write'];
|
25
|
+
uuid: Types['Short Text']['Write'];
|
26
|
+
resource_name: Types['Short Text']['Write'];
|
27
|
+
field_name: Types['Short Text']['Write'];
|
28
|
+
resource_id: Types['Integer']['Write'];
|
29
|
+
upload_id: Types['Short Text']['Write'];
|
30
|
+
file_key: Types['Short Text']['Write'];
|
31
|
+
status: 'pending' | 'completed' | 'cancelled';
|
32
|
+
filename: Types['Short Text']['Write'];
|
33
|
+
content_type: Types['Short Text']['Write'];
|
34
|
+
size: Types['Integer']['Write'];
|
35
|
+
chunk_size: Types['Integer']['Write'];
|
36
|
+
expiry_date: Types['Date Time']['Write'];
|
37
|
+
is_created_by__actor: Types['Integer']['Write'] | null;
|
38
|
+
};
|
39
|
+
}
|
40
|
+
export default interface $Model {
|
41
|
+
multipart_upload: MultipartUpload;
|
42
|
+
}
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"webresource.js","sourceRoot":"","sources":["../../src/webresource-handler/webresource.ts"],"names":[],"mappings":""}
|
@@ -0,0 +1,60 @@
|
|
1
|
+
Vocabulary: webresource
|
2
|
+
|
3
|
+
Term: actor
|
4
|
+
Concept Type: Integer (Type)
|
5
|
+
Term: expiry date
|
6
|
+
Concept Type: Date Time (Type)
|
7
|
+
Term: uuid
|
8
|
+
Concept Type: Short Text (Type)
|
9
|
+
Term: resource name
|
10
|
+
Concept Type: Short Text (Type)
|
11
|
+
Term: field name
|
12
|
+
Concept Type: Short Text (Type)
|
13
|
+
Term: resource id
|
14
|
+
Concept Type: Integer (Type)
|
15
|
+
Term: upload id
|
16
|
+
Concept Type: Short Text (Type)
|
17
|
+
Term: file key
|
18
|
+
Concept Type: Short Text (Type)
|
19
|
+
Term: status
|
20
|
+
Concept Type: Short Text (Type)
|
21
|
+
Term: filename
|
22
|
+
Concept Type: Short Text (Type)
|
23
|
+
Term: content type
|
24
|
+
Concept Type: Short Text (Type)
|
25
|
+
Term: size
|
26
|
+
Concept Type: Integer (Type)
|
27
|
+
Term: chunk size
|
28
|
+
Concept Type: Integer (Type)
|
29
|
+
Term: valid until date
|
30
|
+
Concept Type: Date Time (Type)
|
31
|
+
|
32
|
+
Term: multipart upload
|
33
|
+
Fact type: multipart upload has uuid
|
34
|
+
Necessity: each multipart upload has exactly one uuid
|
35
|
+
Necessity: each uuid is of exactly one multipart upload
|
36
|
+
Fact type: multipart upload has resource name
|
37
|
+
Necessity: each multipart upload has exactly one resource name
|
38
|
+
Fact type: multipart upload has field name
|
39
|
+
Necessity: each multipart upload has exactly one field name
|
40
|
+
Fact type: multipart upload has resource id
|
41
|
+
Necessity: each multipart upload has exactly one resource id
|
42
|
+
Fact type: multipart upload has upload id
|
43
|
+
Necessity: each multipart upload has exactly one upload id
|
44
|
+
Fact type: multipart upload has file key
|
45
|
+
Necessity: each multipart upload has exactly one file key
|
46
|
+
Fact type: multipart upload has status
|
47
|
+
Necessity: each multipart upload has exactly one status
|
48
|
+
Definition: "pending" or "completed" or "cancelled"
|
49
|
+
Fact type: multipart upload has filename
|
50
|
+
Necessity: each multipart upload has exactly one filename
|
51
|
+
Fact type: multipart upload has content type
|
52
|
+
Necessity: each multipart upload has exactly one content type
|
53
|
+
Fact type: multipart upload has size
|
54
|
+
Necessity: each multipart upload has exactly one size
|
55
|
+
Fact type: multipart upload has chunk size
|
56
|
+
Necessity: each multipart upload has exactly one chunk size
|
57
|
+
Fact type: multipart upload has expiry date
|
58
|
+
Necessity: each multipart upload has exactly one expiry date
|
59
|
+
Fact type: multipart upload is created by actor
|
60
|
+
Necessity: each multipart upload is created by at most one actor
|
package/package.json
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
{
|
2
2
|
"name": "@balena/pinejs",
|
3
|
-
"version": "21.
|
3
|
+
"version": "21.4.0-build-large-file-uploads-2-917f66f4e89b38adb4b0671769e30d678c031448-1",
|
4
4
|
"main": "out/server-glue/module.js",
|
5
5
|
"type": "module",
|
6
6
|
"repository": "git@github.com:balena-io/pinejs.git",
|
@@ -20,10 +20,10 @@
|
|
20
20
|
"webpack-build": "npm run webpack-browser && npm run webpack-module && npm run webpack-server",
|
21
21
|
"lint": "balena-lint -t tsconfig.dev.json -e js -e ts src test build typings Gruntfile.cts && npx tsc --project tsconfig.dev.json --noEmit",
|
22
22
|
"test": "npm run build && npm run lint && npm run webpack-build && npm run test:compose && npm run test:generated-types",
|
23
|
-
"test:compose": "trap 'docker compose -f docker-compose.npm-test.yml down ; echo Stopped ; exit 0' INT; docker compose -f docker-compose.npm-test.yml up -d && sleep 2 && DATABASE_URL=postgres://docker:docker@localhost:5431/postgres PINEJS_WEBRESOURCE_MAXFILESIZE=1000000000 S3_ENDPOINT=http://localhost:43680 S3_ACCESS_KEY=USERNAME S3_SECRET_KEY=PASSWORD S3_STORAGE_ADAPTER_BUCKET=balena-pine-web-resources S3_REGION=us-east-1 PINEJS_QUEUE_CONCURRENCY=1 TZ=UTC npx mocha",
|
23
|
+
"test:compose": "trap 'docker compose -f docker-compose.npm-test.yml down ; echo Stopped ; exit 0' INT; docker compose -f docker-compose.npm-test.yml up -d && sleep 2 && DATABASE_URL=postgres://docker:docker@localhost:5431/postgres PINEJS_WEBRESOURCE_MAXFILESIZE=1000000000 S3_ENDPOINT=http://localhost:43680 S3_ACCESS_KEY=USERNAME S3_SECRET_KEY=PASSWORD S3_STORAGE_ADAPTER_BUCKET=balena-pine-web-resources S3_REGION=us-east-1 PINEJS_QUEUE_CONCURRENCY=1 TZ=UTC PINEJS_WEBRESOURCE_MULTIPART_ENABLED=true npx mocha",
|
24
24
|
"test:generated-types": "npm run generate-types && git diff --exit-code ./src/sbvr-api/user.ts ./src/migrator/migrations.ts ./src/sbvr-api/dev.ts",
|
25
25
|
"lint-fix": "balena-lint -t tsconfig.dev.json -e js -e ts --fix src test build typings Gruntfile.cts",
|
26
|
-
"generate-types": "node ./bin/sbvr-compiler.js generate-types ./src/sbvr-api/user.sbvr ./src/sbvr-api/user.ts && node ./bin/sbvr-compiler.js generate-types ./src/migrator/migrations.sbvr ./src/migrator/migrations.ts && node ./bin/sbvr-compiler.js generate-types ./src/sbvr-api/dev.sbvr ./src/sbvr-api/dev.ts && node ./bin/sbvr-compiler.js generate-types ./src/tasks/tasks.sbvr ./src/tasks/tasks.ts && balena-lint -t tsconfig.dev.json --fix ./src/sbvr-api/user.ts ./src/migrator/migrations.ts ./src/sbvr-api/dev.ts"
|
26
|
+
"generate-types": "node ./bin/sbvr-compiler.js generate-types ./src/sbvr-api/user.sbvr ./src/sbvr-api/user.ts && node ./bin/sbvr-compiler.js generate-types ./src/migrator/migrations.sbvr ./src/migrator/migrations.ts && node ./bin/sbvr-compiler.js generate-types ./src/sbvr-api/dev.sbvr ./src/sbvr-api/dev.ts && node ./bin/sbvr-compiler.js generate-types ./src/tasks/tasks.sbvr ./src/tasks/tasks.ts && node ./bin/sbvr-compiler.js generate-types ./src/webresource-handler/webresource.sbvr ./src/webresource-handler/webresource.ts && balena-lint -t tsconfig.dev.json --fix ./src/sbvr-api/user.ts ./src/migrator/migrations.ts ./src/sbvr-api/dev.ts"
|
27
27
|
},
|
28
28
|
"dependencies": {
|
29
29
|
"@balena/abstract-sql-compiler": "^10.2.3",
|
@@ -70,7 +70,7 @@
|
|
70
70
|
"devDependencies": {
|
71
71
|
"@balena/lint": "^9.1.6",
|
72
72
|
"@balena/pinejs": "file:./",
|
73
|
-
"@balena/pinejs-webresource-s3": "
|
73
|
+
"@balena/pinejs-webresource-s3": "2.0.0-build-new-multiparthandle-interface-66b03581234eaa7ce15c6f389e39b5b7ed3d1bc5-1",
|
74
74
|
"@faker-js/faker": "^9.6.0",
|
75
75
|
"@types/busboy": "^1.5.4",
|
76
76
|
"@types/chai": "^5.2.1",
|
@@ -149,6 +149,6 @@
|
|
149
149
|
"recursive": true
|
150
150
|
},
|
151
151
|
"versionist": {
|
152
|
-
"publishedAt": "2025-04-
|
152
|
+
"publishedAt": "2025-04-29T13:05:22.353Z"
|
153
153
|
}
|
154
154
|
}
|
package/src/config-loader/env.ts
CHANGED
@@ -159,6 +159,17 @@ export const tasks = {
|
|
159
159
|
queueIntervalMS: intVar('PINEJS_QUEUE_INTERVAL_MS', 1000),
|
160
160
|
};
|
161
161
|
|
162
|
+
export const webResource = {
|
163
|
+
multipartUploadEnabled: boolVar(
|
164
|
+
'PINEJS_WEBRESOURCE_MULTIPART_ENABLED',
|
165
|
+
false,
|
166
|
+
),
|
167
|
+
singleUploadMaxFilesize: intVar(
|
168
|
+
'PINEJS_WEBRESOURCE_MAXFILESIZE',
|
169
|
+
299 * 1024 * 1024,
|
170
|
+
),
|
171
|
+
};
|
172
|
+
|
162
173
|
export const guardTestMockOnly = () => {
|
163
174
|
if (process.env.DEPLOYMENT !== 'TEST') {
|
164
175
|
throw new Error('Attempting to use TEST_MOCK_ONLY outside of tests');
|
@@ -7,6 +7,7 @@ import * as configLoader from '../config-loader/config-loader.js';
|
|
7
7
|
import * as migrator from '../migrator/sync.js';
|
8
8
|
import type * as migratorUtils from '../migrator/utils.js';
|
9
9
|
import * as tasks from '../tasks/index.js';
|
10
|
+
import * as webresource from '../webresource-handler/index.js';
|
10
11
|
|
11
12
|
import * as sbvrUtils from '../sbvr-api/sbvr-utils.js';
|
12
13
|
import { PINEJS_ADVISORY_LOCK } from '../config-loader/env.js';
|
@@ -66,6 +67,7 @@ export const init = async <T extends string>(
|
|
66
67
|
const cfgLoader = configLoader.setup(app);
|
67
68
|
await cfgLoader.loadConfig(migrator.config);
|
68
69
|
await cfgLoader.loadConfig(tasks.config);
|
70
|
+
await cfgLoader.loadConfig(webresource.config);
|
69
71
|
|
70
72
|
if (!process.env.CONFIG_LOADER_DISABLED) {
|
71
73
|
await cfgLoader.loadApplicationConfig(config);
|
@@ -11,12 +11,20 @@ import {
|
|
11
11
|
odataNameToSqlName,
|
12
12
|
sqlNameToODataName,
|
13
13
|
} from '@balena/odata-to-abstract-sql';
|
14
|
+
import type { ConfigLoader } from '../server-glue/module.js';
|
14
15
|
import { errors, permissions } from '../server-glue/module.js';
|
15
16
|
import type { WebResourceType as WebResource } from '@balena/sbvr-types';
|
16
17
|
import { TypedError } from 'typed-error';
|
17
18
|
import type { Resolvable } from '../sbvr-api/common-types.js';
|
19
|
+
import type WebresourceModel from './webresource.js';
|
20
|
+
import { importSBVR } from '../server-glue/sbvr-loader.js';
|
21
|
+
import {
|
22
|
+
isMultipartUploadAvailable,
|
23
|
+
multipartUploadHooks,
|
24
|
+
} from './multipartUpload.js';
|
18
25
|
|
19
26
|
export * from './handlers/index.js';
|
27
|
+
export type { BeginUploadResponse } from './multipartUpload.js';
|
20
28
|
|
21
29
|
export interface IncomingFile {
|
22
30
|
fieldname: string;
|
@@ -329,6 +337,9 @@ const throwIfWebresourceNotInMultipart = (
|
|
329
337
|
{ req, request }: HookArgs,
|
330
338
|
) => {
|
331
339
|
if (
|
340
|
+
request.custom.isAction !== 'beginUpload' &&
|
341
|
+
request.custom.isAction !== 'commitUpload' &&
|
342
|
+
request.custom.isAction !== 'cancelUpload' &&
|
332
343
|
!req.is?.('multipart') &&
|
333
344
|
webResourceFields.some((field) => request.values[field] != null)
|
334
345
|
) {
|
@@ -525,4 +536,37 @@ export const setupUploadHooks = (
|
|
525
536
|
resourceName,
|
526
537
|
getCreateWebResourceHooks(handler),
|
527
538
|
);
|
539
|
+
|
540
|
+
if (isMultipartUploadAvailable(handler)) {
|
541
|
+
sbvrUtils.addPureHook(
|
542
|
+
'POST',
|
543
|
+
apiRoot,
|
544
|
+
resourceName,
|
545
|
+
multipartUploadHooks(handler),
|
546
|
+
);
|
547
|
+
}
|
548
|
+
};
|
549
|
+
|
550
|
+
const initSql = `
|
551
|
+
CREATE INDEX IF NOT EXISTS idx_multipart_upload_uuid ON "multipart upload" (uuid);
|
552
|
+
CREATE INDEX IF NOT EXISTS idx_multipart_upload_status ON "multipart upload" (status);
|
553
|
+
`;
|
554
|
+
|
555
|
+
const modelText = await importSBVR('./webresource.sbvr', import.meta);
|
556
|
+
|
557
|
+
declare module '../sbvr-api/sbvr-utils.js' {
|
558
|
+
export interface API {
|
559
|
+
webresource: PinejsClient<WebresourceModel>;
|
560
|
+
}
|
561
|
+
}
|
562
|
+
|
563
|
+
export const config: ConfigLoader.Config = {
|
564
|
+
models: [
|
565
|
+
{
|
566
|
+
modelName: 'webresource',
|
567
|
+
apiRoot: 'webresource',
|
568
|
+
modelText,
|
569
|
+
initSql,
|
570
|
+
},
|
571
|
+
],
|
528
572
|
};
|
@@ -0,0 +1,367 @@
|
|
1
|
+
import type { WebResourceType as WebResource } from '@balena/sbvr-types';
|
2
|
+
import { randomUUID } from 'node:crypto';
|
3
|
+
import type { AnyObject } from 'pinejs-client-core';
|
4
|
+
import type {
|
5
|
+
BeginMultipartUploadPayload,
|
6
|
+
UploadPart,
|
7
|
+
WebResourceHandler,
|
8
|
+
} from './index.js';
|
9
|
+
import { getWebResourceFields } from './index.js';
|
10
|
+
import type { PinejsClient } from '../sbvr-api/sbvr-utils.js';
|
11
|
+
import { api } from '../sbvr-api/sbvr-utils.js';
|
12
|
+
import type { ODataRequest } from '../sbvr-api/uri-parser.js';
|
13
|
+
import { errors, sbvrUtils } from '../server-glue/module.js';
|
14
|
+
import { webResource as webResourceEnv } from '../config-loader/env.js';
|
15
|
+
import * as permissions from '../sbvr-api/permissions.js';
|
16
|
+
|
17
|
+
type BeginUploadDbCheck = BeginMultipartUploadPayload & WebResource;
|
18
|
+
|
19
|
+
export interface BeginUploadResponse {
|
20
|
+
[fieldName: string]: {
|
21
|
+
uuid: string;
|
22
|
+
uploadParts: UploadPart[];
|
23
|
+
};
|
24
|
+
}
|
25
|
+
|
26
|
+
// Is WebResourceHandler but with beginMultipartUpload and commitMultipartUpload as not optional
|
27
|
+
type MultipartUploadHandler = WebResourceHandler &
|
28
|
+
Required<Pick<WebResourceHandler, 'multipartUpload'>>;
|
29
|
+
|
30
|
+
export const isMultipartUploadAvailable = (
|
31
|
+
webResourceHandler: WebResourceHandler,
|
32
|
+
): webResourceHandler is MultipartUploadHandler => {
|
33
|
+
return (
|
34
|
+
webResourceEnv.multipartUploadEnabled &&
|
35
|
+
webResourceHandler.multipartUpload != null
|
36
|
+
);
|
37
|
+
};
|
38
|
+
|
39
|
+
export const multipartUploadHooks = (
|
40
|
+
webResourceHandler: MultipartUploadHandler,
|
41
|
+
): sbvrUtils.Hooks => {
|
42
|
+
return {
|
43
|
+
POSTPARSE: async ({ req, request, tx, api: applicationApi }) => {
|
44
|
+
if (request.odataQuery.property?.resource === 'beginUpload') {
|
45
|
+
const uploadParams = await validateBeginUpload(
|
46
|
+
request,
|
47
|
+
applicationApi,
|
48
|
+
webResourceHandler,
|
49
|
+
);
|
50
|
+
|
51
|
+
// This transaction is necessary because beginUpload requests
|
52
|
+
// will rollback the transaction (in order to first validate)
|
53
|
+
// The metadata requested. If we don't pass any transaction
|
54
|
+
// It will use the default transaction handler which will error out
|
55
|
+
// on any rollback.
|
56
|
+
tx = await sbvrUtils.db.transaction();
|
57
|
+
req.tx = tx;
|
58
|
+
request.tx = tx;
|
59
|
+
|
60
|
+
request.method = 'PATCH';
|
61
|
+
request.values = uploadParams;
|
62
|
+
request.odataQuery.resource = request.resourceName;
|
63
|
+
delete request.odataQuery.property;
|
64
|
+
request.custom.isAction = 'beginUpload';
|
65
|
+
} else if (request.odataQuery.property?.resource === 'commitUpload') {
|
66
|
+
const commitPayload = await validateCommitUpload(
|
67
|
+
request,
|
68
|
+
applicationApi,
|
69
|
+
);
|
70
|
+
|
71
|
+
const webresource = await webResourceHandler.multipartUpload.commit({
|
72
|
+
fileKey: commitPayload.metadata.fileKey,
|
73
|
+
uploadId: commitPayload.metadata.uploadId,
|
74
|
+
filename: commitPayload.metadata.filename,
|
75
|
+
providerCommitData: commitPayload.providerCommitData,
|
76
|
+
});
|
77
|
+
|
78
|
+
await api.webresource.patch({
|
79
|
+
resource: 'multipart_upload',
|
80
|
+
body: {
|
81
|
+
status: 'completed',
|
82
|
+
},
|
83
|
+
options: {
|
84
|
+
$filter: {
|
85
|
+
uuid: commitPayload.uuid,
|
86
|
+
},
|
87
|
+
},
|
88
|
+
passthrough: {
|
89
|
+
tx: tx,
|
90
|
+
req: permissions.root,
|
91
|
+
},
|
92
|
+
});
|
93
|
+
|
94
|
+
request.method = 'PATCH';
|
95
|
+
request.values = {
|
96
|
+
[commitPayload.metadata.fieldName]: webresource,
|
97
|
+
};
|
98
|
+
request.odataQuery.resource = request.resourceName;
|
99
|
+
delete request.odataQuery.property;
|
100
|
+
request.custom.isAction = 'commitUpload';
|
101
|
+
request.custom.commitUploadPayload = webresource;
|
102
|
+
} else if (request.odataQuery.property?.resource === 'cancelUpload') {
|
103
|
+
const { uuid, fileKey, uploadId } = await validateCancelPayload(
|
104
|
+
request,
|
105
|
+
applicationApi,
|
106
|
+
);
|
107
|
+
|
108
|
+
await webResourceHandler.multipartUpload.cancel({ fileKey, uploadId });
|
109
|
+
|
110
|
+
await api.webresource.patch({
|
111
|
+
resource: 'multipart_upload',
|
112
|
+
body: {
|
113
|
+
status: 'cancelled',
|
114
|
+
},
|
115
|
+
options: {
|
116
|
+
$filter: { uuid },
|
117
|
+
},
|
118
|
+
passthrough: {
|
119
|
+
tx: tx,
|
120
|
+
req: permissions.root,
|
121
|
+
},
|
122
|
+
});
|
123
|
+
|
124
|
+
request.method = 'GET';
|
125
|
+
request.odataQuery.resource = request.resourceName;
|
126
|
+
delete request.odataQuery.property;
|
127
|
+
request.custom.isAction = 'cancelUpload';
|
128
|
+
}
|
129
|
+
},
|
130
|
+
PRERESPOND: async ({ req, request, response, tx }) => {
|
131
|
+
if (request.custom.isAction === 'beginUpload') {
|
132
|
+
// In the case where the transaction has failed because it had invalid payload
|
133
|
+
// such as breaking a db constraint, this hook wouldn't have been called
|
134
|
+
// and would rather throw with the rule it failed to validate
|
135
|
+
// We rollback here as the patch was just a way to validate the upload payload
|
136
|
+
await tx.rollback();
|
137
|
+
|
138
|
+
response.statusCode = 200;
|
139
|
+
response.body = await beginUpload({
|
140
|
+
webResourceHandler,
|
141
|
+
odataRequest: request,
|
142
|
+
actorId: req.user?.actor,
|
143
|
+
});
|
144
|
+
} else if (request.custom.isAction === 'commitUpload') {
|
145
|
+
response.body = await webResourceHandler.onPreRespond(
|
146
|
+
request.custom.commitUploadPayload,
|
147
|
+
);
|
148
|
+
} else if (request.custom.isAction === 'cancelUpload') {
|
149
|
+
response.statusCode = 204;
|
150
|
+
delete response.body;
|
151
|
+
}
|
152
|
+
},
|
153
|
+
};
|
154
|
+
};
|
155
|
+
|
156
|
+
const beginUpload = async ({
|
157
|
+
webResourceHandler,
|
158
|
+
odataRequest,
|
159
|
+
actorId,
|
160
|
+
}: {
|
161
|
+
webResourceHandler: MultipartUploadHandler;
|
162
|
+
odataRequest: ODataRequest;
|
163
|
+
actorId?: number;
|
164
|
+
}): Promise<BeginUploadResponse> => {
|
165
|
+
const payload = odataRequest.values as {
|
166
|
+
[x: string]: BeginMultipartUploadPayload;
|
167
|
+
};
|
168
|
+
const fieldName = Object.keys(payload)[0];
|
169
|
+
const metadata = payload[fieldName];
|
170
|
+
const { fileKey, uploadId, uploadParts } =
|
171
|
+
await webResourceHandler.multipartUpload.begin(fieldName, metadata);
|
172
|
+
const uuid = randomUUID();
|
173
|
+
|
174
|
+
return await sbvrUtils.db.transaction(async (tx) => {
|
175
|
+
try {
|
176
|
+
await api.webresource.post({
|
177
|
+
resource: 'multipart_upload',
|
178
|
+
body: {
|
179
|
+
uuid,
|
180
|
+
resource_name: odataRequest.resourceName,
|
181
|
+
field_name: fieldName,
|
182
|
+
resource_id: odataRequest.affectedIds?.[0],
|
183
|
+
upload_id: uploadId,
|
184
|
+
file_key: fileKey,
|
185
|
+
status: 'pending',
|
186
|
+
filename: metadata.filename,
|
187
|
+
content_type: metadata.content_type,
|
188
|
+
size: metadata.size,
|
189
|
+
chunk_size: metadata.chunk_size,
|
190
|
+
expiry_date: Date.now() + 7 * 24 * 60 * 60 * 1000, // 7 days in ms
|
191
|
+
is_created_by__actor: actorId,
|
192
|
+
},
|
193
|
+
passthrough: {
|
194
|
+
req: permissions.root,
|
195
|
+
tx,
|
196
|
+
},
|
197
|
+
});
|
198
|
+
return { [fieldName]: { uuid, uploadParts } };
|
199
|
+
} catch (err) {
|
200
|
+
console.error('failed to start multipart upload', err);
|
201
|
+
throw new errors.BadRequestError('Failed to start multipart upload');
|
202
|
+
}
|
203
|
+
});
|
204
|
+
};
|
205
|
+
|
206
|
+
const validateBeginUpload = async (
|
207
|
+
request: ODataRequest,
|
208
|
+
applicationApi: PinejsClient,
|
209
|
+
webResourceHandler: MultipartUploadHandler,
|
210
|
+
) => {
|
211
|
+
await canAccess(request, applicationApi);
|
212
|
+
|
213
|
+
const fieldNames = Object.keys(request.values);
|
214
|
+
if (fieldNames.length !== 1) {
|
215
|
+
throw new errors.BadRequestError(
|
216
|
+
'You can only get upload url for one field at a time',
|
217
|
+
);
|
218
|
+
}
|
219
|
+
|
220
|
+
const [fieldName] = fieldNames;
|
221
|
+
const webResourceFields = getWebResourceFields(request);
|
222
|
+
if (!webResourceFields.includes(fieldName)) {
|
223
|
+
throw new errors.BadRequestError(
|
224
|
+
`The provided field '${fieldName}' is not a valid webresource`,
|
225
|
+
);
|
226
|
+
}
|
227
|
+
|
228
|
+
const beginUploadPayload = parseBeginUploadPayload(
|
229
|
+
request.values[fieldName],
|
230
|
+
webResourceHandler,
|
231
|
+
);
|
232
|
+
if (beginUploadPayload == null) {
|
233
|
+
throw new errors.BadRequestError('Invalid file metadata');
|
234
|
+
}
|
235
|
+
|
236
|
+
const uploadMetadataCheck: BeginUploadDbCheck = {
|
237
|
+
...beginUploadPayload,
|
238
|
+
// This is "probe" request. We don't actually store anything on the application table yet
|
239
|
+
// We just avoid creating the application record if it would fail anyway for some other db constraint
|
240
|
+
href: 'metadata_check_probe',
|
241
|
+
};
|
242
|
+
|
243
|
+
return { [fieldName]: uploadMetadataCheck };
|
244
|
+
};
|
245
|
+
|
246
|
+
const parseBeginUploadPayload = (
|
247
|
+
payload: AnyObject,
|
248
|
+
webResourceHandler: MultipartUploadHandler,
|
249
|
+
): BeginMultipartUploadPayload | null => {
|
250
|
+
if (payload == null || typeof payload !== 'object') {
|
251
|
+
return null;
|
252
|
+
}
|
253
|
+
|
254
|
+
let { filename, content_type, size, chunk_size } = payload;
|
255
|
+
if (
|
256
|
+
typeof filename !== 'string' ||
|
257
|
+
typeof content_type !== 'string' ||
|
258
|
+
typeof size !== 'number' ||
|
259
|
+
(chunk_size != null && typeof chunk_size !== 'number') ||
|
260
|
+
(chunk_size != null &&
|
261
|
+
chunk_size < webResourceHandler.multipartUpload.getMinimumPartSize())
|
262
|
+
) {
|
263
|
+
return null;
|
264
|
+
}
|
265
|
+
|
266
|
+
chunk_size ??= webResourceHandler.multipartUpload.getDefaultPartSize();
|
267
|
+
|
268
|
+
return { filename, content_type, size, chunk_size };
|
269
|
+
};
|
270
|
+
|
271
|
+
const validateCommitUpload = async (
|
272
|
+
request: ODataRequest,
|
273
|
+
applicationApi: PinejsClient,
|
274
|
+
) => {
|
275
|
+
await canAccess(request, applicationApi);
|
276
|
+
|
277
|
+
const { uuid, providerCommitData } = request.values;
|
278
|
+
if (typeof uuid !== 'string') {
|
279
|
+
throw new errors.BadRequestError('Invalid uuid type');
|
280
|
+
}
|
281
|
+
|
282
|
+
const [multipartUpload] = await api.webresource.get({
|
283
|
+
resource: 'multipart_upload',
|
284
|
+
options: {
|
285
|
+
$select: ['id', 'file_key', 'upload_id', 'field_name', 'filename'],
|
286
|
+
$filter: {
|
287
|
+
uuid,
|
288
|
+
status: 'pending',
|
289
|
+
expiry_date: { $gt: { $now: {} } },
|
290
|
+
},
|
291
|
+
},
|
292
|
+
passthrough: {
|
293
|
+
tx: request.tx,
|
294
|
+
req: permissions.rootRead,
|
295
|
+
},
|
296
|
+
});
|
297
|
+
|
298
|
+
if (multipartUpload == null) {
|
299
|
+
throw new errors.BadRequestError(`Invalid upload for uuid ${uuid}`);
|
300
|
+
}
|
301
|
+
|
302
|
+
const metadata = {
|
303
|
+
fileKey: multipartUpload.file_key,
|
304
|
+
uploadId: multipartUpload.upload_id,
|
305
|
+
filename: multipartUpload.filename,
|
306
|
+
fieldName: multipartUpload.field_name,
|
307
|
+
};
|
308
|
+
|
309
|
+
return { uuid, providerCommitData, metadata };
|
310
|
+
};
|
311
|
+
|
312
|
+
const validateCancelPayload = async (
|
313
|
+
request: ODataRequest,
|
314
|
+
applicationApi: PinejsClient,
|
315
|
+
) => {
|
316
|
+
await canAccess(request, applicationApi);
|
317
|
+
|
318
|
+
const { uuid } = request.values;
|
319
|
+
if (typeof uuid !== 'string') {
|
320
|
+
throw new errors.BadRequestError('Invalid uuid type');
|
321
|
+
}
|
322
|
+
|
323
|
+
const [multipartUpload] = await api.webresource.get({
|
324
|
+
resource: 'multipart_upload',
|
325
|
+
options: {
|
326
|
+
$select: ['id', 'file_key', 'upload_id'],
|
327
|
+
$filter: {
|
328
|
+
uuid,
|
329
|
+
status: 'pending',
|
330
|
+
expiry_date: { $gt: { $now: {} } },
|
331
|
+
},
|
332
|
+
},
|
333
|
+
passthrough: {
|
334
|
+
tx: request.tx,
|
335
|
+
req: permissions.rootRead,
|
336
|
+
},
|
337
|
+
});
|
338
|
+
|
339
|
+
if (multipartUpload == null) {
|
340
|
+
throw new errors.BadRequestError(`Invalid upload for uuid ${uuid}`);
|
341
|
+
}
|
342
|
+
|
343
|
+
return {
|
344
|
+
uuid,
|
345
|
+
fileKey: multipartUpload.file_key,
|
346
|
+
uploadId: multipartUpload.upload_id,
|
347
|
+
};
|
348
|
+
};
|
349
|
+
|
350
|
+
const canAccess = async (
|
351
|
+
request: ODataRequest,
|
352
|
+
applicationApi: PinejsClient,
|
353
|
+
) => {
|
354
|
+
if (request.odataQuery.key == null) {
|
355
|
+
throw new errors.BadRequestError();
|
356
|
+
}
|
357
|
+
|
358
|
+
const canAccessUrl = request.url
|
359
|
+
.slice(1)
|
360
|
+
.replace(/(beginUpload|commitUpload|cancelUpload)$/, 'canAccess');
|
361
|
+
|
362
|
+
await applicationApi.request({
|
363
|
+
method: 'POST',
|
364
|
+
url: canAccessUrl,
|
365
|
+
body: { method: 'PATCH' },
|
366
|
+
});
|
367
|
+
};
|
@@ -0,0 +1,60 @@
|
|
1
|
+
Vocabulary: webresource
|
2
|
+
|
3
|
+
Term: actor
|
4
|
+
Concept Type: Integer (Type)
|
5
|
+
Term: expiry date
|
6
|
+
Concept Type: Date Time (Type)
|
7
|
+
Term: uuid
|
8
|
+
Concept Type: Short Text (Type)
|
9
|
+
Term: resource name
|
10
|
+
Concept Type: Short Text (Type)
|
11
|
+
Term: field name
|
12
|
+
Concept Type: Short Text (Type)
|
13
|
+
Term: resource id
|
14
|
+
Concept Type: Integer (Type)
|
15
|
+
Term: upload id
|
16
|
+
Concept Type: Short Text (Type)
|
17
|
+
Term: file key
|
18
|
+
Concept Type: Short Text (Type)
|
19
|
+
Term: status
|
20
|
+
Concept Type: Short Text (Type)
|
21
|
+
Term: filename
|
22
|
+
Concept Type: Short Text (Type)
|
23
|
+
Term: content type
|
24
|
+
Concept Type: Short Text (Type)
|
25
|
+
Term: size
|
26
|
+
Concept Type: Integer (Type)
|
27
|
+
Term: chunk size
|
28
|
+
Concept Type: Integer (Type)
|
29
|
+
Term: valid until date
|
30
|
+
Concept Type: Date Time (Type)
|
31
|
+
|
32
|
+
Term: multipart upload
|
33
|
+
Fact type: multipart upload has uuid
|
34
|
+
Necessity: each multipart upload has exactly one uuid
|
35
|
+
Necessity: each uuid is of exactly one multipart upload
|
36
|
+
Fact type: multipart upload has resource name
|
37
|
+
Necessity: each multipart upload has exactly one resource name
|
38
|
+
Fact type: multipart upload has field name
|
39
|
+
Necessity: each multipart upload has exactly one field name
|
40
|
+
Fact type: multipart upload has resource id
|
41
|
+
Necessity: each multipart upload has exactly one resource id
|
42
|
+
Fact type: multipart upload has upload id
|
43
|
+
Necessity: each multipart upload has exactly one upload id
|
44
|
+
Fact type: multipart upload has file key
|
45
|
+
Necessity: each multipart upload has exactly one file key
|
46
|
+
Fact type: multipart upload has status
|
47
|
+
Necessity: each multipart upload has exactly one status
|
48
|
+
Definition: "pending" or "completed" or "cancelled"
|
49
|
+
Fact type: multipart upload has filename
|
50
|
+
Necessity: each multipart upload has exactly one filename
|
51
|
+
Fact type: multipart upload has content type
|
52
|
+
Necessity: each multipart upload has exactly one content type
|
53
|
+
Fact type: multipart upload has size
|
54
|
+
Necessity: each multipart upload has exactly one size
|
55
|
+
Fact type: multipart upload has chunk size
|
56
|
+
Necessity: each multipart upload has exactly one chunk size
|
57
|
+
Fact type: multipart upload has expiry date
|
58
|
+
Necessity: each multipart upload has exactly one expiry date
|
59
|
+
Fact type: multipart upload is created by actor
|
60
|
+
Necessity: each multipart upload is created by at most one actor
|