@toa.io/extensions.storages 1.0.0-alpha.292 → 1.0.0-alpha.293
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +30 -7
- package/readme.md +15 -0
- package/transpiled/Factory.js +2 -2
- package/transpiled/Factory.js.map +1 -1
- package/transpiled/providers/index.d.ts +7 -1
- package/transpiled/providers/index.js +20 -1
- package/transpiled/providers/index.js.map +1 -1
- package/CHANGELOG.md +0 -34
- package/source/Aspect.ts +0 -32
- package/source/Entry.ts +0 -16
- package/source/Factory.ts +0 -74
- package/source/Provider.ts +0 -40
- package/source/Scanner.ts +0 -170
- package/source/Secrets.ts +0 -6
- package/source/Storage.test.ts +0 -316
- package/source/Storage.ts +0 -129
- package/source/errors.ts +0 -3
- package/source/index.ts +0 -5
- package/source/providers/Cloudinary.ts +0 -318
- package/source/providers/Declaration.ts +0 -13
- package/source/providers/FileSystem.ts +0 -99
- package/source/providers/S3.ts +0 -174
- package/source/providers/Spaces.ts +0 -25
- package/source/providers/Temporary.ts +0 -15
- package/source/providers/Test.ts +0 -11
- package/source/providers/index.test.ts +0 -161
- package/source/providers/index.ts +0 -23
- package/source/providers/index.types.ts +0 -5
- package/source/providers/readme.md +0 -27
- package/source/test/.env.example +0 -9
- package/source/test/albert.jpg +0 -0
- package/source/test/arny.jpg +0 -0
- package/source/test/empty.txt +0 -0
- package/source/test/lenna.48x48.jpeg +0 -0
- package/source/test/lenna.ascii +0 -64
- package/source/test/lenna.png +0 -0
- package/source/test/plank.mp4 +0 -0
- package/source/test/sample.avi +0 -0
- package/source/test/sample.avif +0 -0
- package/source/test/sample.gif +0 -0
- package/source/test/sample.heic +0 -0
- package/source/test/sample.jpeg +0 -0
- package/source/test/sample.jxl +0 -0
- package/source/test/sample.svg +0 -4
- package/source/test/sample.wav +0 -0
- package/source/test/sample.webp +0 -0
- package/source/test/sport.mp4 +0 -0
- package/source/test/util.ts +0 -113
- package/tsconfig.json +0 -13
- package/tsconfig.tsbuildinfo +0 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@toa.io/extensions.storages",
|
|
3
|
-
"version": "1.0.0-alpha.
|
|
3
|
+
"version": "1.0.0-alpha.293",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Toa Storages",
|
|
6
6
|
"author": "temich <tema.gurtovoy@gmail.com>",
|
|
@@ -17,22 +17,45 @@
|
|
|
17
17
|
"publishConfig": {
|
|
18
18
|
"access": "public"
|
|
19
19
|
},
|
|
20
|
+
"files": [
|
|
21
|
+
"transpiled",
|
|
22
|
+
"readme.md",
|
|
23
|
+
"!**/tsconfig.tsbuildinfo",
|
|
24
|
+
"!**/*.test.*",
|
|
25
|
+
"!**/.env*"
|
|
26
|
+
],
|
|
20
27
|
"scripts": {
|
|
21
28
|
"test": "echo \"Error: run tests from root\" && exit 1",
|
|
22
29
|
"transpile": "npx tsc"
|
|
23
30
|
},
|
|
24
31
|
"devDependencies": {
|
|
32
|
+
"@aws-sdk/client-s3": "3.1125.0",
|
|
33
|
+
"@aws-sdk/lib-storage": "3.1125.0",
|
|
34
|
+
"cloudinary": "2.11.0",
|
|
25
35
|
"dotenv": "17.4.2"
|
|
26
36
|
},
|
|
27
|
-
"
|
|
37
|
+
"peerDependencies": {
|
|
28
38
|
"@aws-sdk/client-s3": "3.1125.0",
|
|
29
39
|
"@aws-sdk/lib-storage": "3.1125.0",
|
|
30
|
-
"
|
|
40
|
+
"cloudinary": "2.11.0"
|
|
41
|
+
},
|
|
42
|
+
"peerDependenciesMeta": {
|
|
43
|
+
"@aws-sdk/client-s3": {
|
|
44
|
+
"optional": true
|
|
45
|
+
},
|
|
46
|
+
"@aws-sdk/lib-storage": {
|
|
47
|
+
"optional": true
|
|
48
|
+
},
|
|
49
|
+
"cloudinary": {
|
|
50
|
+
"optional": true
|
|
51
|
+
}
|
|
52
|
+
},
|
|
53
|
+
"dependencies": {
|
|
54
|
+
"@toa.io/definitions": "1.0.0-alpha.293",
|
|
31
55
|
"@toa.io/generic": "1.0.0-alpha.292",
|
|
32
|
-
"@toa.io/schemas": "1.0.0-alpha.
|
|
33
|
-
"cloudinary": "2.11.0",
|
|
56
|
+
"@toa.io/schemas": "1.0.0-alpha.293",
|
|
34
57
|
"negotiator": "1.1.0",
|
|
35
|
-
"openspan": "1.0.0-alpha.
|
|
58
|
+
"openspan": "1.0.0-alpha.293"
|
|
36
59
|
},
|
|
37
|
-
"gitHead": "
|
|
60
|
+
"gitHead": "1e05b3ab64758324c58c425f1f8e52e9f35f1da8"
|
|
38
61
|
}
|
package/readme.md
CHANGED
|
@@ -2,6 +2,21 @@
|
|
|
2
2
|
|
|
3
3
|
Shared BLOB storage.
|
|
4
4
|
|
|
5
|
+
## Installing a provider
|
|
6
|
+
|
|
7
|
+
A provider's SDK is not installed with this extension: `s3` and `spaces` need
|
|
8
|
+
`@aws-sdk/client-s3` and `@aws-sdk/lib-storage`, `cloudinary` needs `cloudinary`, and `fs`, `tmp`
|
|
9
|
+
and `test` need nothing. A deploy installs what the components of each composition declare, so
|
|
10
|
+
an image carries the SDK of a provider it uses and no other.
|
|
11
|
+
|
|
12
|
+
A workspace installs the same set itself, once, and again when a storage is added:
|
|
13
|
+
|
|
14
|
+
```shell
|
|
15
|
+
$ toa npm -p application
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
Without it a composition fails to start, naming the package and version it wants.
|
|
19
|
+
|
|
5
20
|
## Entry
|
|
6
21
|
|
|
7
22
|
BLOBs are stored with the meta-information object (Entry) having the following properties:
|
package/transpiled/Factory.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import assert from 'node:assert';
|
|
2
2
|
import { console } from 'openspan';
|
|
3
|
-
import {
|
|
3
|
+
import { provider as load } from './providers/index.js';
|
|
4
4
|
import { Storage } from './Storage.js';
|
|
5
5
|
import { Aspect } from './Aspect.js';
|
|
6
6
|
import { ENV_PREFIX, validateAnnotation } from '@toa.io/definitions/extensions.storages';
|
|
@@ -25,7 +25,7 @@ export class Factory {
|
|
|
25
25
|
}
|
|
26
26
|
async createStorage(name, declaration) {
|
|
27
27
|
const { provider: id, ...options } = declaration;
|
|
28
|
-
const Provider = await
|
|
28
|
+
const Provider = await load(id);
|
|
29
29
|
const secrets = this.resolveSecrets(name, Provider);
|
|
30
30
|
const provider = new Provider(options, secrets);
|
|
31
31
|
console.debug('Storage created', {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Factory.js","sourceRoot":"","sources":["../source/Factory.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAA;AAChC,OAAO,EAAE,OAAO,EAAE,MAAM,UAAU,CAAA;AAClC,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"Factory.js","sourceRoot":"","sources":["../source/Factory.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAA;AAChC,OAAO,EAAE,OAAO,EAAE,MAAM,UAAU,CAAA;AAClC,OAAO,EAAE,QAAQ,IAAI,IAAI,EAAE,MAAM,sBAAsB,CAAA;AACvD,OAAO,EAAE,OAAO,EAAiB,MAAM,cAAc,CAAA;AACrD,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AACpC,OAAO,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,yCAAyC,CAAA;AACxF,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAM7C,MAAM,OAAO,OAAO;IACD,UAAU,CAAY;IAEvC;QACE,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;QAEvC,MAAM,CAAC,EAAE,CAAC,GAAG,KAAK,SAAS,EAAE,GAAG,UAAU,iBAAiB,CAAC,CAAA;QAE5D,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QAEjC,kBAAkB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IACrC,CAAC;IAEM,MAAM;QACX,OAAO,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAAA;IAChD,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,MAAM,QAAQ,GAAa,EAAE,CAAA;QAE7B,KAAK,MAAM,CAAC,IAAI,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC;YAC/D,wEAAwE;YACxE,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,WAA0B,CAAC,CAAA;QAE7E,OAAO,QAAQ,CAAA;IACjB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,IAAY,EAAE,WAAwB;QAChE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,GAAG,OAAO,EAAE,GAAG,WAAW,CAAA;QAChD,MAAM,QAAQ,GAAgB,MAAM,IAAI,CAAC,EAAE,CAAC,CAAA;QAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;QACnD,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QAE/C,OAAO,CAAC,KAAK,CAAC,iBAAiB,EAAE;YAC/B,IAAI;YACJ,QAAQ,EAAE,EAAE;YACZ,GAAG,CAAC,QAAQ,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC;SACvE,CAAC,CAAA;QAEF,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAA;IACtD,CAAC;IAEO,cAAc,CAAC,WAAmB,EAAE,KAAkB;QAC5D,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS;YAAE,OAAO,EAAE,CAAA;QAE1C,MAAM,OAAO,GAAuC,EAAE,CAAA;QAEtD,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,GAAG,UAAU,IAAI,WAAW,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;YAC5E,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;YAEvC,MAAM,CAAC,EAAE,CACP,MAAM,CAAC,QAAQ,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,EAC/C,IAAI,QAAQ,kBAAkB,CAC/B,CAAA;YAED,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAA;QAC9B,CAAC;QAED,OAAO,OAAO,CAAA;IAChB,CAAC;CACF"}
|
|
@@ -1,6 +1,9 @@
|
|
|
1
|
+
import type { Constructor } from '../Provider.js';
|
|
1
2
|
/**
|
|
2
3
|
* A provider is loaded when a storage names it: the SDK one is written against is not paid
|
|
3
|
-
* for by a process that declares another
|
|
4
|
+
* for by a process that declares another, and is not installed by one either — it is an
|
|
5
|
+
* optional peer of this package, which a deploy installs for the storages a component
|
|
6
|
+
* declares. See `provider` below for what a workspace that has not gets told.
|
|
4
7
|
*/
|
|
5
8
|
export declare const providers: {
|
|
6
9
|
readonly s3: () => Promise<typeof import("./S3.js").S3>;
|
|
@@ -10,6 +13,9 @@ export declare const providers: {
|
|
|
10
13
|
readonly tmp: () => Promise<typeof import("./Temporary.js").Temporary>;
|
|
11
14
|
readonly test: () => Promise<typeof import("./Test.js").Test>;
|
|
12
15
|
};
|
|
16
|
+
export type Id = keyof typeof providers;
|
|
17
|
+
/** The provider a declaration names, or what to install to have it. */
|
|
18
|
+
export declare function provider(id: Id): Promise<Constructor>;
|
|
13
19
|
export type { Declaration } from './Declaration.js';
|
|
14
20
|
export type { S3Options } from './S3.js';
|
|
15
21
|
export type { SpacesOptions } from './Spaces.js';
|
|
@@ -1,6 +1,9 @@
|
|
|
1
|
+
import { packages } from '@toa.io/definitions/extensions.storages';
|
|
1
2
|
/**
|
|
2
3
|
* A provider is loaded when a storage names it: the SDK one is written against is not paid
|
|
3
|
-
* for by a process that declares another
|
|
4
|
+
* for by a process that declares another, and is not installed by one either — it is an
|
|
5
|
+
* optional peer of this package, which a deploy installs for the storages a component
|
|
6
|
+
* declares. See `provider` below for what a workspace that has not gets told.
|
|
4
7
|
*/
|
|
5
8
|
export const providers = {
|
|
6
9
|
s3: async () => (await import('./S3.js')).S3,
|
|
@@ -10,4 +13,20 @@ export const providers = {
|
|
|
10
13
|
tmp: async () => (await import('./Temporary.js')).Temporary,
|
|
11
14
|
test: async () => (await import('./Test.js')).Test
|
|
12
15
|
};
|
|
16
|
+
/** The provider a declaration names, or what to install to have it. */
|
|
17
|
+
export async function provider(id) {
|
|
18
|
+
try {
|
|
19
|
+
return await providers[id]();
|
|
20
|
+
}
|
|
21
|
+
catch (error) {
|
|
22
|
+
const missing = Object.entries(packages[id]).map(([name, version]) => `${name}@${version}`);
|
|
23
|
+
if (missing.length === 0 || !isMissingModule(error))
|
|
24
|
+
throw error;
|
|
25
|
+
throw new Error(`Storage provider '${id}' needs ${missing.join(' ')}, which is not installed. ` +
|
|
26
|
+
`\`toa npm\` installs what this context's components declare.`, { cause: error });
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
function isMissingModule(error) {
|
|
30
|
+
return error?.code === 'ERR_MODULE_NOT_FOUND';
|
|
31
|
+
}
|
|
13
32
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../source/providers/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../source/providers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,yCAAyC,CAAA;AAGlE;;;;;GAKG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG;IACvB,EAAE,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE;IAC5C,MAAM,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM;IACxD,UAAU,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,UAAU;IACpE,EAAE,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,UAAU;IAC5D,GAAG,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,SAAS;IAC3D,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI;CACW,CAAA;AAI/D,uEAAuE;AACvE,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,EAAM;IACnC,IAAI,CAAC;QACH,OAAO,MAAM,SAAS,CAAC,EAAE,CAAC,EAAE,CAAA;IAC9B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAC9C,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,IAAI,OAAO,EAAE,CAC1C,CAAA;QAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC;YAAE,MAAM,KAAK,CAAA;QAEhE,MAAM,IAAI,KAAK,CACb,qBAAqB,EAAE,WAAW,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,4BAA4B;YAC7E,8DAA8D,EAChE,EAAE,KAAK,EAAE,KAAK,EAAE,CACjB,CAAA;IACH,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,OAAQ,KAA2B,EAAE,IAAI,KAAK,sBAAsB,CAAA;AACtE,CAAC"}
|
package/CHANGELOG.md
DELETED
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
# Change Log
|
|
2
|
-
|
|
3
|
-
All notable changes to this project will be documented in this file.
|
|
4
|
-
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
|
5
|
-
|
|
6
|
-
# [1.0.0-alpha.292](https://github.com/toa-io/toa/compare/v1.0.0-alpha.291...v1.0.0-alpha.292) (2026-09-07)
|
|
7
|
-
|
|
8
|
-
### Features
|
|
9
|
-
|
|
10
|
-
* **definitions:** what a package declares is read from one package ([7052341](https://github.com/toa-io/toa/commit/70523411b5d9e2b204c999aa02365173f3bb518d))
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
# [1.0.0-alpha.291](https://github.com/toa-io/toa/compare/v1.0.0-alpha.290...v1.0.0-alpha.291) (2026-09-07)
|
|
14
|
-
|
|
15
|
-
**Note:** Version bump only for package @toa.io/extensions.storages
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
# [1.0.0-alpha.289](https://github.com/toa-io/toa/compare/v1.0.0-alpha.288...v1.0.0-alpha.289) (2026-09-07)
|
|
22
|
-
|
|
23
|
-
* A deploy moves only what changed, and a component sees none of the runtime's environment (#1073) ([f38e3db](https://github.com/toa-io/toa/commit/f38e3db533f24db866c57bfa1ef294eff1eaf499)), closes [#1073](https://github.com/toa-io/toa/issues/1073) [#1064](https://github.com/toa-io/toa/issues/1064) [#1066](https://github.com/toa-io/toa/issues/1066) [#1067](https://github.com/toa-io/toa/issues/1067) [#1068](https://github.com/toa-io/toa/issues/1068) [#1069](https://github.com/toa-io/toa/issues/1069) [#1071](https://github.com/toa-io/toa/issues/1071) [#1070](https://github.com/toa-io/toa/issues/1070) [#1072](https://github.com/toa-io/toa/issues/1072)
|
|
24
|
-
|
|
25
|
-
### BREAKING CHANGES
|
|
26
|
-
|
|
27
|
-
* a component that read `process.env.TOA_*` reads `context` instead;
|
|
28
|
-
`echo(input)` no longer substitutes from the environment; a bash operation sees no
|
|
29
|
-
`TOA_*`; images no longer set `USER node` — see migrations/289.md.
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
# [1.0.0-alpha.287](https://github.com/toa-io/toa/compare/v1.0.0-alpha.286...v1.0.0-alpha.287) (2026-09-06)
|
|
33
|
-
|
|
34
|
-
**Note:** Version bump only for package @toa.io/extensions.storages
|
package/source/Aspect.ts
DELETED
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
import assert from 'node:assert'
|
|
2
|
-
import { Connector } from '@toa.io/core'
|
|
3
|
-
import type { extensions } from '@toa.io/core/types'
|
|
4
|
-
import { type Storage, type Storages } from './Storage.js'
|
|
5
|
-
|
|
6
|
-
export class Aspect extends Connector implements extensions.Aspect {
|
|
7
|
-
public readonly name = 'storages'
|
|
8
|
-
|
|
9
|
-
private readonly create: () => Promise<Storages>
|
|
10
|
-
private storages: Storages = {}
|
|
11
|
-
|
|
12
|
-
/** The storages are made as the aspect connects: their providers load then, not before. */
|
|
13
|
-
public constructor(create: () => Promise<Storages>) {
|
|
14
|
-
super()
|
|
15
|
-
|
|
16
|
-
this.create = create
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
protected override async open(): Promise<void> {
|
|
20
|
-
this.storages = await this.create()
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
public invoke(name: string, method: keyof Storage, ...args: unknown[]): unknown {
|
|
24
|
-
const storage = this.storages[name]
|
|
25
|
-
|
|
26
|
-
assert.ok(storage !== undefined, `Storage '${name}' is not defined`)
|
|
27
|
-
|
|
28
|
-
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
29
|
-
// @ts-expect-error
|
|
30
|
-
return storage[method](...args)
|
|
31
|
-
}
|
|
32
|
-
}
|
package/source/Entry.ts
DELETED
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
import type { Readable } from 'node:stream'
|
|
2
|
-
|
|
3
|
-
export type Entry = { id: string } & Metadata
|
|
4
|
-
export type Stream = { stream: Readable } & Metadata
|
|
5
|
-
|
|
6
|
-
export interface Metadata {
|
|
7
|
-
type: string
|
|
8
|
-
size: number | null
|
|
9
|
-
checksum: string
|
|
10
|
-
created: string
|
|
11
|
-
attributes: Attributes
|
|
12
|
-
range?: string
|
|
13
|
-
partial?: boolean
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
export type Attributes = Record<string, string>
|
package/source/Factory.ts
DELETED
|
@@ -1,74 +0,0 @@
|
|
|
1
|
-
import assert from 'node:assert'
|
|
2
|
-
import { console } from 'openspan'
|
|
3
|
-
import { providers } from './providers/index.js'
|
|
4
|
-
import { Storage, type Storages } from './Storage.js'
|
|
5
|
-
import { Aspect } from './Aspect.js'
|
|
6
|
-
import { ENV_PREFIX, validateAnnotation } from '@toa.io/definitions/extensions.storages'
|
|
7
|
-
import { environment } from '@toa.io/generic'
|
|
8
|
-
import type { Constructor } from './Provider.js'
|
|
9
|
-
import type { Declaration } from './providers/index.js'
|
|
10
|
-
import type { Secrets } from './Secrets.js'
|
|
11
|
-
import type { Annotation } from '@toa.io/definitions/extensions.storages'
|
|
12
|
-
|
|
13
|
-
export class Factory {
|
|
14
|
-
private readonly annotation: Annotation
|
|
15
|
-
|
|
16
|
-
public constructor() {
|
|
17
|
-
const env = environment.get(ENV_PREFIX)
|
|
18
|
-
|
|
19
|
-
assert.ok(env !== undefined, `${ENV_PREFIX} is not defined`)
|
|
20
|
-
|
|
21
|
-
this.annotation = JSON.parse(env)
|
|
22
|
-
|
|
23
|
-
validateAnnotation(this.annotation)
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
public aspect(): Aspect {
|
|
27
|
-
return new Aspect(() => this.createStorages())
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
private async createStorages(): Promise<Storages> {
|
|
31
|
-
const storages: Storages = {}
|
|
32
|
-
|
|
33
|
-
for (const [name, declaration] of Object.entries(this.annotation))
|
|
34
|
-
// the annotation is validated above, so a declaration is its provider's
|
|
35
|
-
storages[name] = await this.createStorage(name, declaration as Declaration)
|
|
36
|
-
|
|
37
|
-
return storages
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
private async createStorage(name: string, declaration: Declaration): Promise<Storage> {
|
|
41
|
-
const { provider: id, ...options } = declaration
|
|
42
|
-
const Provider: Constructor = await providers[id]()
|
|
43
|
-
const secrets = this.resolveSecrets(name, Provider)
|
|
44
|
-
const provider = new Provider(options, secrets)
|
|
45
|
-
|
|
46
|
-
console.debug('Storage created', {
|
|
47
|
-
name,
|
|
48
|
-
provider: id,
|
|
49
|
-
...(provider.root === undefined ? undefined : { root: provider.root })
|
|
50
|
-
})
|
|
51
|
-
|
|
52
|
-
return new Storage(provider, { name, provider: id })
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
private resolveSecrets(storageName: string, Class: Constructor): Secrets {
|
|
56
|
-
if (Class.SECRETS === undefined) return {}
|
|
57
|
-
|
|
58
|
-
const secrets: Record<string, string | undefined> = {}
|
|
59
|
-
|
|
60
|
-
for (const secret of Class.SECRETS) {
|
|
61
|
-
const variable = `${ENV_PREFIX}_${storageName}_${secret.name}`.toUpperCase()
|
|
62
|
-
const value = environment.get(variable)
|
|
63
|
-
|
|
64
|
-
assert.ok(
|
|
65
|
-
secret.optional === true || value !== undefined,
|
|
66
|
-
`'${variable}' is not defined`
|
|
67
|
-
)
|
|
68
|
-
|
|
69
|
-
secrets[secret.name] = value
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
return secrets
|
|
73
|
-
}
|
|
74
|
-
}
|
package/source/Provider.ts
DELETED
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
import * as assert from 'node:assert'
|
|
2
|
-
import type { Metadata, Stream } from './Entry.js'
|
|
3
|
-
import type { Readable } from 'node:stream'
|
|
4
|
-
import type { Maybe } from '@toa.io/core/types'
|
|
5
|
-
import type { Secret, Secrets } from './Secrets.js'
|
|
6
|
-
|
|
7
|
-
export abstract class Provider<Options = unknown> {
|
|
8
|
-
public static readonly SECRETS?: readonly Secret[]
|
|
9
|
-
public readonly root?: string
|
|
10
|
-
public readonly options: Options
|
|
11
|
-
|
|
12
|
-
protected constructor(options: Options, secrets?: Secrets) {
|
|
13
|
-
this.options = options
|
|
14
|
-
|
|
15
|
-
new.target.SECRETS?.forEach(({ name, optional }) =>
|
|
16
|
-
assert.ok(
|
|
17
|
-
optional === true || secrets?.[name] !== undefined,
|
|
18
|
-
`Missing secret '${name}'`
|
|
19
|
-
)
|
|
20
|
-
)
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
public abstract get(path: string, options?: unknown): Promise<Maybe<Stream>>
|
|
24
|
-
|
|
25
|
-
public abstract head(path: string): Promise<Maybe<Metadata>>
|
|
26
|
-
|
|
27
|
-
public abstract put(path: string, stream: Readable): Promise<void>
|
|
28
|
-
|
|
29
|
-
public abstract commit(path: string, metadata: Metadata): Promise<void>
|
|
30
|
-
|
|
31
|
-
public abstract delete(path: string): Promise<void>
|
|
32
|
-
|
|
33
|
-
public abstract move(from: string, to: string): Promise<Maybe<void>>
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
export interface Constructor<Options = any> {
|
|
37
|
-
SECRETS?: readonly Secret[]
|
|
38
|
-
|
|
39
|
-
new (options: Options, secrets?: Secrets): Provider<Options>
|
|
40
|
-
}
|
package/source/Scanner.ts
DELETED
|
@@ -1,170 +0,0 @@
|
|
|
1
|
-
import { PassThrough, type TransformCallback } from 'node:stream'
|
|
2
|
-
import { createHash } from 'node:crypto'
|
|
3
|
-
import Negotiator from 'negotiator'
|
|
4
|
-
|
|
5
|
-
export class Scanner extends PassThrough {
|
|
6
|
-
public size = 0
|
|
7
|
-
public type = 'application/octet-stream'
|
|
8
|
-
public error?: Error
|
|
9
|
-
|
|
10
|
-
private readonly hash = createHash('md5')
|
|
11
|
-
private readonly claim?: string
|
|
12
|
-
private readonly accept?: string
|
|
13
|
-
private readonly limit?: number
|
|
14
|
-
private position = 0
|
|
15
|
-
private completed = false
|
|
16
|
-
private readonly chunks: Buffer[] = []
|
|
17
|
-
|
|
18
|
-
public constructor(control?: ScanOptions) {
|
|
19
|
-
super()
|
|
20
|
-
|
|
21
|
-
this.claim = control?.claim
|
|
22
|
-
this.accept = control?.accept
|
|
23
|
-
this.limit = control?.limit
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
public digest(): string {
|
|
27
|
-
return this.hash.digest('hex')
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
public override _transform(
|
|
31
|
-
buffer: Buffer,
|
|
32
|
-
encoding: BufferEncoding,
|
|
33
|
-
callback: TransformCallback
|
|
34
|
-
): void {
|
|
35
|
-
super._transform(buffer, encoding, callback)
|
|
36
|
-
|
|
37
|
-
this.process(buffer)
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
private readonly process = (buffer: Buffer): void => {
|
|
41
|
-
this.size += buffer.length
|
|
42
|
-
this.hash.update(buffer)
|
|
43
|
-
|
|
44
|
-
if (this.completed) return
|
|
45
|
-
|
|
46
|
-
if (this.position + buffer.length > HEADER_SIZE)
|
|
47
|
-
buffer = buffer.subarray(0, HEADER_SIZE - this.position)
|
|
48
|
-
|
|
49
|
-
this.chunks.push(buffer)
|
|
50
|
-
this.position += buffer.length
|
|
51
|
-
|
|
52
|
-
if (this.position === HEADER_SIZE) this.complete()
|
|
53
|
-
|
|
54
|
-
if (this.limit !== undefined && this.size > this.limit)
|
|
55
|
-
this.interrupt(ERR_LIMIT_EXCEEDED)
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
private complete(): void {
|
|
59
|
-
const header = Buffer.concat(this.chunks).toString('hex')
|
|
60
|
-
|
|
61
|
-
const signature = SIGNATURES.find(({ off, hex, expression }) => {
|
|
62
|
-
const sig = header.slice(off, off + hex.length)
|
|
63
|
-
|
|
64
|
-
return expression === undefined ? sig === hex : expression.test(sig)
|
|
65
|
-
})
|
|
66
|
-
|
|
67
|
-
const type = signature?.type ?? this.claim
|
|
68
|
-
|
|
69
|
-
if (type !== undefined) {
|
|
70
|
-
this.match(type)
|
|
71
|
-
this.type = type
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
this.verify(signature)
|
|
75
|
-
this.completed = true
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
private verify(signature: Signature | undefined): void {
|
|
79
|
-
if (this.claim === undefined || this.claim === 'application/octet-stream') return
|
|
80
|
-
|
|
81
|
-
const mismatch =
|
|
82
|
-
signature === undefined
|
|
83
|
-
? KNOWN_TYPES.has(this.claim)
|
|
84
|
-
: this.claim !== signature.type
|
|
85
|
-
|
|
86
|
-
if (mismatch) this.interrupt(ERR_TYPE_MISMATCH)
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
private match(type: string): void {
|
|
90
|
-
if (this.accept === undefined) return
|
|
91
|
-
|
|
92
|
-
const unacceptable = this.negotiate(this.accept, [type]) === null
|
|
93
|
-
|
|
94
|
-
if (unacceptable) this.interrupt(ERR_NOT_ACCEPTABLE)
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
private negotiate(accept: string, type: string[]): string | null {
|
|
98
|
-
return new Negotiator({ headers: { accept } }).mediaType(type) ?? null
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
private interrupt(error: Error): void {
|
|
102
|
-
this.completed = true
|
|
103
|
-
this.error = error
|
|
104
|
-
this.destroy(error)
|
|
105
|
-
}
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
// https://en.wikipedia.org/wiki/List_of_file_signatures
|
|
109
|
-
const SIGNATURES: Signature[] = [
|
|
110
|
-
{ hex: 'ff d8 ff e0', off: 0, type: 'image/jpeg' },
|
|
111
|
-
{ hex: 'ff d8 ff e1', off: 0, type: 'image/jpeg' },
|
|
112
|
-
{ hex: 'ff d8 ff e2', off: 0, type: 'image/jpeg' },
|
|
113
|
-
{ hex: 'ff d8 ff ee', off: 0, type: 'image/jpeg' },
|
|
114
|
-
{ hex: 'ff d8 ff db', off: 0, type: 'image/jpeg' },
|
|
115
|
-
{ hex: '89 50 4e 47', off: 0, type: 'image/png' },
|
|
116
|
-
{ hex: '47 49 46 38', off: 0, type: 'image/gif' },
|
|
117
|
-
{ hex: '52 49 46 46 ?? ?? ?? ?? 57 45 42 50', off: 0, type: 'image/webp' },
|
|
118
|
-
{ hex: '4a 58 4c 20 0d 0a 87 0a', off: 8, type: 'image/jxl' },
|
|
119
|
-
{ hex: '66 74 79 70 68 65 69 63', off: 8, type: 'image/heic' },
|
|
120
|
-
{ hex: '66 74 79 70 61 76 69 66', off: 8, type: 'image/avif' },
|
|
121
|
-
{ hex: '52 49 46 46 ?? ?? ?? ?? 41 56 49 20', off: 0, type: 'video/avi' },
|
|
122
|
-
{ hex: '52 49 46 46 ?? ?? ?? ?? 57 41 56 45', off: 0, type: 'audio/wav' }
|
|
123
|
-
/*
|
|
124
|
-
When adding a new signature, include a copyright-free sample file in the `.tests` directory
|
|
125
|
-
and update the `signatures` test group in `Storage.test.ts`.
|
|
126
|
-
*/
|
|
127
|
-
].map((signature: Signature) => {
|
|
128
|
-
signature.hex = signature.hex.replaceAll(' ', '')
|
|
129
|
-
|
|
130
|
-
if (signature.hex.includes('??')) {
|
|
131
|
-
const expression = signature.hex.replaceAll(
|
|
132
|
-
/(?<wildcards>\?{1,24})/g,
|
|
133
|
-
(_, wildcards) => `[0-9a-f]{${wildcards.length}}`
|
|
134
|
-
)
|
|
135
|
-
|
|
136
|
-
signature.expression = new RegExp(expression, 'i')
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
return signature
|
|
140
|
-
})
|
|
141
|
-
|
|
142
|
-
const HEADER_SIZE =
|
|
143
|
-
SIGNATURES.reduce((max, { off, hex }) => Math.max(max, off + hex.length), 0) / 2
|
|
144
|
-
|
|
145
|
-
const KNOWN_TYPES = new Set(SIGNATURES.map(({ type }) => type))
|
|
146
|
-
|
|
147
|
-
const ERR_TYPE_MISMATCH = new (class TypeMismatchError extends Error {
|
|
148
|
-
public readonly code = 'TYPE_MISMATCH'
|
|
149
|
-
})()
|
|
150
|
-
|
|
151
|
-
const ERR_NOT_ACCEPTABLE = new (class NotAcceptableError extends Error {
|
|
152
|
-
public readonly code = 'NOT_ACCEPTABLE'
|
|
153
|
-
})()
|
|
154
|
-
|
|
155
|
-
const ERR_LIMIT_EXCEEDED = new (class LimitExceededError extends Error {
|
|
156
|
-
public readonly code = 'LIMIT_EXCEEDED'
|
|
157
|
-
})()
|
|
158
|
-
|
|
159
|
-
export interface ScanOptions {
|
|
160
|
-
claim?: string
|
|
161
|
-
accept?: string
|
|
162
|
-
limit?: number
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
interface Signature {
|
|
166
|
-
type: string
|
|
167
|
-
off: number
|
|
168
|
-
hex: string
|
|
169
|
-
expression?: RegExp
|
|
170
|
-
}
|