@tstdl/base 0.93.51 → 0.93.53
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/browser/utils.js +2 -2
- package/object-storage/google/google.object-storage-provider.d.ts +44 -0
- package/object-storage/google/google.object-storage-provider.js +81 -0
- package/object-storage/google/google.object-storage.d.ts +32 -0
- package/object-storage/google/google.object-storage.js +220 -0
- package/object-storage/google/google.object.d.ts +16 -0
- package/object-storage/google/google.object.js +69 -0
- package/object-storage/google/index.d.ts +3 -0
- package/object-storage/google/index.js +3 -0
- package/object-storage/object-storage.d.ts +7 -4
- package/object-storage/s3/s3.object-storage-provider.js +1 -1
- package/object-storage/s3/s3.object-storage.d.ts +3 -3
- package/object-storage/s3/s3.object-storage.js +21 -11
- package/object-storage/s3/s3.object.d.ts +1 -1
- package/object-storage/s3/s3.object.js +1 -1
- package/orm/sqls.js +3 -3
- package/package.json +2 -2
- package/schema/converters/zod-converter.d.ts +8 -2
- package/schema/converters/zod-converter.js +182 -136
- package/schema/schemas/bigint.js +2 -2
- package/schema/schemas/boolean.d.ts +1 -0
- package/schema/schemas/boolean.js +16 -16
- package/schema/schemas/constraint.d.ts +19 -0
- package/schema/schemas/constraint.js +36 -0
- package/schema/schemas/index.d.ts +1 -0
- package/schema/schemas/index.js +1 -0
- package/schema/schemas/object.d.ts +3 -1
- package/schema/schemas/object.js +54 -28
- package/schema/schemas/simple.d.ts +6 -1
- package/schema/schemas/simple.js +11 -2
- package/schema/schemas/string.d.ts +4 -2
- package/schema/schemas/string.js +10 -5
- package/schema/schemas/transform.d.ts +4 -0
- package/schema/schemas/transform.js +4 -0
- package/schema/schemas/uint8-array.d.ts +1 -0
- package/schema/schemas/uint8-array.js +10 -7
- package/templates/template.model.js +2 -2
- package/test6.js +112 -29
- package/types/types.d.ts +1 -1
- package/utils/base64.js +1 -2
- package/utils/encoding.js +2 -5
- package/utils/format-error.js +1 -1
- package/utils/object/object.js +7 -1
- package/utils/z-base32.js +2 -4
package/browser/utils.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { objectKeys } from '../utils/object/object.js';
|
|
1
|
+
import { objectEntries, objectKeys } from '../utils/object/object.js';
|
|
2
2
|
import { timeout } from '../utils/timing.js';
|
|
3
3
|
import { assert, isDefined, isNull, isString, isUndefined } from '../utils/type-guards.js';
|
|
4
4
|
import { resolveValueOrProvider } from '../utils/value-or-provider.js';
|
|
@@ -49,7 +49,7 @@ export function attachLogger(loggable, logger) {
|
|
|
49
49
|
const location = consoleMessage.location();
|
|
50
50
|
const rawLocationText = (location.lineNumber == 0) ? location.url : `${location.url}:${location.lineNumber}:${location.columnNumber}`;
|
|
51
51
|
const locationText = rawLocationText.length == 0 ? undefined : rawLocationText;
|
|
52
|
-
const additions =
|
|
52
|
+
const additions = objectEntries({ location: locationText }).filter(([_, value]) => isDefined(value)).map(([key, value]) => `${key}: ${value}`).join(', ');
|
|
53
53
|
const message = `${url}: ${consoleMessage.text()}${(additions.length > 0) ? ` (${additions})` : ''}`;
|
|
54
54
|
switch (consoleMessage.type()) {
|
|
55
55
|
case 'debug':
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { ObjectStorageProvider } from '../../object-storage/index.js';
|
|
2
|
+
import { GoogleObjectStorage } from './google.object-storage.js';
|
|
3
|
+
export declare class GoogleObjectStorageProviderConfig {
|
|
4
|
+
/**
|
|
5
|
+
* Google Cloud Project ID
|
|
6
|
+
*/
|
|
7
|
+
projectId?: string;
|
|
8
|
+
/**
|
|
9
|
+
* Path to the JSON key file.
|
|
10
|
+
* Mutually exclusive with credentials.
|
|
11
|
+
*/
|
|
12
|
+
keyFilename?: string;
|
|
13
|
+
/**
|
|
14
|
+
* Object containing client_email and private_key.
|
|
15
|
+
* Mutually exclusive with keyFilename.
|
|
16
|
+
*/
|
|
17
|
+
credentials?: {
|
|
18
|
+
client_email: string;
|
|
19
|
+
private_key: string;
|
|
20
|
+
};
|
|
21
|
+
/**
|
|
22
|
+
* Use a single bucket for all modules (which will become transparent key prefixes)
|
|
23
|
+
* mutually exclusive with bucketPerModule
|
|
24
|
+
*/
|
|
25
|
+
bucket?: string;
|
|
26
|
+
/**
|
|
27
|
+
* Use an own bucket for every module
|
|
28
|
+
* mutually exclusive with bucket
|
|
29
|
+
*/
|
|
30
|
+
bucketPerModule?: boolean;
|
|
31
|
+
}
|
|
32
|
+
export declare const bucketPerModule: unique symbol;
|
|
33
|
+
export declare class GoogleObjectStorageProvider extends ObjectStorageProvider<GoogleObjectStorage> {
|
|
34
|
+
private readonly client;
|
|
35
|
+
private readonly bucket;
|
|
36
|
+
constructor(config: GoogleObjectStorageProviderConfig);
|
|
37
|
+
get(module: string): GoogleObjectStorage;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Configure google object storage provider
|
|
41
|
+
* @param config google storage config
|
|
42
|
+
* @param register whether to register for {@link ObjectStorage} and {@link ObjectStorageProvider}
|
|
43
|
+
*/
|
|
44
|
+
export declare function configureGoogleObjectStorage(config: GoogleObjectStorageProviderConfig, register?: boolean): void;
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
2
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
3
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
4
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
5
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
6
|
+
};
|
|
7
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
8
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
9
|
+
};
|
|
10
|
+
import { Storage } from '@google-cloud/storage';
|
|
11
|
+
import { Singleton } from '../../injector/decorators.js';
|
|
12
|
+
import { Injector } from '../../injector/injector.js';
|
|
13
|
+
import { ObjectStorage, ObjectStorageProvider } from '../../object-storage/index.js';
|
|
14
|
+
import { assertDefinedPass, assertStringPass, isDefined } from '../../utils/type-guards.js';
|
|
15
|
+
import { GoogleObjectStorage } from './google.object-storage.js';
|
|
16
|
+
export class GoogleObjectStorageProviderConfig {
|
|
17
|
+
/**
|
|
18
|
+
* Google Cloud Project ID
|
|
19
|
+
*/
|
|
20
|
+
projectId;
|
|
21
|
+
/**
|
|
22
|
+
* Path to the JSON key file.
|
|
23
|
+
* Mutually exclusive with credentials.
|
|
24
|
+
*/
|
|
25
|
+
keyFilename;
|
|
26
|
+
/**
|
|
27
|
+
* Object containing client_email and private_key.
|
|
28
|
+
* Mutually exclusive with keyFilename.
|
|
29
|
+
*/
|
|
30
|
+
credentials;
|
|
31
|
+
/**
|
|
32
|
+
* Use a single bucket for all modules (which will become transparent key prefixes)
|
|
33
|
+
* mutually exclusive with bucketPerModule
|
|
34
|
+
*/
|
|
35
|
+
bucket;
|
|
36
|
+
/**
|
|
37
|
+
* Use an own bucket for every module
|
|
38
|
+
* mutually exclusive with bucket
|
|
39
|
+
*/
|
|
40
|
+
bucketPerModule;
|
|
41
|
+
}
|
|
42
|
+
export const bucketPerModule = Symbol('bucket per module');
|
|
43
|
+
let GoogleObjectStorageProvider = class GoogleObjectStorageProvider extends ObjectStorageProvider {
|
|
44
|
+
client;
|
|
45
|
+
bucket;
|
|
46
|
+
constructor(config) {
|
|
47
|
+
super();
|
|
48
|
+
if (isDefined(config.bucket) && (config.bucketPerModule == true)) {
|
|
49
|
+
throw new Error('bucket and bucketPerModule is mutually exclusive');
|
|
50
|
+
}
|
|
51
|
+
const storageOptions = {
|
|
52
|
+
projectId: config.projectId,
|
|
53
|
+
keyFilename: config.keyFilename,
|
|
54
|
+
credentials: config.credentials,
|
|
55
|
+
};
|
|
56
|
+
this.client = new Storage(storageOptions);
|
|
57
|
+
this.bucket = assertDefinedPass((config.bucketPerModule == true) ? true : config.bucket, 'either bucket or bucketPerModule must be specified');
|
|
58
|
+
}
|
|
59
|
+
get(module) {
|
|
60
|
+
const bucket = (this.bucket == true) ? module : assertStringPass(this.bucket);
|
|
61
|
+
const prefix = (this.bucket == true) ? '' : ((module == '') ? '' : `${module}/`);
|
|
62
|
+
return new GoogleObjectStorage(this.client, bucket, module, prefix);
|
|
63
|
+
}
|
|
64
|
+
};
|
|
65
|
+
GoogleObjectStorageProvider = __decorate([
|
|
66
|
+
Singleton(),
|
|
67
|
+
__metadata("design:paramtypes", [GoogleObjectStorageProviderConfig])
|
|
68
|
+
], GoogleObjectStorageProvider);
|
|
69
|
+
export { GoogleObjectStorageProvider };
|
|
70
|
+
/**
|
|
71
|
+
* Configure google object storage provider
|
|
72
|
+
* @param config google storage config
|
|
73
|
+
* @param register whether to register for {@link ObjectStorage} and {@link ObjectStorageProvider}
|
|
74
|
+
*/
|
|
75
|
+
export function configureGoogleObjectStorage(config, register = true) {
|
|
76
|
+
Injector.register(GoogleObjectStorageProviderConfig, { useValue: config });
|
|
77
|
+
if (register) {
|
|
78
|
+
Injector.registerSingleton(ObjectStorageProvider, { useToken: GoogleObjectStorageProvider });
|
|
79
|
+
Injector.registerSingleton(ObjectStorage, { useToken: GoogleObjectStorage });
|
|
80
|
+
}
|
|
81
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { Storage, type Bucket, type File } from '@google-cloud/storage';
|
|
2
|
+
import { ObjectStorage, type CopyObjectOptions, type MoveObjectOptions, type ObjectStorageConfiguration, type UploadObjectOptions, type UploadUrlOptions } from '../../object-storage/index.js';
|
|
3
|
+
import { GoogleObject } from './google.object.js';
|
|
4
|
+
export declare class GoogleObjectStorage extends ObjectStorage {
|
|
5
|
+
readonly client: Storage;
|
|
6
|
+
readonly bucket: Bucket;
|
|
7
|
+
readonly prefix: string;
|
|
8
|
+
constructor(client: Storage, bucket: string, module: string, keyPrefix: string);
|
|
9
|
+
ensureBucketExists(location?: string, options?: {
|
|
10
|
+
objectLocking?: boolean;
|
|
11
|
+
}): Promise<void>;
|
|
12
|
+
configureBucket(configuration: ObjectStorageConfiguration): Promise<void>;
|
|
13
|
+
getFile(key: string): File;
|
|
14
|
+
exists(key: string): Promise<boolean>;
|
|
15
|
+
uploadObject(key: string, content: Uint8Array | ReadableStream<Uint8Array>, options?: UploadObjectOptions): Promise<GoogleObject>;
|
|
16
|
+
copyObject(source: GoogleObject | string, destination: GoogleObject | string | [ObjectStorage, string], options?: CopyObjectOptions): Promise<GoogleObject>;
|
|
17
|
+
moveObject(source: GoogleObject | string, destination: GoogleObject | string | [ObjectStorage, string], options?: MoveObjectOptions): Promise<GoogleObject>;
|
|
18
|
+
getContent(key: string): Promise<Uint8Array<ArrayBuffer>>;
|
|
19
|
+
getContentStream(key: string): ReadableStream<Uint8Array<ArrayBuffer>>;
|
|
20
|
+
getObjects(): Promise<GoogleObject[]>;
|
|
21
|
+
getObjectsCursor(): AsyncIterable<GoogleObject>;
|
|
22
|
+
getObject(key: string): Promise<GoogleObject>;
|
|
23
|
+
getResourceUri(key: string): Promise<string>;
|
|
24
|
+
getDownloadUrl(key: string, expirationTimestamp: number, responseHeaders?: Record<string, string>): Promise<string>;
|
|
25
|
+
getUploadUrl(key: string, expirationTimestamp: number, options?: UploadUrlOptions): Promise<string>;
|
|
26
|
+
deleteObject(key: string): Promise<void>;
|
|
27
|
+
deleteObjects(keys: string[]): Promise<void>;
|
|
28
|
+
private _getObject;
|
|
29
|
+
getBucketKey(key: string): string;
|
|
30
|
+
getResourceUriSync(key: string): string;
|
|
31
|
+
private getKey;
|
|
32
|
+
}
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
2
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
3
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
4
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
5
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
6
|
+
};
|
|
7
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
8
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
9
|
+
};
|
|
10
|
+
var GoogleObjectStorage_1;
|
|
11
|
+
import { Readable } from 'node:stream';
|
|
12
|
+
import { pipeline } from 'node:stream/promises';
|
|
13
|
+
import { Storage } from '@google-cloud/storage';
|
|
14
|
+
import { match, P } from 'ts-pattern';
|
|
15
|
+
import { AsyncEnumerable } from '../../enumerable/async-enumerable.js';
|
|
16
|
+
import { Singleton } from '../../injector/decorators.js';
|
|
17
|
+
import { registerAfterResolve } from '../../injector/resolution.js';
|
|
18
|
+
import { ObjectStorage } from '../../object-storage/index.js';
|
|
19
|
+
import { toArrayAsync } from '../../utils/async-iterable-helpers/to-array.js';
|
|
20
|
+
import { mapObjectKeys, objectKeys } from '../../utils/object/object.js';
|
|
21
|
+
import { _throw } from '../../utils/throw.js';
|
|
22
|
+
import { assertDefinedPass, isDefined, isNotNull, isString, isUint8Array } from '../../utils/type-guards.js';
|
|
23
|
+
import { secondsPerDay } from '../../utils/units.js';
|
|
24
|
+
import { GoogleObjectStorageProvider } from './google.object-storage-provider.js';
|
|
25
|
+
import { GoogleObject } from './google.object.js';
|
|
26
|
+
let GoogleObjectStorage = GoogleObjectStorage_1 = class GoogleObjectStorage extends ObjectStorage {
|
|
27
|
+
client;
|
|
28
|
+
bucket;
|
|
29
|
+
prefix;
|
|
30
|
+
constructor(client, bucket, module, keyPrefix) {
|
|
31
|
+
super(module);
|
|
32
|
+
this.client = client;
|
|
33
|
+
this.bucket = client.bucket(bucket);
|
|
34
|
+
this.prefix = keyPrefix;
|
|
35
|
+
registerAfterResolve(this, async (argument) => {
|
|
36
|
+
const configuration = (isString(argument) ? undefined : argument.configuration) ?? {};
|
|
37
|
+
await this.ensureBucketExists();
|
|
38
|
+
await this.configureBucket(configuration);
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
async ensureBucketExists(location, options) {
|
|
42
|
+
const [exists] = await this.bucket.exists();
|
|
43
|
+
if (exists) {
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
if (isDefined(options?.objectLocking) && options.objectLocking) {
|
|
47
|
+
throw new Error('Object locking not yet supported for GoogleObjectStorage.');
|
|
48
|
+
}
|
|
49
|
+
await this.client.createBucket(this.bucket.name, { location });
|
|
50
|
+
}
|
|
51
|
+
async configureBucket(configuration) {
|
|
52
|
+
const [metadata] = await this.bucket.getMetadata();
|
|
53
|
+
// Check for existing rules. GCS doesn't strictly use string IDs like AWS/Minio for rules,
|
|
54
|
+
// so we identify our "TstdlExpireObjects" rule by its specific condition and action.
|
|
55
|
+
const currentLifecycleRules = metadata.lifecycle?.rule ?? [];
|
|
56
|
+
const targetExpirationDays = configuration.lifecycle?.expiration?.after;
|
|
57
|
+
const targetExpiration = isDefined(targetExpirationDays) ? Math.ceil(targetExpirationDays / secondsPerDay) : undefined;
|
|
58
|
+
// Identify if our specific rule exists
|
|
59
|
+
const tstdlRuleIndex = currentLifecycleRules.findIndex((rule) => (rule.action.type == 'Delete') && isDefined(rule.condition.age) && (objectKeys(rule.condition).length == 1));
|
|
60
|
+
const tstdlRule = (tstdlRuleIndex >= 0) ? currentLifecycleRules[tstdlRuleIndex] : undefined;
|
|
61
|
+
const currentExpiration = tstdlRule?.condition.age;
|
|
62
|
+
if (currentExpiration == targetExpiration) {
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
65
|
+
// Filter out the rule we manage (if it exists) to preserve others
|
|
66
|
+
const otherRules = currentLifecycleRules.filter((_, index) => index != tstdlRuleIndex);
|
|
67
|
+
const newRules = [
|
|
68
|
+
...otherRules,
|
|
69
|
+
isDefined(targetExpiration) ? { action: { type: 'Delete' }, condition: { age: targetExpiration } } : null,
|
|
70
|
+
].filter(isNotNull);
|
|
71
|
+
// Update the bucket lifecycle configuration
|
|
72
|
+
// null removes all rules
|
|
73
|
+
await this.bucket.setMetadata({ lifecycle: (newRules.length > 0) ? { rule: newRules } : null });
|
|
74
|
+
}
|
|
75
|
+
getFile(key) {
|
|
76
|
+
const bucketKey = this.getBucketKey(key);
|
|
77
|
+
return this.bucket.file(bucketKey);
|
|
78
|
+
}
|
|
79
|
+
async exists(key) {
|
|
80
|
+
const object = await this.getObject(key);
|
|
81
|
+
const [exists] = await object.file.exists();
|
|
82
|
+
return exists;
|
|
83
|
+
}
|
|
84
|
+
async uploadObject(key, content, options) {
|
|
85
|
+
const object = await this.getObject(key);
|
|
86
|
+
const saveOptions = { contentType: options?.contentType, metadata: { contentLength: options?.contentLength, ...options?.metadata } };
|
|
87
|
+
if (isUint8Array(content)) {
|
|
88
|
+
await object.file.save(content, saveOptions);
|
|
89
|
+
}
|
|
90
|
+
else {
|
|
91
|
+
const readable = Readable.fromWeb(content);
|
|
92
|
+
const writeStream = object.file.createWriteStream(saveOptions);
|
|
93
|
+
await pipeline(readable, writeStream);
|
|
94
|
+
}
|
|
95
|
+
return object;
|
|
96
|
+
}
|
|
97
|
+
async copyObject(source, destination, options) {
|
|
98
|
+
const sourceObject = await match(source)
|
|
99
|
+
.with(P.instanceOf(GoogleObject), (obj) => obj)
|
|
100
|
+
.with(P.string, async (key) => await this.getObject(key))
|
|
101
|
+
.exhaustive();
|
|
102
|
+
const destinationObject = await match(destination)
|
|
103
|
+
.with(P.instanceOf(GoogleObject), (obj) => obj)
|
|
104
|
+
.with(P.string, async (key) => await this.getObject(key))
|
|
105
|
+
.with([P.instanceOf(GoogleObjectStorage_1), P.string], async ([storage, key]) => await storage.getObject(key))
|
|
106
|
+
.otherwise(() => _throw(new Error('Destination must be a GoogleObject, string key, or [GoogleObjectStorage, string] tuple.')));
|
|
107
|
+
const metadata = await match(options?.metadata)
|
|
108
|
+
.with(P.nullish, () => undefined)
|
|
109
|
+
.with(P.any, async (newMetadata) => {
|
|
110
|
+
const sourceMetadata = await sourceObject.getMetadata();
|
|
111
|
+
return { ...sourceMetadata, ...newMetadata };
|
|
112
|
+
})
|
|
113
|
+
.exhaustive();
|
|
114
|
+
await sourceObject.file.copy(destinationObject.file, { metadata });
|
|
115
|
+
return destinationObject;
|
|
116
|
+
}
|
|
117
|
+
async moveObject(source, destination, options) {
|
|
118
|
+
const sourceObject = await match(source)
|
|
119
|
+
.with(P.instanceOf(GoogleObject), (obj) => obj)
|
|
120
|
+
.with(P.string, async (key) => await this.getObject(key))
|
|
121
|
+
.exhaustive();
|
|
122
|
+
const destinationObject = await match(destination)
|
|
123
|
+
.with(P.instanceOf(GoogleObject), (obj) => obj)
|
|
124
|
+
.with(P.string, async (key) => await this.getObject(key))
|
|
125
|
+
.with([P.instanceOf(GoogleObjectStorage_1), P.string], async ([storage, key]) => await storage.getObject(key))
|
|
126
|
+
.otherwise(() => _throw(new Error('Destination must be a GoogleObject, string key, or [GoogleObjectStorage, string] tuple.')));
|
|
127
|
+
if (isDefined(options?.metadata)) {
|
|
128
|
+
throw new Error('Metadata update on move is not supported in GoogleObjectStorage.');
|
|
129
|
+
}
|
|
130
|
+
if (sourceObject.file.bucket.name == destinationObject.file.bucket.name) {
|
|
131
|
+
await sourceObject.file.moveFileAtomic(destinationObject.file);
|
|
132
|
+
}
|
|
133
|
+
else {
|
|
134
|
+
await sourceObject.file.move(destinationObject.file);
|
|
135
|
+
}
|
|
136
|
+
return destinationObject;
|
|
137
|
+
}
|
|
138
|
+
async getContent(key) {
|
|
139
|
+
const object = this._getObject(key);
|
|
140
|
+
return await object.getContent();
|
|
141
|
+
}
|
|
142
|
+
getContentStream(key) {
|
|
143
|
+
const object = this._getObject(key);
|
|
144
|
+
return object.getContentStream();
|
|
145
|
+
}
|
|
146
|
+
async getObjects() {
|
|
147
|
+
return await toArrayAsync(this.getObjectsCursor());
|
|
148
|
+
}
|
|
149
|
+
async *getObjectsCursor() {
|
|
150
|
+
const nodeStream = this.bucket.getFilesStream({ prefix: this.prefix, autoPaginate: true });
|
|
151
|
+
const stream = Readable.toWeb(nodeStream);
|
|
152
|
+
for await (const file of stream) {
|
|
153
|
+
yield new GoogleObject(this.module, this.getKey(file.name), this, file);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
// eslint-disable-next-line @typescript-eslint/require-await
|
|
157
|
+
async getObject(key) {
|
|
158
|
+
return this._getObject(key);
|
|
159
|
+
}
|
|
160
|
+
async getResourceUri(key) {
|
|
161
|
+
return this.getResourceUriSync(key);
|
|
162
|
+
}
|
|
163
|
+
async getDownloadUrl(key, expirationTimestamp, responseHeaders) {
|
|
164
|
+
const object = await this.getObject(key);
|
|
165
|
+
const [url] = await object.file.getSignedUrl({
|
|
166
|
+
action: 'read',
|
|
167
|
+
expires: new Date(expirationTimestamp),
|
|
168
|
+
extensionHeaders: responseHeaders,
|
|
169
|
+
});
|
|
170
|
+
return url;
|
|
171
|
+
}
|
|
172
|
+
async getUploadUrl(key, expirationTimestamp, options) {
|
|
173
|
+
const bucketKey = this.getBucketKey(key);
|
|
174
|
+
const [url] = await this.bucket.file(bucketKey).getSignedUrl({
|
|
175
|
+
action: 'write',
|
|
176
|
+
expires: new Date(expirationTimestamp),
|
|
177
|
+
contentType: options?.contentType,
|
|
178
|
+
extensionHeaders: options?.metadata ? convertMetadataToHeaders(options.metadata) : undefined,
|
|
179
|
+
});
|
|
180
|
+
return url;
|
|
181
|
+
}
|
|
182
|
+
async deleteObject(key) {
|
|
183
|
+
const bucketKey = this.getBucketKey(key);
|
|
184
|
+
await this.bucket.file(bucketKey).delete({ ignoreNotFound: true });
|
|
185
|
+
}
|
|
186
|
+
async deleteObjects(keys) {
|
|
187
|
+
await AsyncEnumerable.from(keys)
|
|
188
|
+
.parallelForEach(25, async (key) => await this.deleteObject(key));
|
|
189
|
+
}
|
|
190
|
+
_getObject(key) {
|
|
191
|
+
const resourceUri = this.getResourceUriSync(key);
|
|
192
|
+
return new GoogleObject(this.module, key, this, resourceUri);
|
|
193
|
+
}
|
|
194
|
+
getBucketKey(key) {
|
|
195
|
+
return `${this.prefix}${key}`;
|
|
196
|
+
}
|
|
197
|
+
getResourceUriSync(key) {
|
|
198
|
+
const bucketKey = this.getBucketKey(key);
|
|
199
|
+
return `gs://${this.bucket.name}/${bucketKey}`;
|
|
200
|
+
}
|
|
201
|
+
getKey(bucketKey) {
|
|
202
|
+
return bucketKey.slice(this.prefix.length);
|
|
203
|
+
}
|
|
204
|
+
};
|
|
205
|
+
GoogleObjectStorage = GoogleObjectStorage_1 = __decorate([
|
|
206
|
+
Singleton({
|
|
207
|
+
provider: {
|
|
208
|
+
useFactory: (argument, context) => {
|
|
209
|
+
const { module } = (isString(argument) ? { module: argument } : assertDefinedPass(argument, 'argument must be a string or an object'));
|
|
210
|
+
return context.resolve(GoogleObjectStorageProvider).get(module);
|
|
211
|
+
},
|
|
212
|
+
},
|
|
213
|
+
argumentIdentityProvider: JSON.stringify,
|
|
214
|
+
}),
|
|
215
|
+
__metadata("design:paramtypes", [Storage, String, String, String])
|
|
216
|
+
], GoogleObjectStorage);
|
|
217
|
+
export { GoogleObjectStorage };
|
|
218
|
+
function convertMetadataToHeaders(metadata) {
|
|
219
|
+
return mapObjectKeys(metadata, (key) => `x-goog-meta-${key}`);
|
|
220
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { File } from '@google-cloud/storage';
|
|
2
|
+
import { ObjectStorageObject, type ObjectMetadata } from '../../object-storage/object.js';
|
|
3
|
+
import type { GoogleObjectStorage } from './google.object-storage.js';
|
|
4
|
+
export declare class GoogleObject extends ObjectStorageObject {
|
|
5
|
+
#private;
|
|
6
|
+
get file(): File;
|
|
7
|
+
constructor(module: string, key: string, objectStorage: GoogleObjectStorage, fileOrResourceUri: File | string);
|
|
8
|
+
getResourceUri(): Promise<string>;
|
|
9
|
+
getContentLength(): Promise<number>;
|
|
10
|
+
getMetadata(): Promise<ObjectMetadata>;
|
|
11
|
+
getContent(): Promise<Uint8Array<ArrayBuffer>>;
|
|
12
|
+
getContentStream(): ReadableStream<Uint8Array<ArrayBuffer>>;
|
|
13
|
+
private getFileMetadata;
|
|
14
|
+
private _getFileMetadata;
|
|
15
|
+
private _getContentLength;
|
|
16
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { Readable } from 'node:stream';
|
|
2
|
+
import { ObjectStorageObject } from '../../object-storage/object.js';
|
|
3
|
+
import { isDefined, isString, isUndefined } from '../../utils/type-guards.js';
|
|
4
|
+
export class GoogleObject extends ObjectStorageObject {
|
|
5
|
+
#objectStorage;
|
|
6
|
+
#file;
|
|
7
|
+
#contentLength;
|
|
8
|
+
#fileMetadata;
|
|
9
|
+
#resourceUri;
|
|
10
|
+
get file() {
|
|
11
|
+
if (isUndefined(this.#file)) {
|
|
12
|
+
this.#file = this.#objectStorage.getFile(this.key);
|
|
13
|
+
}
|
|
14
|
+
return this.#file;
|
|
15
|
+
}
|
|
16
|
+
constructor(module, key, objectStorage, fileOrResourceUri) {
|
|
17
|
+
super(module, key);
|
|
18
|
+
this.#objectStorage = objectStorage;
|
|
19
|
+
if (isString(fileOrResourceUri)) {
|
|
20
|
+
this.#resourceUri = fileOrResourceUri;
|
|
21
|
+
}
|
|
22
|
+
else {
|
|
23
|
+
this.#file = fileOrResourceUri;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
// eslint-disable-next-line @typescript-eslint/require-await
|
|
27
|
+
async getResourceUri() {
|
|
28
|
+
if (isUndefined(this.#resourceUri)) {
|
|
29
|
+
this.#resourceUri = this.#file?.cloudStorageURI.href ?? this.#objectStorage.getResourceUriSync(this.key);
|
|
30
|
+
}
|
|
31
|
+
return this.#resourceUri;
|
|
32
|
+
}
|
|
33
|
+
async getContentLength() {
|
|
34
|
+
if (isDefined(this.#contentLength)) {
|
|
35
|
+
return await this.#contentLength;
|
|
36
|
+
}
|
|
37
|
+
this.#contentLength = this._getContentLength(); // without await to avoid double fetching in case of concurrent calls
|
|
38
|
+
this.#contentLength = await this.#contentLength;
|
|
39
|
+
return this.#contentLength;
|
|
40
|
+
}
|
|
41
|
+
async getMetadata() {
|
|
42
|
+
const fileMetadata = await this.getFileMetadata();
|
|
43
|
+
return fileMetadata.metadata ?? {};
|
|
44
|
+
}
|
|
45
|
+
async getContent() {
|
|
46
|
+
const [buffer] = await this.file.download();
|
|
47
|
+
return buffer;
|
|
48
|
+
}
|
|
49
|
+
getContentStream() {
|
|
50
|
+
const stream = this.file.createReadStream();
|
|
51
|
+
return Readable.toWeb(stream);
|
|
52
|
+
}
|
|
53
|
+
async getFileMetadata() {
|
|
54
|
+
if (isDefined(this.#fileMetadata)) {
|
|
55
|
+
return await this.#fileMetadata;
|
|
56
|
+
}
|
|
57
|
+
this.#fileMetadata = this._getFileMetadata(); // without await to avoid double fetching in case of concurrent calls
|
|
58
|
+
this.#fileMetadata = await this.#fileMetadata;
|
|
59
|
+
return this.#fileMetadata;
|
|
60
|
+
}
|
|
61
|
+
async _getFileMetadata() {
|
|
62
|
+
const [metadata] = await this.file.getMetadata();
|
|
63
|
+
return metadata;
|
|
64
|
+
}
|
|
65
|
+
async _getContentLength() {
|
|
66
|
+
const fileMetadata = await this.getFileMetadata();
|
|
67
|
+
return Number(fileMetadata.size);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
@@ -43,22 +43,25 @@ export declare abstract class ObjectStorage implements Resolvable<ObjectStorageA
|
|
|
43
43
|
* @param key object key
|
|
44
44
|
* @param content content of object
|
|
45
45
|
* @param options options
|
|
46
|
+
* @return uploaded object
|
|
46
47
|
*/
|
|
47
|
-
abstract uploadObject(key: string, content: Uint8Array | ReadableStream<Uint8Array>, options?: UploadObjectOptions): Promise<
|
|
48
|
+
abstract uploadObject(key: string, content: Uint8Array | ReadableStream<Uint8Array>, options?: UploadObjectOptions): Promise<ObjectStorageObject>;
|
|
48
49
|
/**
|
|
49
50
|
* Copies an object
|
|
50
51
|
* @param sourceKey source object key
|
|
51
|
-
* @param
|
|
52
|
+
* @param destination destination object, key or destination storage and key
|
|
52
53
|
* @param options options
|
|
54
|
+
* @return copied object
|
|
53
55
|
*/
|
|
54
|
-
abstract copyObject(sourceKey: string,
|
|
56
|
+
abstract copyObject(sourceKey: ObjectStorageObject | string, destination: ObjectStorageObject | string | [ObjectStorage, string], options?: CopyObjectOptions): Promise<ObjectStorageObject>;
|
|
55
57
|
/**
|
|
56
58
|
* Moves an object
|
|
57
59
|
* @param sourceKey source object key
|
|
58
60
|
* @param destinationKey destination object key or destination storage and key
|
|
59
61
|
* @param options options
|
|
62
|
+
* @return moved object
|
|
60
63
|
*/
|
|
61
|
-
abstract moveObject(sourceKey: string, destinationKey: string | [ObjectStorage, string], options?: MoveObjectOptions): Promise<
|
|
64
|
+
abstract moveObject(sourceKey: ObjectStorageObject | string, destinationKey: ObjectStorageObject | string | [ObjectStorage, string], options?: MoveObjectOptions): Promise<ObjectStorageObject>;
|
|
62
65
|
/**
|
|
63
66
|
* Get an url which can be used to upload the object without further authorization
|
|
64
67
|
* @param key object key
|
|
@@ -7,11 +7,11 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
|
|
|
7
7
|
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
8
8
|
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
9
9
|
};
|
|
10
|
+
import { Client } from 'minio';
|
|
10
11
|
import { Singleton } from '../../injector/decorators.js';
|
|
11
12
|
import { Injector } from '../../injector/injector.js';
|
|
12
13
|
import { ObjectStorage, ObjectStorageProvider } from '../../object-storage/index.js';
|
|
13
14
|
import { assertDefinedPass, assertStringPass, isDefined } from '../../utils/type-guards.js';
|
|
14
|
-
import { Client } from 'minio';
|
|
15
15
|
import { S3ObjectStorage } from './s3.object-storage.js';
|
|
16
16
|
export class S3ObjectStorageProviderConfig {
|
|
17
17
|
/**
|
|
@@ -12,9 +12,9 @@ export declare class S3ObjectStorage extends ObjectStorage {
|
|
|
12
12
|
configureBucket(configuration: ObjectStorageConfiguration): Promise<void>;
|
|
13
13
|
exists(key: string): Promise<boolean>;
|
|
14
14
|
statObject(key: string): Promise<BucketItemStat>;
|
|
15
|
-
uploadObject(key: string, content: Uint8Array | ReadableStream<Uint8Array>, options?: UploadObjectOptions): Promise<
|
|
16
|
-
copyObject(source: string, destination: string | [ObjectStorage, string], options?: CopyObjectOptions): Promise<
|
|
17
|
-
moveObject(
|
|
15
|
+
uploadObject(key: string, content: Uint8Array | ReadableStream<Uint8Array>, options?: UploadObjectOptions): Promise<S3Object>;
|
|
16
|
+
copyObject(source: S3Object | string, destination: S3Object | string | [ObjectStorage, string], options?: CopyObjectOptions): Promise<S3Object>;
|
|
17
|
+
moveObject(source: S3Object | string, destination: S3Object | string | [ObjectStorage, string], options?: MoveObjectOptions): Promise<S3Object>;
|
|
18
18
|
getContent(key: string): Promise<Uint8Array<ArrayBuffer>>;
|
|
19
19
|
getContentStream(key: string): ReadableStream<Uint8Array<ArrayBuffer>>;
|
|
20
20
|
getObjects(): Promise<S3Object[]>;
|
|
@@ -10,6 +10,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
|
|
|
10
10
|
var S3ObjectStorage_1;
|
|
11
11
|
import { Readable } from 'node:stream';
|
|
12
12
|
import { CopyDestinationOptions, CopySourceOptions } from 'minio';
|
|
13
|
+
import { match, P } from 'ts-pattern';
|
|
13
14
|
import { Singleton } from '../../injector/decorators.js';
|
|
14
15
|
import { registerAfterResolve } from '../../injector/resolution.js';
|
|
15
16
|
import { ObjectStorage } from '../../object-storage/index.js';
|
|
@@ -20,7 +21,8 @@ import { now } from '../../utils/date-time.js';
|
|
|
20
21
|
import { mapObjectKeys } from '../../utils/object/object.js';
|
|
21
22
|
import { readableStreamFromPromise } from '../../utils/stream/index.js';
|
|
22
23
|
import { readBinaryStream } from '../../utils/stream/stream-reader.js';
|
|
23
|
-
import {
|
|
24
|
+
import { _throw } from '../../utils/throw.js';
|
|
25
|
+
import { assertDefinedPass, isDefined, isObject, isString, isUint8Array, isUndefined } from '../../utils/type-guards.js';
|
|
24
26
|
import { secondsPerDay } from '../../utils/units.js';
|
|
25
27
|
import { S3ObjectStorageProvider } from './s3.object-storage-provider.js';
|
|
26
28
|
import { S3Object } from './s3.object.js';
|
|
@@ -119,24 +121,32 @@ let S3ObjectStorage = S3ObjectStorage_1 = class S3ObjectStorage extends ObjectSt
|
|
|
119
121
|
errorPromise,
|
|
120
122
|
]);
|
|
121
123
|
}
|
|
124
|
+
return await this.getObject(key);
|
|
122
125
|
}
|
|
123
126
|
async copyObject(source, destination, options) {
|
|
124
|
-
const
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
const
|
|
127
|
+
const sourceObject = await match(source)
|
|
128
|
+
.with(P.instanceOf(S3Object), (obj) => obj)
|
|
129
|
+
.with(P.string, async (key) => await this.getObject(key))
|
|
130
|
+
.exhaustive();
|
|
131
|
+
const destinationObject = await match(destination)
|
|
132
|
+
.with(P.instanceOf(S3Object), (obj) => obj)
|
|
133
|
+
.with(P.string, async (key) => await this.getObject(key))
|
|
134
|
+
.with([P.instanceOf(S3ObjectStorage_1), P.string], async ([storage, key]) => await storage.getObject(key))
|
|
135
|
+
.otherwise(() => _throw(new Error('Destination must be a S3Object, string key, or [S3ObjectStorage, string] tuple.')));
|
|
129
136
|
const sourceMetadata = await sourceObject.getMetadata();
|
|
130
|
-
await this.client.copyObject(new CopySourceOptions({ Bucket: this.bucket, Object:
|
|
131
|
-
Bucket:
|
|
132
|
-
Object:
|
|
137
|
+
await this.client.copyObject(new CopySourceOptions({ Bucket: this.bucket, Object: sourceObject.storage.getBucketKey(sourceObject.key) }), new CopyDestinationOptions({
|
|
138
|
+
Bucket: destinationObject.storage.bucket,
|
|
139
|
+
Object: destinationObject.storage.getBucketKey(destinationObject.key),
|
|
133
140
|
MetadataDirective: 'REPLACE',
|
|
134
141
|
UserMetadata: { ...sourceMetadata, ...options?.metadata },
|
|
135
142
|
}));
|
|
143
|
+
return destinationObject;
|
|
136
144
|
}
|
|
137
|
-
async moveObject(
|
|
138
|
-
await this.copyObject(
|
|
145
|
+
async moveObject(source, destination, options) {
|
|
146
|
+
const object = await this.copyObject(source, destination, options);
|
|
147
|
+
const sourceKey = isString(source) ? source : source.key;
|
|
139
148
|
await this.deleteObject(sourceKey);
|
|
149
|
+
return object;
|
|
140
150
|
}
|
|
141
151
|
async getContent(key) {
|
|
142
152
|
const bucketKey = this.getBucketKey(key);
|
|
@@ -2,10 +2,10 @@ import type { ObjectMetadata } from '../../object-storage/object.js';
|
|
|
2
2
|
import { ObjectStorageObject } from '../../object-storage/object.js';
|
|
3
3
|
import type { S3ObjectStorage } from './s3.object-storage.js';
|
|
4
4
|
export declare class S3Object extends ObjectStorageObject {
|
|
5
|
-
private readonly storage;
|
|
6
5
|
private readonly resourceUri;
|
|
7
6
|
private contentLength;
|
|
8
7
|
private statPromise;
|
|
8
|
+
readonly storage: S3ObjectStorage;
|
|
9
9
|
constructor(module: string, key: string, resourceUri: string, contentLength: number | undefined, storage: S3ObjectStorage);
|
|
10
10
|
getResourceUri(): Promise<string>;
|
|
11
11
|
getContentLength(): Promise<number>;
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { ObjectStorageObject } from '../../object-storage/object.js';
|
|
2
2
|
import { isUndefined } from '../../utils/type-guards.js';
|
|
3
3
|
export class S3Object extends ObjectStorageObject {
|
|
4
|
-
storage;
|
|
5
4
|
resourceUri;
|
|
6
5
|
contentLength;
|
|
7
6
|
statPromise;
|
|
7
|
+
storage;
|
|
8
8
|
constructor(module, key, resourceUri, contentLength, storage) {
|
|
9
9
|
super(module, key);
|
|
10
10
|
this.resourceUri = resourceUri;
|