@n8n/blob-storage 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +88 -0
- package/README.md +8 -0
- package/dist/azure-blob/azure-blob.config.d.ts +8 -0
- package/dist/azure-blob/azure-blob.config.js +52 -0
- package/dist/azure-blob/azure-blob.config.js.map +1 -0
- package/dist/azure-blob/azure-blob.service.ee.d.ts +25 -0
- package/dist/azure-blob/azure-blob.service.ee.js +157 -0
- package/dist/azure-blob/azure-blob.service.ee.js.map +1 -0
- package/dist/azure-blob/index.d.ts +2 -0
- package/dist/azure-blob/index.js +8 -0
- package/dist/azure-blob/index.js.map +1 -0
- package/dist/build.tsbuildinfo +1 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.js +10 -0
- package/dist/index.js.map +1 -0
- package/dist/object-store/index.d.ts +3 -0
- package/dist/object-store/index.js +8 -0
- package/dist/object-store/index.js.map +1 -0
- package/dist/object-store/object-store.config.d.ts +21 -0
- package/dist/object-store/object-store.config.js +93 -0
- package/dist/object-store/object-store.config.js.map +1 -0
- package/dist/object-store/object-store.service.ee.d.ts +49 -0
- package/dist/object-store/object-store.service.ee.js +290 -0
- package/dist/object-store/object-store.service.ee.js.map +1 -0
- package/dist/object-store/types.d.ts +8 -0
- package/dist/object-store/types.js +3 -0
- package/dist/object-store/types.js.map +1 -0
- package/dist/stream-utils.d.ts +2 -0
- package/dist/stream-utils.js +54 -0
- package/dist/stream-utils.js.map +1 -0
- package/dist/types.d.ts +6 -0
- package/dist/types.js +3 -0
- package/dist/types.js.map +1 -0
- package/package.json +67 -0
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export type { BlobMetadata, PreWriteBlobMetadata } from './types';
|
|
2
|
+
export { createFixedSizeChunker } from './stream-utils';
|
|
3
|
+
export { ObjectStoreConfig } from './object-store/object-store.config';
|
|
4
|
+
export type { MetadataResponseHeaders } from './object-store/types';
|
|
5
|
+
export { AzureBlobConfig } from './azure-blob/azure-blob.config';
|
|
6
|
+
export type { ObjectStoreService } from './object-store/object-store.service.ee';
|
|
7
|
+
export type { AzureBlobService } from './azure-blob/azure-blob.service.ee';
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.AzureBlobConfig = exports.ObjectStoreConfig = exports.createFixedSizeChunker = void 0;
|
|
4
|
+
var stream_utils_1 = require("./stream-utils");
|
|
5
|
+
Object.defineProperty(exports, "createFixedSizeChunker", { enumerable: true, get: function () { return stream_utils_1.createFixedSizeChunker; } });
|
|
6
|
+
var object_store_config_1 = require("./object-store/object-store.config");
|
|
7
|
+
Object.defineProperty(exports, "ObjectStoreConfig", { enumerable: true, get: function () { return object_store_config_1.ObjectStoreConfig; } });
|
|
8
|
+
var azure_blob_config_1 = require("./azure-blob/azure-blob.config");
|
|
9
|
+
Object.defineProperty(exports, "AzureBlobConfig", { enumerable: true, get: function () { return azure_blob_config_1.AzureBlobConfig; } });
|
|
10
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AACA,+CAAwD;AAA/C,sHAAA,sBAAsB,OAAA;AAC/B,0EAAuE;AAA9D,wHAAA,iBAAiB,OAAA;AAE1B,oEAAiE;AAAxD,oHAAA,eAAe,OAAA"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ObjectStoreConfig = exports.ObjectStoreService = void 0;
|
|
4
|
+
var object_store_service_ee_1 = require("./object-store.service.ee");
|
|
5
|
+
Object.defineProperty(exports, "ObjectStoreService", { enumerable: true, get: function () { return object_store_service_ee_1.ObjectStoreService; } });
|
|
6
|
+
var object_store_config_1 = require("./object-store.config");
|
|
7
|
+
Object.defineProperty(exports, "ObjectStoreConfig", { enumerable: true, get: function () { return object_store_config_1.ObjectStoreConfig; } });
|
|
8
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/object-store/index.ts"],"names":[],"mappings":";;;AAAA,qEAA+D;AAAtD,6HAAA,kBAAkB,OAAA;AAC3B,6DAA0D;AAAjD,wHAAA,iBAAiB,OAAA"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
declare const protocolSchema: z.ZodEnum<["http", "https"]>;
|
|
3
|
+
export type Protocol = z.infer<typeof protocolSchema>;
|
|
4
|
+
declare class ObjectStoreBucketConfig {
|
|
5
|
+
name: string;
|
|
6
|
+
region: string;
|
|
7
|
+
}
|
|
8
|
+
declare class ObjectStoreCredentialsConfig {
|
|
9
|
+
accessKey: string;
|
|
10
|
+
accessSecret: string;
|
|
11
|
+
authAutoDetect: boolean;
|
|
12
|
+
}
|
|
13
|
+
export declare class ObjectStoreConfig {
|
|
14
|
+
host: string;
|
|
15
|
+
protocol: Protocol;
|
|
16
|
+
forcePathStyle: boolean;
|
|
17
|
+
maxAttempts: number;
|
|
18
|
+
bucket: ObjectStoreBucketConfig;
|
|
19
|
+
credentials: ObjectStoreCredentialsConfig;
|
|
20
|
+
}
|
|
21
|
+
export {};
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.ObjectStoreConfig = void 0;
|
|
13
|
+
const config_1 = require("@n8n/config");
|
|
14
|
+
const zod_1 = require("zod");
|
|
15
|
+
const protocolSchema = zod_1.z.enum(['http', 'https']);
|
|
16
|
+
let ObjectStoreBucketConfig = class ObjectStoreBucketConfig {
|
|
17
|
+
constructor() {
|
|
18
|
+
this.name = '';
|
|
19
|
+
this.region = '';
|
|
20
|
+
}
|
|
21
|
+
};
|
|
22
|
+
__decorate([
|
|
23
|
+
(0, config_1.Env)('N8N_EXTERNAL_STORAGE_S3_BUCKET_NAME'),
|
|
24
|
+
__metadata("design:type", String)
|
|
25
|
+
], ObjectStoreBucketConfig.prototype, "name", void 0);
|
|
26
|
+
__decorate([
|
|
27
|
+
(0, config_1.Env)('N8N_EXTERNAL_STORAGE_S3_BUCKET_REGION'),
|
|
28
|
+
__metadata("design:type", String)
|
|
29
|
+
], ObjectStoreBucketConfig.prototype, "region", void 0);
|
|
30
|
+
ObjectStoreBucketConfig = __decorate([
|
|
31
|
+
config_1.Config
|
|
32
|
+
], ObjectStoreBucketConfig);
|
|
33
|
+
let ObjectStoreCredentialsConfig = class ObjectStoreCredentialsConfig {
|
|
34
|
+
constructor() {
|
|
35
|
+
this.accessKey = '';
|
|
36
|
+
this.accessSecret = '';
|
|
37
|
+
this.authAutoDetect = false;
|
|
38
|
+
}
|
|
39
|
+
};
|
|
40
|
+
__decorate([
|
|
41
|
+
(0, config_1.Env)('N8N_EXTERNAL_STORAGE_S3_ACCESS_KEY'),
|
|
42
|
+
__metadata("design:type", String)
|
|
43
|
+
], ObjectStoreCredentialsConfig.prototype, "accessKey", void 0);
|
|
44
|
+
__decorate([
|
|
45
|
+
(0, config_1.Env)('N8N_EXTERNAL_STORAGE_S3_ACCESS_SECRET'),
|
|
46
|
+
__metadata("design:type", String)
|
|
47
|
+
], ObjectStoreCredentialsConfig.prototype, "accessSecret", void 0);
|
|
48
|
+
__decorate([
|
|
49
|
+
(0, config_1.Env)('N8N_EXTERNAL_STORAGE_S3_AUTH_AUTO_DETECT'),
|
|
50
|
+
__metadata("design:type", Boolean)
|
|
51
|
+
], ObjectStoreCredentialsConfig.prototype, "authAutoDetect", void 0);
|
|
52
|
+
ObjectStoreCredentialsConfig = __decorate([
|
|
53
|
+
config_1.Config
|
|
54
|
+
], ObjectStoreCredentialsConfig);
|
|
55
|
+
let ObjectStoreConfig = class ObjectStoreConfig {
|
|
56
|
+
constructor() {
|
|
57
|
+
this.host = '';
|
|
58
|
+
this.protocol = 'https';
|
|
59
|
+
this.forcePathStyle = true;
|
|
60
|
+
this.maxAttempts = 3;
|
|
61
|
+
this.bucket = {};
|
|
62
|
+
this.credentials = {};
|
|
63
|
+
}
|
|
64
|
+
};
|
|
65
|
+
exports.ObjectStoreConfig = ObjectStoreConfig;
|
|
66
|
+
__decorate([
|
|
67
|
+
(0, config_1.Env)('N8N_EXTERNAL_STORAGE_S3_HOST'),
|
|
68
|
+
__metadata("design:type", String)
|
|
69
|
+
], ObjectStoreConfig.prototype, "host", void 0);
|
|
70
|
+
__decorate([
|
|
71
|
+
(0, config_1.Env)('N8N_EXTERNAL_STORAGE_S3_PROTOCOL', protocolSchema),
|
|
72
|
+
__metadata("design:type", String)
|
|
73
|
+
], ObjectStoreConfig.prototype, "protocol", void 0);
|
|
74
|
+
__decorate([
|
|
75
|
+
(0, config_1.Env)('N8N_EXTERNAL_STORAGE_S3_FORCE_PATH_STYLE'),
|
|
76
|
+
__metadata("design:type", Boolean)
|
|
77
|
+
], ObjectStoreConfig.prototype, "forcePathStyle", void 0);
|
|
78
|
+
__decorate([
|
|
79
|
+
(0, config_1.Env)('N8N_EXTERNAL_STORAGE_S3_MAX_ATTEMPTS'),
|
|
80
|
+
__metadata("design:type", Number)
|
|
81
|
+
], ObjectStoreConfig.prototype, "maxAttempts", void 0);
|
|
82
|
+
__decorate([
|
|
83
|
+
config_1.Nested,
|
|
84
|
+
__metadata("design:type", ObjectStoreBucketConfig)
|
|
85
|
+
], ObjectStoreConfig.prototype, "bucket", void 0);
|
|
86
|
+
__decorate([
|
|
87
|
+
config_1.Nested,
|
|
88
|
+
__metadata("design:type", ObjectStoreCredentialsConfig)
|
|
89
|
+
], ObjectStoreConfig.prototype, "credentials", void 0);
|
|
90
|
+
exports.ObjectStoreConfig = ObjectStoreConfig = __decorate([
|
|
91
|
+
config_1.Config
|
|
92
|
+
], ObjectStoreConfig);
|
|
93
|
+
//# sourceMappingURL=object-store.config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"object-store.config.js","sourceRoot":"","sources":["../../src/object-store/object-store.config.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,wCAAkD;AAClD,6BAAwB;AAExB,MAAM,cAAc,GAAG,OAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;AAKjD,IAAM,uBAAuB,GAA7B,MAAM,uBAAuB;IAA7B;QAGC,SAAI,GAAW,EAAE,CAAC;QAIlB,WAAM,GAAW,EAAE,CAAC;IACrB,CAAC;CAAA,CAAA;AALA;IADC,IAAA,YAAG,EAAC,qCAAqC,CAAC;;qDACzB;AAIlB;IADC,IAAA,YAAG,EAAC,uCAAuC,CAAC;;uDACzB;AAPf,uBAAuB;IAD5B,eAAM;GACD,uBAAuB,CAQ5B;AAGD,IAAM,4BAA4B,GAAlC,MAAM,4BAA4B;IAAlC;QAGC,cAAS,GAAW,EAAE,CAAC;QAIvB,iBAAY,GAAW,EAAE,CAAC;QAQ1B,mBAAc,GAAY,KAAK,CAAC;IACjC,CAAC;CAAA,CAAA;AAbA;IADC,IAAA,YAAG,EAAC,oCAAoC,CAAC;;+DACnB;AAIvB;IADC,IAAA,YAAG,EAAC,uCAAuC,CAAC;;kEACnB;AAQ1B;IADC,IAAA,YAAG,EAAC,0CAA0C,CAAC;;oEAChB;AAf3B,4BAA4B;IADjC,eAAM;GACD,4BAA4B,CAgBjC;AAGM,IAAM,iBAAiB,GAAvB,MAAM,iBAAiB;IAAvB;QAMN,SAAI,GAAW,EAAE,CAAC;QAGlB,aAAQ,GAAa,OAAO,CAAC;QAI7B,mBAAc,GAAY,IAAI,CAAC;QAI/B,gBAAW,GAAW,CAAC,CAAC;QAGxB,WAAM,GAA4B,EAA6B,CAAC;QAGhE,gBAAW,GAAiC,EAAkC,CAAC;IAChF,CAAC;CAAA,CAAA;;AAlBA;IADC,IAAA,YAAG,EAAC,8BAA8B,CAAC;;+CAClB;AAGlB;IADC,IAAA,YAAG,EAAC,kCAAkC,EAAE,cAAc,CAAC;;mDAC3B;AAI7B;IADC,IAAA,YAAG,EAAC,0CAA0C,CAAC;;yDACjB;AAI/B;IADC,IAAA,YAAG,EAAC,sCAAsC,CAAC;;sDACpB;AAGxB;IADC,eAAM;8BACC,uBAAuB;iDAAiC;AAGhE;IADC,eAAM;8BACM,4BAA4B;sDAAsC;4BAvBnE,iBAAiB;IAD7B,eAAM;GACM,iBAAiB,CAwB7B"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { S3ClientConfig } from '@aws-sdk/client-s3';
|
|
2
|
+
import { Logger } from '@n8n/backend-common';
|
|
3
|
+
import { Readable } from 'node:stream';
|
|
4
|
+
import { ObjectStoreConfig } from './object-store.config';
|
|
5
|
+
import type { MetadataResponseHeaders } from './types';
|
|
6
|
+
import type { PreWriteBlobMetadata } from '../types';
|
|
7
|
+
export declare class ObjectStoreService {
|
|
8
|
+
private readonly logger;
|
|
9
|
+
private readonly s3Config;
|
|
10
|
+
private s3Client;
|
|
11
|
+
private isReady;
|
|
12
|
+
private bucket;
|
|
13
|
+
constructor(logger: Logger, s3Config: ObjectStoreConfig);
|
|
14
|
+
getClientConfig(): S3ClientConfig;
|
|
15
|
+
init(): Promise<void>;
|
|
16
|
+
setReady(newState: boolean): void;
|
|
17
|
+
checkConnection(): Promise<void>;
|
|
18
|
+
put(filename: string, buffer: Buffer, metadata?: PreWriteBlobMetadata): Promise<import("@aws-sdk/client-s3").PutObjectCommandOutput>;
|
|
19
|
+
get(fileId: string, opts: {
|
|
20
|
+
mode: 'buffer';
|
|
21
|
+
}): Promise<Buffer>;
|
|
22
|
+
get(fileId: string, opts: {
|
|
23
|
+
mode: 'stream';
|
|
24
|
+
chunkSize?: number;
|
|
25
|
+
}): Promise<Readable>;
|
|
26
|
+
getMetadata(fileId: string): Promise<MetadataResponseHeaders>;
|
|
27
|
+
deleteOne(fileId: string): Promise<import("@aws-sdk/client-s3").DeleteObjectCommandOutput>;
|
|
28
|
+
deleteMany(prefix: string): Promise<void>;
|
|
29
|
+
deleteByKeys(keys: string[]): Promise<void>;
|
|
30
|
+
list(prefix: string): Promise<{
|
|
31
|
+
key: string;
|
|
32
|
+
lastModified: string;
|
|
33
|
+
eTag: string;
|
|
34
|
+
size: number;
|
|
35
|
+
storageClass: string;
|
|
36
|
+
}[]>;
|
|
37
|
+
getListPage(prefix: string, continuationToken?: string): Promise<{
|
|
38
|
+
contents: {
|
|
39
|
+
key: string;
|
|
40
|
+
lastModified: string;
|
|
41
|
+
eTag: string;
|
|
42
|
+
size: number;
|
|
43
|
+
storageClass: string;
|
|
44
|
+
}[];
|
|
45
|
+
isTruncated: boolean;
|
|
46
|
+
nextContinuationToken: string | undefined;
|
|
47
|
+
}>;
|
|
48
|
+
private handleS3Error;
|
|
49
|
+
}
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
+
};
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
exports.ObjectStoreService = void 0;
|
|
16
|
+
const client_s3_1 = require("@aws-sdk/client-s3");
|
|
17
|
+
const backend_common_1 = require("@n8n/backend-common");
|
|
18
|
+
const backend_network_1 = require("@n8n/backend-network");
|
|
19
|
+
const di_1 = require("@n8n/di");
|
|
20
|
+
const ensure_error_1 = require("@n8n/utils/errors/ensure-error");
|
|
21
|
+
const chunk_1 = __importDefault(require("lodash/chunk"));
|
|
22
|
+
const n8n_workflow_1 = require("n8n-workflow");
|
|
23
|
+
const node_crypto_1 = require("node:crypto");
|
|
24
|
+
const node_stream_1 = require("node:stream");
|
|
25
|
+
const object_store_config_1 = require("./object-store.config");
|
|
26
|
+
const stream_utils_1 = require("../stream-utils");
|
|
27
|
+
const MAX_REPORTED_DELETE_ERRORS = 5;
|
|
28
|
+
let ObjectStoreService = class ObjectStoreService {
|
|
29
|
+
constructor(logger, s3Config) {
|
|
30
|
+
this.logger = logger;
|
|
31
|
+
this.s3Config = s3Config;
|
|
32
|
+
this.isReady = false;
|
|
33
|
+
const { bucket } = s3Config;
|
|
34
|
+
if (bucket.name === '') {
|
|
35
|
+
throw new n8n_workflow_1.UnexpectedError('External storage bucket name not configured. Please set `N8N_EXTERNAL_STORAGE_S3_BUCKET_NAME`.');
|
|
36
|
+
}
|
|
37
|
+
this.bucket = bucket.name;
|
|
38
|
+
this.s3Client = new client_s3_1.S3Client(this.getClientConfig());
|
|
39
|
+
}
|
|
40
|
+
getClientConfig() {
|
|
41
|
+
const { host, bucket, protocol, credentials, maxAttempts } = this.s3Config;
|
|
42
|
+
const clientConfig = {};
|
|
43
|
+
const endpoint = host ? `${protocol}://${host}` : undefined;
|
|
44
|
+
if (endpoint) {
|
|
45
|
+
clientConfig.endpoint = endpoint;
|
|
46
|
+
clientConfig.forcePathStyle = this.s3Config.forcePathStyle;
|
|
47
|
+
}
|
|
48
|
+
if (bucket.region.length) {
|
|
49
|
+
clientConfig.region = bucket.region;
|
|
50
|
+
}
|
|
51
|
+
if (!credentials.authAutoDetect) {
|
|
52
|
+
clientConfig.credentials = {
|
|
53
|
+
accessKeyId: credentials.accessKey,
|
|
54
|
+
secretAccessKey: credentials.accessSecret,
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
clientConfig.maxAttempts = maxAttempts;
|
|
58
|
+
return clientConfig;
|
|
59
|
+
}
|
|
60
|
+
async init() {
|
|
61
|
+
await this.checkConnection();
|
|
62
|
+
this.setReady(true);
|
|
63
|
+
}
|
|
64
|
+
setReady(newState) {
|
|
65
|
+
this.isReady = newState;
|
|
66
|
+
}
|
|
67
|
+
async checkConnection() {
|
|
68
|
+
if (this.isReady)
|
|
69
|
+
return;
|
|
70
|
+
try {
|
|
71
|
+
this.logger.debug('Checking connection to S3 bucket', { bucket: this.bucket });
|
|
72
|
+
const command = new client_s3_1.HeadBucketCommand({ Bucket: this.bucket });
|
|
73
|
+
await this.s3Client.send(command);
|
|
74
|
+
}
|
|
75
|
+
catch (e) {
|
|
76
|
+
this.handleS3Error(e);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
async put(filename, buffer, metadata = {}) {
|
|
80
|
+
try {
|
|
81
|
+
const params = {
|
|
82
|
+
Bucket: this.bucket,
|
|
83
|
+
Key: filename,
|
|
84
|
+
Body: buffer,
|
|
85
|
+
ContentLength: buffer.length,
|
|
86
|
+
ContentMD5: (0, node_crypto_1.createHash)('md5').update(buffer).digest('base64'),
|
|
87
|
+
};
|
|
88
|
+
if (metadata.fileName) {
|
|
89
|
+
params.Metadata = { filename: encodeURIComponent(metadata.fileName) };
|
|
90
|
+
}
|
|
91
|
+
if (metadata.mimeType) {
|
|
92
|
+
params.ContentType = metadata.mimeType;
|
|
93
|
+
}
|
|
94
|
+
const { Body: _body, ...logParams } = params;
|
|
95
|
+
this.logger.debug('Sending PUT request to S3', { params: logParams });
|
|
96
|
+
const command = new client_s3_1.PutObjectCommand(params);
|
|
97
|
+
return await this.s3Client.send(command);
|
|
98
|
+
}
|
|
99
|
+
catch (e) {
|
|
100
|
+
this.handleS3Error(e);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
async get(fileId, { mode, chunkSize = 0 }) {
|
|
104
|
+
this.logger.debug('Sending GET request to S3', { bucket: this.bucket, key: fileId });
|
|
105
|
+
const command = new client_s3_1.GetObjectCommand({
|
|
106
|
+
Bucket: this.bucket,
|
|
107
|
+
Key: fileId,
|
|
108
|
+
});
|
|
109
|
+
try {
|
|
110
|
+
if (mode === 'stream') {
|
|
111
|
+
const abortController = new AbortController();
|
|
112
|
+
const { Body: body } = await this.s3Client.send(command, {
|
|
113
|
+
abortSignal: abortController.signal,
|
|
114
|
+
});
|
|
115
|
+
if (!body)
|
|
116
|
+
throw new n8n_workflow_1.UnexpectedError('Received empty response body');
|
|
117
|
+
if (!(body instanceof node_stream_1.Readable)) {
|
|
118
|
+
throw new n8n_workflow_1.UnexpectedError('Expected stream but received different type', {
|
|
119
|
+
extra: { bodyType: typeof body },
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
const wrapper = new node_stream_1.PassThrough();
|
|
123
|
+
let bodyFullyConsumed = false;
|
|
124
|
+
body.on('end', () => {
|
|
125
|
+
bodyFullyConsumed = true;
|
|
126
|
+
});
|
|
127
|
+
wrapper.on('close', () => {
|
|
128
|
+
if (!bodyFullyConsumed) {
|
|
129
|
+
abortController.abort();
|
|
130
|
+
body.destroy();
|
|
131
|
+
}
|
|
132
|
+
});
|
|
133
|
+
body.on('error', (error) => wrapper.destroy(error));
|
|
134
|
+
body.pipe(wrapper);
|
|
135
|
+
if (chunkSize > 0) {
|
|
136
|
+
const rechunker = (0, stream_utils_1.createFixedSizeChunker)(chunkSize);
|
|
137
|
+
(0, node_stream_1.pipeline)(wrapper, rechunker, () => { });
|
|
138
|
+
return rechunker;
|
|
139
|
+
}
|
|
140
|
+
return wrapper;
|
|
141
|
+
}
|
|
142
|
+
const { Body: body } = await this.s3Client.send(command);
|
|
143
|
+
if (!body)
|
|
144
|
+
throw new n8n_workflow_1.UnexpectedError('Received empty response body');
|
|
145
|
+
return await (0, backend_network_1.streamToBuffer)(body);
|
|
146
|
+
}
|
|
147
|
+
catch (e) {
|
|
148
|
+
if (e instanceof n8n_workflow_1.UnexpectedError)
|
|
149
|
+
throw e;
|
|
150
|
+
this.handleS3Error(e);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
async getMetadata(fileId) {
|
|
154
|
+
try {
|
|
155
|
+
const command = new client_s3_1.HeadObjectCommand({
|
|
156
|
+
Bucket: this.bucket,
|
|
157
|
+
Key: fileId,
|
|
158
|
+
});
|
|
159
|
+
this.logger.debug('Sending HEAD request to S3', { bucket: this.bucket, key: fileId });
|
|
160
|
+
const response = await this.s3Client.send(command);
|
|
161
|
+
const headers = {};
|
|
162
|
+
if (response.ContentType)
|
|
163
|
+
headers['content-type'] = response.ContentType;
|
|
164
|
+
if (response.ContentLength)
|
|
165
|
+
headers['content-length'] = String(response.ContentLength);
|
|
166
|
+
if (response.ETag)
|
|
167
|
+
headers.etag = response.ETag;
|
|
168
|
+
if (response.LastModified)
|
|
169
|
+
headers['last-modified'] = response.LastModified.toUTCString();
|
|
170
|
+
if (response.Metadata) {
|
|
171
|
+
Object.entries(response.Metadata).forEach(([key, value]) => {
|
|
172
|
+
headers[`x-amz-meta-${key.toLowerCase()}`] =
|
|
173
|
+
key === 'filename' ? decodeURIComponent(value) : value;
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
return headers;
|
|
177
|
+
}
|
|
178
|
+
catch (e) {
|
|
179
|
+
this.handleS3Error(e);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
async deleteOne(fileId) {
|
|
183
|
+
try {
|
|
184
|
+
const command = new client_s3_1.DeleteObjectCommand({
|
|
185
|
+
Bucket: this.bucket,
|
|
186
|
+
Key: fileId,
|
|
187
|
+
});
|
|
188
|
+
this.logger.debug('Sending DELETE request to S3', { bucket: this.bucket, key: fileId });
|
|
189
|
+
return await this.s3Client.send(command);
|
|
190
|
+
}
|
|
191
|
+
catch (e) {
|
|
192
|
+
this.handleS3Error(e);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
async deleteMany(prefix) {
|
|
196
|
+
const objects = await this.list(prefix);
|
|
197
|
+
await this.deleteByKeys(objects.map(({ key }) => key));
|
|
198
|
+
}
|
|
199
|
+
async deleteByKeys(keys) {
|
|
200
|
+
if (keys.length === 0)
|
|
201
|
+
return;
|
|
202
|
+
try {
|
|
203
|
+
for (const batch of (0, chunk_1.default)(keys, 1000)) {
|
|
204
|
+
const params = {
|
|
205
|
+
Bucket: this.bucket,
|
|
206
|
+
Delete: {
|
|
207
|
+
Objects: batch.map((key) => ({ Key: key })),
|
|
208
|
+
},
|
|
209
|
+
};
|
|
210
|
+
this.logger.debug('Sending DELETE MANY request to S3', {
|
|
211
|
+
bucket: this.bucket,
|
|
212
|
+
objectCount: batch.length,
|
|
213
|
+
});
|
|
214
|
+
const { Errors: errors } = await this.s3Client.send(new client_s3_1.DeleteObjectsCommand(params));
|
|
215
|
+
if (errors && errors.length > 0) {
|
|
216
|
+
this.logger.error('Failed to delete objects from S3', {
|
|
217
|
+
bucket: this.bucket,
|
|
218
|
+
failures: errors.map((e) => ({ key: e.Key, code: e.Code, message: e.Message })),
|
|
219
|
+
});
|
|
220
|
+
const summary = errors
|
|
221
|
+
.slice(0, MAX_REPORTED_DELETE_ERRORS)
|
|
222
|
+
.map((e) => `${e.Key ?? '<unknown key>'} (${e.Code ?? '?'}: ${e.Message ?? '?'})`)
|
|
223
|
+
.join(', ');
|
|
224
|
+
throw new n8n_workflow_1.UnexpectedError(`Failed to delete ${errors.length} of ${batch.length} objects: ${summary}`);
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
catch (e) {
|
|
229
|
+
this.handleS3Error(e);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
async list(prefix) {
|
|
233
|
+
const items = [];
|
|
234
|
+
let isTruncated = true;
|
|
235
|
+
let continuationToken;
|
|
236
|
+
try {
|
|
237
|
+
while (isTruncated) {
|
|
238
|
+
const listPage = await this.getListPage(prefix, continuationToken);
|
|
239
|
+
if (listPage.contents?.length > 0) {
|
|
240
|
+
items.push(...listPage.contents);
|
|
241
|
+
}
|
|
242
|
+
isTruncated = listPage.isTruncated;
|
|
243
|
+
continuationToken = listPage.nextContinuationToken;
|
|
244
|
+
}
|
|
245
|
+
return items;
|
|
246
|
+
}
|
|
247
|
+
catch (e) {
|
|
248
|
+
this.handleS3Error(e);
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
async getListPage(prefix, continuationToken) {
|
|
252
|
+
try {
|
|
253
|
+
const params = {
|
|
254
|
+
Bucket: this.bucket,
|
|
255
|
+
Prefix: prefix,
|
|
256
|
+
};
|
|
257
|
+
if (continuationToken) {
|
|
258
|
+
params.ContinuationToken = continuationToken;
|
|
259
|
+
}
|
|
260
|
+
this.logger.debug('Sending list request to S3', { bucket: this.bucket, prefix });
|
|
261
|
+
const command = new client_s3_1.ListObjectsV2Command(params);
|
|
262
|
+
const response = await this.s3Client.send(command);
|
|
263
|
+
const contents = response.Contents?.map((item) => ({
|
|
264
|
+
key: item.Key ?? '',
|
|
265
|
+
lastModified: item.LastModified?.toISOString() ?? '',
|
|
266
|
+
eTag: item.ETag ?? '',
|
|
267
|
+
size: item.Size ?? 0,
|
|
268
|
+
storageClass: item.StorageClass ?? '',
|
|
269
|
+
})) ?? [];
|
|
270
|
+
return {
|
|
271
|
+
contents,
|
|
272
|
+
isTruncated: response.IsTruncated ?? false,
|
|
273
|
+
nextContinuationToken: response.NextContinuationToken,
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
catch (e) {
|
|
277
|
+
this.handleS3Error(e);
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
handleS3Error(e) {
|
|
281
|
+
const error = (0, ensure_error_1.ensureError)(e);
|
|
282
|
+
throw new n8n_workflow_1.UnexpectedError(`Request to S3 failed: ${error.message}`, { cause: error });
|
|
283
|
+
}
|
|
284
|
+
};
|
|
285
|
+
exports.ObjectStoreService = ObjectStoreService;
|
|
286
|
+
exports.ObjectStoreService = ObjectStoreService = __decorate([
|
|
287
|
+
(0, di_1.Service)(),
|
|
288
|
+
__metadata("design:paramtypes", [backend_common_1.Logger, object_store_config_1.ObjectStoreConfig])
|
|
289
|
+
], ObjectStoreService);
|
|
290
|
+
//# sourceMappingURL=object-store.service.ee.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"object-store.service.ee.js","sourceRoot":"","sources":["../../src/object-store/object-store.service.ee.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAMA,kDAS4B;AAC5B,wDAA6C;AAC7C,0DAAsD;AACtD,gCAAkC;AAClC,iEAA6D;AAC7D,yDAAiC;AACjC,+CAA+C;AAC/C,6CAAyC;AACzC,6CAA8D;AAE9D,+DAA0D;AAE1D,kDAAyD;AAIzD,MAAM,0BAA0B,GAAG,CAAC,CAAC;AAG9B,IAAM,kBAAkB,GAAxB,MAAM,kBAAkB;IAO9B,YACkB,MAAc,EACd,QAA2B;QAD3B,WAAM,GAAN,MAAM,CAAQ;QACd,aAAQ,GAAR,QAAQ,CAAmB;QANrC,YAAO,GAAG,KAAK,CAAC;QAQvB,MAAM,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC;QAC5B,IAAI,MAAM,CAAC,IAAI,KAAK,EAAE,EAAE,CAAC;YACxB,MAAM,IAAI,8BAAe,CACxB,gGAAgG,CAChG,CAAC;QACH,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC;QAC1B,IAAI,CAAC,QAAQ,GAAG,IAAI,oBAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;IACtD,CAAC;IAGD,eAAe;QACd,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC3E,MAAM,YAAY,GAAmB,EAAE,CAAC;QACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,QAAQ,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAC5D,IAAI,QAAQ,EAAE,CAAC;YACd,YAAY,CAAC,QAAQ,GAAG,QAAQ,CAAC;YACjC,YAAY,CAAC,cAAc,GAAG,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC;QAC5D,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YAC1B,YAAY,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QACrC,CAAC;QACD,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE,CAAC;YACjC,YAAY,CAAC,WAAW,GAAG;gBAC1B,WAAW,EAAE,WAAW,CAAC,SAAS;gBAClC,eAAe,EAAE,WAAW,CAAC,YAAY;aACzC,CAAC;QACH,CAAC;QACD,YAAY,CAAC,WAAW,GAAG,WAAW,CAAC;QACvC,OAAO,YAAY,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,IAAI;QACT,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAC7B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IACrB,CAAC;IAED,QAAQ,CAAC,QAAiB;QACzB,IAAI,CAAC,OAAO,GAAG,QAAQ,CAAC;IACzB,CAAC;IAKD,KAAK,CAAC,eAAe;QACpB,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QAEzB,IAAI,CAAC;YACJ,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,kCAAkC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YAC/E,MAAM,OAAO,GAAG,IAAI,6BAAiB,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YAC/D,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACnC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACZ,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC;IACF,CAAC;IAKD,KAAK,CAAC,GAAG,CAAC,QAAgB,EAAE,MAAc,EAAE,QAAQ,GAAyB,EAAE;QAC9E,IAAI,CAAC;YACJ,MAAM,MAAM,GAA0B;gBACrC,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,GAAG,EAAE,QAAQ;gBACb,IAAI,EAAE,MAAM;gBACZ,aAAa,EAAE,MAAM,CAAC,MAAM;gBAC5B,UAAU,EAAE,IAAA,wBAAU,EAAC,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC;aAC7D,CAAC;YAEF,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;gBACvB,MAAM,CAAC,QAAQ,GAAG,EAAE,QAAQ,EAAE,kBAAkB,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YACvE,CAAC;YAED,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;gBACvB,MAAM,CAAC,WAAW,GAAG,QAAQ,CAAC,QAAQ,CAAC;YACxC,CAAC;YAED,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,SAAS,EAAE,GAAG,MAAM,CAAC;YAC7C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;YACtE,MAAM,OAAO,GAAG,IAAI,4BAAgB,CAAC,MAAM,CAAC,CAAC;YAC7C,OAAO,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC1C,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACZ,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC;IACF,CAAC;IAYD,KAAK,CAAC,GAAG,CACR,MAAc,EACd,EAAE,IAAI,EAAE,SAAS,GAAG,CAAC,EAAqD;QAE1E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;QAErF,MAAM,OAAO,GAAG,IAAI,4BAAgB,CAAC;YACpC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,GAAG,EAAE,MAAM;SACX,CAAC,CAAC;QAEH,IAAI,CAAC;YACJ,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;gBAC9C,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE;oBACxD,WAAW,EAAE,eAAe,CAAC,MAAM;iBACnC,CAAC,CAAC;gBACH,IAAI,CAAC,IAAI;oBAAE,MAAM,IAAI,8BAAe,CAAC,8BAA8B,CAAC,CAAC;gBAErE,IAAI,CAAC,CAAC,IAAI,YAAY,sBAAQ,CAAC,EAAE,CAAC;oBACjC,MAAM,IAAI,8BAAe,CAAC,6CAA6C,EAAE;wBACxE,KAAK,EAAE,EAAE,QAAQ,EAAE,OAAO,IAAI,EAAE;qBAChC,CAAC,CAAC;gBACJ,CAAC;gBAKD,MAAM,OAAO,GAAG,IAAI,yBAAW,EAAE,CAAC;gBAClC,IAAI,iBAAiB,GAAG,KAAK,CAAC;gBAE9B,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;oBACnB,iBAAiB,GAAG,IAAI,CAAC;gBAC1B,CAAC,CAAC,CAAC;gBAEH,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;oBACxB,IAAI,CAAC,iBAAiB,EAAE,CAAC;wBACxB,eAAe,CAAC,KAAK,EAAE,CAAC;wBACxB,IAAI,CAAC,OAAO,EAAE,CAAC;oBAChB,CAAC;gBACF,CAAC,CAAC,CAAC;gBAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;gBACpD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAEnB,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;oBACnB,MAAM,SAAS,GAAG,IAAA,qCAAsB,EAAC,SAAS,CAAC,CAAC;oBACpD,IAAA,sBAAQ,EAAC,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;oBACvC,OAAO,SAAS,CAAC;gBAClB,CAAC;gBACD,OAAO,OAAO,CAAC;YAChB,CAAC;YAED,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACzD,IAAI,CAAC,IAAI;gBAAE,MAAM,IAAI,8BAAe,CAAC,8BAA8B,CAAC,CAAC;YAErE,OAAO,MAAM,IAAA,gCAAc,EAAC,IAAgB,CAAC,CAAC;QAC/C,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACZ,IAAI,CAAC,YAAY,8BAAe;gBAAE,MAAM,CAAC,CAAC;YAC1C,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC;IACF,CAAC;IAKD,KAAK,CAAC,WAAW,CAAC,MAAc;QAC/B,IAAI,CAAC;YACJ,MAAM,OAAO,GAAG,IAAI,6BAAiB,CAAC;gBACrC,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,GAAG,EAAE,MAAM;aACX,CAAC,CAAC;YAEH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,4BAA4B,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;YACtF,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAGnD,MAAM,OAAO,GAA4B,EAAE,CAAC;YAE5C,IAAI,QAAQ,CAAC,WAAW;gBAAE,OAAO,CAAC,cAAc,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC;YACzE,IAAI,QAAQ,CAAC,aAAa;gBAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;YACvF,IAAI,QAAQ,CAAC,IAAI;gBAAE,OAAO,CAAC,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;YAChD,IAAI,QAAQ,CAAC,YAAY;gBAAE,OAAO,CAAC,eAAe,CAAC,GAAG,QAAQ,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC;YAG1F,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;gBACvB,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;oBAC1D,OAAO,CAAC,cAAc,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC;wBACzC,GAAG,KAAK,UAAU,CAAC,CAAC,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;gBACzD,CAAC,CAAC,CAAC;YACJ,CAAC;YAED,OAAO,OAAO,CAAC;QAChB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACZ,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC;IACF,CAAC;IAKD,KAAK,CAAC,SAAS,CAAC,MAAc;QAC7B,IAAI,CAAC;YACJ,MAAM,OAAO,GAAG,IAAI,+BAAmB,CAAC;gBACvC,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,GAAG,EAAE,MAAM;aACX,CAAC,CAAC;YAEH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;YACxF,OAAO,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC1C,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACZ,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC;IACF,CAAC;IAKD,KAAK,CAAC,UAAU,CAAC,MAAc;QAC9B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAExC,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACxD,CAAC;IAMD,KAAK,CAAC,YAAY,CAAC,IAAc;QAChC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAE9B,IAAI,CAAC;YACJ,KAAK,MAAM,KAAK,IAAI,IAAA,eAAK,EAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;gBACvC,MAAM,MAAM,GAA8B;oBACzC,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,MAAM,EAAE;wBACP,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;qBAC3C;iBACD,CAAC;gBAEF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAmC,EAAE;oBACtD,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,WAAW,EAAE,KAAK,CAAC,MAAM;iBACzB,CAAC,CAAC;gBAGH,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,gCAAoB,CAAC,MAAM,CAAC,CAAC,CAAC;gBAEtF,IAAI,MAAM,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACjC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,kCAAkC,EAAE;wBACrD,MAAM,EAAE,IAAI,CAAC,MAAM;wBACnB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;qBAC/E,CAAC,CAAC;oBAEH,MAAM,OAAO,GAAG,MAAM;yBACpB,KAAK,CAAC,CAAC,EAAE,0BAA0B,CAAC;yBACpC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,eAAe,KAAK,CAAC,CAAC,IAAI,IAAI,GAAG,KAAK,CAAC,CAAC,OAAO,IAAI,GAAG,GAAG,CAAC;yBACjF,IAAI,CAAC,IAAI,CAAC,CAAC;oBACb,MAAM,IAAI,8BAAe,CACxB,oBAAoB,MAAM,CAAC,MAAM,OAAO,KAAK,CAAC,MAAM,aAAa,OAAO,EAAE,CAC1E,CAAC;gBACH,CAAC;YACF,CAAC;QACF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACZ,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC;IACF,CAAC;IAKD,KAAK,CAAC,IAAI,CAAC,MAAc;QACxB,MAAM,KAAK,GAAG,EAAE,CAAC;QACjB,IAAI,WAAW,GAAG,IAAI,CAAC;QACvB,IAAI,iBAAiB,CAAC;QAEtB,IAAI,CAAC;YACJ,OAAO,WAAW,EAAE,CAAC;gBACpB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC;gBAEnE,IAAI,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,EAAE,CAAC;oBACnC,KAAK,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;gBAClC,CAAC;gBAED,WAAW,GAAG,QAAQ,CAAC,WAAW,CAAC;gBACnC,iBAAiB,GAAG,QAAQ,CAAC,qBAAqB,CAAC;YACpD,CAAC;YAED,OAAO,KAAK,CAAC;QACd,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACZ,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC;IACF,CAAC;IAKD,KAAK,CAAC,WAAW,CAAC,MAAc,EAAE,iBAA0B;QAC3D,IAAI,CAAC;YACJ,MAAM,MAAM,GAA8B;gBACzC,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE,MAAM;aACd,CAAC;YAEF,IAAI,iBAAiB,EAAE,CAAC;gBACvB,MAAM,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;YAC9C,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,4BAA4B,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;YACjF,MAAM,OAAO,GAAG,IAAI,gCAAoB,CAAC,MAAM,CAAC,CAAC;YACjD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAGnD,MAAM,QAAQ,GACb,QAAQ,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBACjC,GAAG,EAAE,IAAI,CAAC,GAAG,IAAI,EAAE;gBACnB,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,WAAW,EAAE,IAAI,EAAE;gBACpD,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE;gBACrB,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC;gBACpB,YAAY,EAAE,IAAI,CAAC,YAAY,IAAI,EAAE;aACrC,CAAC,CAAC,IAAI,EAAE,CAAC;YAEX,OAAO;gBACN,QAAQ;gBACR,WAAW,EAAE,QAAQ,CAAC,WAAW,IAAI,KAAK;gBAC1C,qBAAqB,EAAE,QAAQ,CAAC,qBAAqB;aACrD,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACZ,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC;IACF,CAAC;IAEO,aAAa,CAAC,CAAU;QAC/B,MAAM,KAAK,GAAG,IAAA,0BAAW,EAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,IAAI,8BAAe,CAAC,yBAAyB,KAAK,CAAC,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IACvF,CAAC;CACD,CAAA;;6BAxVY,kBAAkB;IAD9B,IAAA,YAAO,GAAE;qCASiB,uBAAM,EACJ,uCAAiB;GATjC,kBAAkB,CAwV9B"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { PreWriteBlobMetadata } from '../types';
|
|
2
|
+
export type MetadataResponseHeaders = Record<string, string> & {
|
|
3
|
+
'content-length'?: string;
|
|
4
|
+
'content-type'?: string;
|
|
5
|
+
'x-amz-meta-filename'?: string;
|
|
6
|
+
etag?: string;
|
|
7
|
+
'last-modified'?: string;
|
|
8
|
+
} & PreWriteBlobMetadata;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/object-store/types.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.createFixedSizeChunker = createFixedSizeChunker;
|
|
4
|
+
const n8n_workflow_1 = require("n8n-workflow");
|
|
5
|
+
const node_stream_1 = require("node:stream");
|
|
6
|
+
function createFixedSizeChunker(chunkSize) {
|
|
7
|
+
if (chunkSize <= 0) {
|
|
8
|
+
throw new n8n_workflow_1.UnexpectedError(`createFixedSizeChunker requires chunkSize > 0, got ${chunkSize}`);
|
|
9
|
+
}
|
|
10
|
+
const queue = [];
|
|
11
|
+
let queued = 0;
|
|
12
|
+
const take = (size) => {
|
|
13
|
+
const out = Buffer.allocUnsafe(size);
|
|
14
|
+
let written = 0;
|
|
15
|
+
while (written < size) {
|
|
16
|
+
const head = queue[0];
|
|
17
|
+
const need = size - written;
|
|
18
|
+
if (head.length <= need) {
|
|
19
|
+
head.copy(out, written);
|
|
20
|
+
written += head.length;
|
|
21
|
+
queue.shift();
|
|
22
|
+
}
|
|
23
|
+
else {
|
|
24
|
+
head.copy(out, written, 0, need);
|
|
25
|
+
queue[0] = head.subarray(need);
|
|
26
|
+
written += need;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
queued -= size;
|
|
30
|
+
return out;
|
|
31
|
+
};
|
|
32
|
+
return new node_stream_1.Transform({
|
|
33
|
+
transform(chunk, _encoding, done) {
|
|
34
|
+
queue.push(chunk);
|
|
35
|
+
queued += chunk.length;
|
|
36
|
+
while (queued >= chunkSize) {
|
|
37
|
+
this.push(take(chunkSize));
|
|
38
|
+
}
|
|
39
|
+
done();
|
|
40
|
+
},
|
|
41
|
+
flush(done) {
|
|
42
|
+
if (queued > 0) {
|
|
43
|
+
this.push(take(queued));
|
|
44
|
+
}
|
|
45
|
+
done();
|
|
46
|
+
},
|
|
47
|
+
destroy(error, done) {
|
|
48
|
+
queue.length = 0;
|
|
49
|
+
queued = 0;
|
|
50
|
+
done(error);
|
|
51
|
+
},
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
//# sourceMappingURL=stream-utils.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stream-utils.js","sourceRoot":"","sources":["../src/stream-utils.ts"],"names":[],"mappings":";;;AAAA,+CAA+C;AAC/C,6CAAwC;AAcxC,gCAAuC,SAAiB;IACvD,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;QACpB,MAAM,IAAI,8BAAe,CAAC,sDAAsD,SAAS,EAAE,CAAC,CAAC;IAC9F,CAAC;IAED,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,MAAM,IAAI,GAAG,CAAC,IAAY,EAAU,EAAE;QACrC,MAAM,GAAG,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,OAAO,OAAO,GAAG,IAAI,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,IAAI,GAAG,IAAI,GAAG,OAAO,CAAC;YAC5B,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;gBACzB,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;gBACxB,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC;gBACvB,KAAK,CAAC,KAAK,EAAE,CAAC;YACf,CAAC;iBAAM,CAAC;gBACP,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;gBACjC,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gBAC/B,OAAO,IAAI,IAAI,CAAC;YACjB,CAAC;QACF,CAAC;QACD,MAAM,IAAI,IAAI,CAAC;QACf,OAAO,GAAG,CAAC;IACZ,CAAC,CAAC;IAEF,OAAO,IAAI,uBAAS,CAAC;QACpB,SAAS,CAAC,KAAa,EAAE,SAAS,EAAE,IAAI;YACvC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;YACvB,OAAO,MAAM,IAAI,SAAS,EAAE,CAAC;gBAC5B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;YAC5B,CAAC;YACD,IAAI,EAAE,CAAC;QACR,CAAC;QACD,KAAK,CAAC,IAAI;YACT,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;YACzB,CAAC;YACD,IAAI,EAAE,CAAC;QACR,CAAC;QACD,OAAO,CAAC,KAAK,EAAE,IAAI;YAClB,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;YACjB,MAAM,GAAG,CAAC,CAAC;YACX,IAAI,CAAC,KAAK,CAAC,CAAC;QACb,CAAC;KACD,CAAC,CAAC;AACJ,CAAC"}
|
package/dist/types.d.ts
ADDED
package/dist/types.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}
|