@toa.io/extensions.storages 1.0.0-alpha.7 → 1.0.0-alpha.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -8
- package/readme.md +32 -22
- package/schemas/fs.cos.yaml +2 -0
- package/schemas/s3.cos.yaml +1 -0
- package/schemas/tmp.cos.yaml +0 -1
- package/source/Entry.ts +1 -0
- package/source/Factory.ts +7 -4
- package/source/Provider.ts +5 -0
- package/source/Scanner.ts +46 -17
- package/source/Storage.test.ts +126 -76
- package/source/Storage.ts +92 -91
- package/source/deployment.ts +38 -7
- package/source/providers/FileSystem.test.ts +6 -0
- package/source/providers/FileSystem.ts +15 -2
- package/source/providers/Memory.ts +21 -7
- package/source/providers/S3.test.ts +1 -0
- package/source/providers/S3.ts +36 -6
- package/source/providers/index.test.ts +10 -0
- package/source/test/arny.jpg +0 -0
- package/source/test/sample.avi +0 -0
- package/source/test/sample.svg +4 -0
- package/source/test/sample.wav +0 -0
- package/transpiled/Entry.d.ts +1 -0
- package/transpiled/Factory.js +5 -3
- package/transpiled/Factory.js.map +1 -1
- package/transpiled/Provider.d.ts +3 -0
- package/transpiled/Provider.js +1 -0
- package/transpiled/Provider.js.map +1 -1
- package/transpiled/Scanner.d.ts +3 -1
- package/transpiled/Scanner.js +36 -15
- package/transpiled/Scanner.js.map +1 -1
- package/transpiled/Storage.d.ts +6 -6
- package/transpiled/Storage.js +66 -78
- package/transpiled/Storage.js.map +1 -1
- package/transpiled/deployment.d.ts +1 -1
- package/transpiled/deployment.js +30 -7
- package/transpiled/deployment.js.map +1 -1
- package/transpiled/providers/FileSystem.d.ts +4 -1
- package/transpiled/providers/FileSystem.js +10 -1
- package/transpiled/providers/FileSystem.js.map +1 -1
- package/transpiled/providers/Memory.d.ts +2 -0
- package/transpiled/providers/Memory.js +13 -1
- package/transpiled/providers/Memory.js.map +1 -1
- package/transpiled/providers/S3.d.ts +3 -0
- package/transpiled/providers/S3.js +23 -4
- package/transpiled/providers/S3.js.map +1 -1
- package/transpiled/tsconfig.tsbuildinfo +1 -1
package/source/Storage.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { Readable } from 'node:stream'
|
|
2
2
|
import { posix } from 'node:path'
|
|
3
|
+
import { buffer } from 'node:stream/consumers'
|
|
3
4
|
import { decode, encode } from 'msgpackr'
|
|
4
|
-
import {
|
|
5
|
+
import { newid } from '@toa.io/generic'
|
|
5
6
|
import { Err } from 'error-value'
|
|
6
7
|
import { Scanner } from './Scanner'
|
|
7
8
|
import type { ScanOptions } from './Scanner'
|
|
@@ -27,15 +28,33 @@ export class Storage {
|
|
|
27
28
|
|
|
28
29
|
await this.persist(tempname, id)
|
|
29
30
|
|
|
30
|
-
|
|
31
|
+
const entry: Entry = {
|
|
32
|
+
id,
|
|
33
|
+
size: scanner.size,
|
|
34
|
+
type: scanner.type,
|
|
35
|
+
origin: options?.origin,
|
|
36
|
+
created: Date.now(),
|
|
37
|
+
variants: [],
|
|
38
|
+
meta: options?.meta ?? {}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
return await this.create(path, entry)
|
|
31
42
|
}
|
|
32
43
|
|
|
33
44
|
public async get (path: string): Maybe<Entry> {
|
|
34
|
-
const
|
|
35
|
-
const result = await this.provider.get(
|
|
45
|
+
const paths = this.destruct(path)
|
|
46
|
+
const result = await this.provider.get(paths.metafile)
|
|
36
47
|
|
|
37
|
-
if (result === null)
|
|
38
|
-
|
|
48
|
+
if (result === null)
|
|
49
|
+
return ERR_NOT_FOUND
|
|
50
|
+
else
|
|
51
|
+
return decode(await buffer(result))
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
public async list (path: string): Promise<string[]> {
|
|
55
|
+
const dir = posix.join(ENTRIES_ROOT, path, ENTRIES_DIR)
|
|
56
|
+
|
|
57
|
+
return await this.provider.list(dir)
|
|
39
58
|
}
|
|
40
59
|
|
|
41
60
|
public async fetch (path: string): Maybe<Readable> {
|
|
@@ -50,12 +69,14 @@ export class Storage {
|
|
|
50
69
|
|
|
51
70
|
const blob = variant === null
|
|
52
71
|
? posix.join(BLOBs, id)
|
|
53
|
-
: posix.join(
|
|
72
|
+
: posix.join(ENTRIES_ROOT, rel, id, variant)
|
|
54
73
|
|
|
55
74
|
const stream = await this.provider.get(blob)
|
|
56
75
|
|
|
57
|
-
if (stream === null)
|
|
58
|
-
|
|
76
|
+
if (stream === null)
|
|
77
|
+
return ERR_NOT_FOUND
|
|
78
|
+
else
|
|
79
|
+
return stream
|
|
59
80
|
}
|
|
60
81
|
|
|
61
82
|
public async delete (path: string): Maybe<void> {
|
|
@@ -64,57 +85,38 @@ export class Storage {
|
|
|
64
85
|
if (entry instanceof Error)
|
|
65
86
|
return entry
|
|
66
87
|
|
|
67
|
-
|
|
68
|
-
await this.provider.delete(posix.join(ENTRIES, path))
|
|
69
|
-
}
|
|
88
|
+
const paths = this.destruct(path)
|
|
70
89
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
return stream === null ? [] : decode(await buffer(stream))
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
public async permute (path: string, ids: string[]): Maybe<void> {
|
|
79
|
-
const unique = new Set(ids)
|
|
80
|
-
const dir = posix.join(ENTRIES, path)
|
|
81
|
-
const list = await this.list(path)
|
|
82
|
-
|
|
83
|
-
if (list.length !== ids.length || unique.size !== ids.length)
|
|
84
|
-
return ERR_PERMUTATION_MISMATCH
|
|
85
|
-
|
|
86
|
-
for (const id of ids)
|
|
87
|
-
if (!list.includes(id))
|
|
88
|
-
return ERR_PERMUTATION_MISMATCH
|
|
89
|
-
|
|
90
|
-
await this.provider.put(dir, LIST, Readable.from(encode(ids)))
|
|
90
|
+
await Promise.all([
|
|
91
|
+
this.provider.delete(paths.metafile),
|
|
92
|
+
this.provider.delete(paths.vardir)
|
|
93
|
+
])
|
|
91
94
|
}
|
|
92
95
|
|
|
93
|
-
public async
|
|
94
|
-
const
|
|
95
|
-
const
|
|
96
|
-
const
|
|
97
|
-
const index = list.indexOf(id)
|
|
98
|
-
|
|
99
|
-
if (index === -1)
|
|
100
|
-
return ERR_NOT_FOUND
|
|
96
|
+
public async move (path: string, to: string): Maybe<void> {
|
|
97
|
+
const source = this.destruct(path)
|
|
98
|
+
const rel = to.startsWith('.')
|
|
99
|
+
const dir = to.endsWith('/')
|
|
101
100
|
|
|
102
|
-
|
|
101
|
+
if (rel)
|
|
102
|
+
to = posix.resolve(source.rel + '/', to)
|
|
103
103
|
|
|
104
|
-
|
|
105
|
-
|
|
104
|
+
if (dir)
|
|
105
|
+
to = posix.join(to, source.ent)
|
|
106
106
|
|
|
107
|
-
|
|
108
|
-
const { id, rel } = this.parse(path)
|
|
107
|
+
const target = this.destruct(to)
|
|
109
108
|
|
|
110
|
-
|
|
109
|
+
await Promise.all([
|
|
110
|
+
this.provider.move(source.metafile, target.metafile),
|
|
111
|
+
this.provider.moveDir(source.vardir, target.vardir)
|
|
112
|
+
])
|
|
111
113
|
}
|
|
112
114
|
|
|
113
115
|
public async diversify (path: string, name: string, stream: Readable): Maybe<void> {
|
|
114
116
|
const scanner = new Scanner()
|
|
115
117
|
const pipe = stream.pipe(scanner)
|
|
116
118
|
|
|
117
|
-
await this.provider.put(posix.join(
|
|
119
|
+
await this.provider.put(posix.join(ENTRIES_ROOT, path), name, pipe)
|
|
118
120
|
|
|
119
121
|
if (scanner.error !== null)
|
|
120
122
|
return scanner.error
|
|
@@ -131,19 +133,30 @@ export class Storage {
|
|
|
131
133
|
await this.save(path, entry)
|
|
132
134
|
}
|
|
133
135
|
|
|
134
|
-
public async annotate (path: string, key: string, value?: unknown): Maybe<void> {
|
|
136
|
+
public async annotate (path: string, key: string | Record<string, unknown>, value?: unknown): Maybe<void> {
|
|
135
137
|
const entry = await this.get(path)
|
|
136
138
|
|
|
137
139
|
if (entry instanceof Error)
|
|
138
140
|
return entry
|
|
139
141
|
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
142
|
+
const update = typeof key === 'string'
|
|
143
|
+
? { [key]: value }
|
|
144
|
+
: key
|
|
145
|
+
|
|
146
|
+
Object.assign(entry.meta, update)
|
|
147
|
+
|
|
148
|
+
// filter undefined values
|
|
149
|
+
for (const key of Object.keys(entry.meta))
|
|
150
|
+
if (entry.meta[key] === undefined)
|
|
151
|
+
delete entry.meta[key]
|
|
143
152
|
|
|
144
153
|
await this.save(path, entry)
|
|
145
154
|
}
|
|
146
155
|
|
|
156
|
+
public path (): string | null {
|
|
157
|
+
return this.provider.path
|
|
158
|
+
}
|
|
159
|
+
|
|
147
160
|
private async transit (stream: Readable): Promise<string> {
|
|
148
161
|
const tempname = newid()
|
|
149
162
|
|
|
@@ -161,52 +174,21 @@ export class Storage {
|
|
|
161
174
|
|
|
162
175
|
// eslint-disable-next-line max-params
|
|
163
176
|
private async create
|
|
164
|
-
(path: string,
|
|
165
|
-
const entry: Entry = {
|
|
166
|
-
id,
|
|
167
|
-
size,
|
|
168
|
-
type,
|
|
169
|
-
created: Date.now(),
|
|
170
|
-
variants: [],
|
|
171
|
-
meta
|
|
172
|
-
}
|
|
173
|
-
|
|
177
|
+
(path: string, entry: Entry): Promise<Entry> {
|
|
174
178
|
const metafile = posix.join(path, entry.id)
|
|
175
179
|
const existing = await this.get(metafile)
|
|
176
180
|
|
|
177
181
|
if (existing instanceof Error)
|
|
178
182
|
await this.save(metafile, entry)
|
|
179
183
|
|
|
180
|
-
await this.enroll(path, id, true)
|
|
181
|
-
|
|
182
184
|
return entry
|
|
183
185
|
}
|
|
184
186
|
|
|
185
|
-
private async save (
|
|
186
|
-
const
|
|
187
|
-
const stream = Readable.from(
|
|
188
|
-
|
|
189
|
-
await this.provider.put(posix.join(ENTRIES, rel), META, stream)
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
private async enroll (path: string, id: string, addition: boolean = false): Maybe<void> {
|
|
193
|
-
const dir = posix.join(ENTRIES, path)
|
|
194
|
-
const list = await this.list(path)
|
|
195
|
-
const index = list.indexOf(id)
|
|
187
|
+
private async save (path: string, entry: Entry): Promise<void> {
|
|
188
|
+
const paths = this.destruct(path)
|
|
189
|
+
const stream = Readable.from(encode(entry))
|
|
196
190
|
|
|
197
|
-
|
|
198
|
-
if (addition) list.splice(index, 1)
|
|
199
|
-
else return
|
|
200
|
-
else if (!addition) {
|
|
201
|
-
const entry = await this.get(posix.join(path, id))
|
|
202
|
-
|
|
203
|
-
if (entry instanceof Error)
|
|
204
|
-
return entry
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
list.push(id)
|
|
208
|
-
|
|
209
|
-
await this.provider.put(dir, LIST, Readable.from(encode(list)))
|
|
191
|
+
await this.provider.put(paths.metadir, paths.ent, stream)
|
|
210
192
|
}
|
|
211
193
|
|
|
212
194
|
private parse (path: string): Path {
|
|
@@ -217,16 +199,25 @@ export class Storage {
|
|
|
217
199
|
|
|
218
200
|
return { rel, id, variant }
|
|
219
201
|
}
|
|
202
|
+
|
|
203
|
+
private destruct (path: string): Paths {
|
|
204
|
+
const rel = posix.dirname(path)
|
|
205
|
+
const dir = posix.join(ENTRIES_ROOT, rel)
|
|
206
|
+
const ent = posix.basename(path)
|
|
207
|
+
const metadir = posix.join(dir, ENTRIES_DIR)
|
|
208
|
+
const metafile = posix.join(metadir, ent)
|
|
209
|
+
const vardir = posix.join(dir, ent)
|
|
210
|
+
|
|
211
|
+
return { rel, dir, ent, metadir, metafile, vardir }
|
|
212
|
+
}
|
|
220
213
|
}
|
|
221
214
|
|
|
222
215
|
const ERR_NOT_FOUND = Err('NOT_FOUND')
|
|
223
|
-
const ERR_PERMUTATION_MISMATCH = Err('PERMUTATION_MISMATCH')
|
|
224
216
|
|
|
225
217
|
const TEMP = '/temp'
|
|
226
218
|
const BLOBs = '/blobs'
|
|
227
|
-
const
|
|
228
|
-
const
|
|
229
|
-
const META = '.meta'
|
|
219
|
+
const ENTRIES_ROOT = '/entries'
|
|
220
|
+
const ENTRIES_DIR = '.meta'
|
|
230
221
|
|
|
231
222
|
type Maybe<T> = Promise<T | Error>
|
|
232
223
|
|
|
@@ -236,9 +227,19 @@ interface Path {
|
|
|
236
227
|
variant: string | null
|
|
237
228
|
}
|
|
238
229
|
|
|
230
|
+
interface Paths {
|
|
231
|
+
rel: string
|
|
232
|
+
dir: string
|
|
233
|
+
ent: string
|
|
234
|
+
metadir: string
|
|
235
|
+
metafile: string
|
|
236
|
+
vardir: string
|
|
237
|
+
}
|
|
238
|
+
|
|
239
239
|
type Meta = Record<string, string>
|
|
240
240
|
|
|
241
241
|
interface Options extends ScanOptions {
|
|
242
|
+
origin?: string
|
|
242
243
|
meta?: Meta
|
|
243
244
|
}
|
|
244
245
|
|
package/source/deployment.ts
CHANGED
|
@@ -3,21 +3,25 @@ import { encode } from '@toa.io/generic'
|
|
|
3
3
|
import { providers } from './providers'
|
|
4
4
|
import { validateAnnotation } from './Annotation'
|
|
5
5
|
import type { Annotation } from './Annotation'
|
|
6
|
-
import type { Dependency, Variable } from '@toa.io/operations'
|
|
6
|
+
import type { Dependency, Variable, Mounts } from '@toa.io/operations'
|
|
7
7
|
import type { context } from '@toa.io/norm'
|
|
8
8
|
|
|
9
|
-
export const
|
|
9
|
+
export const ENV_PREFIX = 'TOA_STORAGES'
|
|
10
10
|
|
|
11
11
|
export function deployment (instances: Instance[], annotation: unknown): Dependency {
|
|
12
12
|
validate(instances, annotation)
|
|
13
13
|
|
|
14
14
|
const value = encode(annotation)
|
|
15
|
-
const pointer: Variable = { name:
|
|
15
|
+
const pointer: Variable = { name: ENV_PREFIX, value }
|
|
16
16
|
const secrets = getSecrets(annotation)
|
|
17
|
+
const mounts = getMounts(instances, annotation)
|
|
17
18
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
19
|
+
const dependency: Dependency = { variables: { global: [pointer, ...secrets] } }
|
|
20
|
+
|
|
21
|
+
if (mounts !== null)
|
|
22
|
+
dependency.mounts = mounts
|
|
23
|
+
|
|
24
|
+
return dependency
|
|
21
25
|
}
|
|
22
26
|
|
|
23
27
|
function validate (instances: Instance[], annotation: unknown): asserts annotation is Annotation {
|
|
@@ -42,7 +46,7 @@ function getSecrets (annotation: Annotation): Variable[] {
|
|
|
42
46
|
|
|
43
47
|
for (const secret of Provider.SECRETS)
|
|
44
48
|
secrets.push({
|
|
45
|
-
name: `${
|
|
49
|
+
name: `${ENV_PREFIX}_${name}_${secret.name}`.toUpperCase(),
|
|
46
50
|
secret: {
|
|
47
51
|
name: `toa-storages-${name}`,
|
|
48
52
|
key: secret.name,
|
|
@@ -54,4 +58,31 @@ function getSecrets (annotation: Annotation): Variable[] {
|
|
|
54
58
|
return secrets
|
|
55
59
|
}
|
|
56
60
|
|
|
61
|
+
function getMounts (instances: Instance[], annotation: Annotation): Mounts | null {
|
|
62
|
+
let mounts: Mounts | null = null
|
|
63
|
+
|
|
64
|
+
for (const { locator, manifest } of instances)
|
|
65
|
+
for (const name of manifest) {
|
|
66
|
+
const declaration = annotation[name]
|
|
67
|
+
|
|
68
|
+
// eslint-disable-next-line max-depth
|
|
69
|
+
if (declaration.provider !== 'fs')
|
|
70
|
+
continue
|
|
71
|
+
|
|
72
|
+
// eslint-disable-next-line max-depth
|
|
73
|
+
if (declaration.claim !== undefined) {
|
|
74
|
+
mounts ??= {}
|
|
75
|
+
mounts[locator.label] ??= []
|
|
76
|
+
|
|
77
|
+
mounts[locator.label].push({
|
|
78
|
+
name,
|
|
79
|
+
path: declaration.path,
|
|
80
|
+
claim: declaration.claim
|
|
81
|
+
})
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
return mounts
|
|
86
|
+
}
|
|
87
|
+
|
|
57
88
|
export type Instance = context.Dependency<string[]>
|
|
@@ -3,3 +3,9 @@ import { FileSystem } from './FileSystem'
|
|
|
3
3
|
it('should be ok', async () => {
|
|
4
4
|
expect(() => new FileSystem({ path: 'path' })).not.toThrow()
|
|
5
5
|
})
|
|
6
|
+
|
|
7
|
+
it('should expose path', () => {
|
|
8
|
+
const fs = new FileSystem({ path: '/tmp/foo' })
|
|
9
|
+
|
|
10
|
+
expect(fs.path).toBe('/tmp/foo')
|
|
11
|
+
})
|
|
@@ -6,10 +6,11 @@ import type { ProviderSecrets } from '../Provider'
|
|
|
6
6
|
|
|
7
7
|
export interface FileSystemOptions {
|
|
8
8
|
path: string
|
|
9
|
+
claim?: string
|
|
9
10
|
}
|
|
10
11
|
|
|
11
12
|
export class FileSystem extends Provider<FileSystemOptions> {
|
|
12
|
-
|
|
13
|
+
public override readonly path: string
|
|
13
14
|
|
|
14
15
|
public constructor (options: FileSystemOptions, secrets?: ProviderSecrets) {
|
|
15
16
|
super(options, secrets)
|
|
@@ -29,11 +30,19 @@ export class FileSystem extends Provider<FileSystemOptions> {
|
|
|
29
30
|
}
|
|
30
31
|
}
|
|
31
32
|
|
|
33
|
+
public async list (path: string): Promise<string[]> {
|
|
34
|
+
const dir = join(this.path, path)
|
|
35
|
+
|
|
36
|
+
return (await fs.readdir(dir, { withFileTypes: true }))
|
|
37
|
+
.filter((dirent) => dirent.isFile())
|
|
38
|
+
.map((dirent) => dirent.name)
|
|
39
|
+
}
|
|
40
|
+
|
|
32
41
|
public async put (rel: string, filename: string, stream: Readable): Promise<void> {
|
|
33
42
|
const dir = join(this.path, rel)
|
|
34
43
|
const path = join(dir, filename)
|
|
35
44
|
|
|
36
|
-
await fs.mkdir(
|
|
45
|
+
await fs.mkdir(dir, { recursive: true })
|
|
37
46
|
await fs.writeFile(path, stream)
|
|
38
47
|
}
|
|
39
48
|
|
|
@@ -48,4 +57,8 @@ export class FileSystem extends Provider<FileSystemOptions> {
|
|
|
48
57
|
await fs.mkdir(dirname(to), { recursive: true })
|
|
49
58
|
await fs.rename(from, to)
|
|
50
59
|
}
|
|
60
|
+
|
|
61
|
+
public async moveDir (from: string, to: string): Promise<void> {
|
|
62
|
+
await this.move(from, to).catch(() => null)
|
|
63
|
+
}
|
|
51
64
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Readable } from 'node:stream'
|
|
2
|
-
import { join } from 'node:path'
|
|
2
|
+
import { join, posix } from 'node:path'
|
|
3
3
|
import { buffer } from 'node:stream/consumers'
|
|
4
4
|
import * as assert from 'node:assert'
|
|
5
5
|
|
|
@@ -11,7 +11,7 @@ import { Provider } from '../Provider'
|
|
|
11
11
|
export class InMemory extends Provider {
|
|
12
12
|
private readonly storage = new Map<string, Buffer>()
|
|
13
13
|
|
|
14
|
-
public
|
|
14
|
+
public async get (path: string): Promise<Readable | null> {
|
|
15
15
|
const data = this.storage.get(path)
|
|
16
16
|
|
|
17
17
|
if (data === undefined) return null
|
|
@@ -19,18 +19,22 @@ export class InMemory extends Provider {
|
|
|
19
19
|
return Readable.from(data)
|
|
20
20
|
}
|
|
21
21
|
|
|
22
|
-
public
|
|
22
|
+
public async list (path: string): Promise<string[]> {
|
|
23
|
+
return Array.from(this.storage.keys())
|
|
24
|
+
.filter((f) => posix.dirname(f) === path)
|
|
25
|
+
.map((f) => posix.basename(f))
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
public async put (path: string, filename: string, stream: Readable): Promise<void> {
|
|
23
29
|
this.storage.set(join(path, filename), await buffer(stream))
|
|
24
30
|
}
|
|
25
31
|
|
|
26
|
-
public
|
|
32
|
+
public async delete (path: string): Promise<void> {
|
|
27
33
|
for (const f of this.storage.keys())
|
|
28
34
|
if (f.startsWith(path)) this.storage.delete(f)
|
|
29
35
|
}
|
|
30
36
|
|
|
31
|
-
public
|
|
32
|
-
assert.notEqual(from, to, 'Source and destination are the same')
|
|
33
|
-
|
|
37
|
+
public async move (from: string, to: string): Promise<void> {
|
|
34
38
|
const buf = this.storage.get(from)
|
|
35
39
|
|
|
36
40
|
assert.ok(buf !== undefined, `File not found: ${from}`)
|
|
@@ -38,4 +42,14 @@ export class InMemory extends Provider {
|
|
|
38
42
|
this.storage.set(to, buf)
|
|
39
43
|
this.storage.delete(from)
|
|
40
44
|
}
|
|
45
|
+
|
|
46
|
+
public async moveDir (from: string, to: string): Promise<void> {
|
|
47
|
+
for (const f of this.storage.keys())
|
|
48
|
+
if (f.startsWith(from)) {
|
|
49
|
+
const toPath = to + f.slice(from.length)
|
|
50
|
+
|
|
51
|
+
this.storage.set(toPath, this.storage.get(f)!)
|
|
52
|
+
this.storage.delete(f)
|
|
53
|
+
}
|
|
54
|
+
}
|
|
41
55
|
}
|
|
@@ -105,6 +105,7 @@ describe('S3 storage provider', () => {
|
|
|
105
105
|
deleteHandler))
|
|
106
106
|
|
|
107
107
|
await provider.delete('/some/absolute/path_to_remove')
|
|
108
|
+
|
|
108
109
|
expect(headHandler).toHaveBeenCalledTimes(1)
|
|
109
110
|
expect(listHandler).toHaveBeenCalledTimes(2)
|
|
110
111
|
expect(deleteHandler).toHaveBeenCalledTimes(1)
|
package/source/providers/S3.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { Readable } from 'node:stream'
|
|
|
2
2
|
import { Blob } from 'node:buffer'
|
|
3
3
|
import { join } from 'node:path/posix'
|
|
4
4
|
import assert from 'node:assert'
|
|
5
|
+
import { posix } from 'node:path'
|
|
5
6
|
import { Upload } from '@aws-sdk/lib-storage'
|
|
6
7
|
import {
|
|
7
8
|
S3Client,
|
|
@@ -70,10 +71,12 @@ export class S3 extends Provider<S3Options> {
|
|
|
70
71
|
this.client.middlewareStack.add((next, _context) => async (args) => {
|
|
71
72
|
if ('Key' in args.input && typeof args.input.Key === 'string')
|
|
72
73
|
// removes leading slash
|
|
74
|
+
|
|
73
75
|
args.input.Key = args.input.Key.replace(/^\//, '')
|
|
74
76
|
|
|
75
77
|
if ('Prefix' in args.input && typeof args.input.Prefix === 'string')
|
|
76
78
|
// removes leading slash and ensures finishing slash
|
|
79
|
+
|
|
77
80
|
args.input.Prefix = args.input.Prefix.replace(/^\/|\/$/g, '') + '/'
|
|
78
81
|
|
|
79
82
|
return next(args)
|
|
@@ -100,11 +103,18 @@ export class S3 extends Provider<S3Options> {
|
|
|
100
103
|
? fileResponse.Body.stream()
|
|
101
104
|
: fileResponse.Body) as ReadableStream) // types mismatch between Node 20 and aws-sdk
|
|
102
105
|
} catch (err) {
|
|
103
|
-
if (err instanceof NotFound || err instanceof NoSuchKey)
|
|
104
|
-
|
|
106
|
+
if (err instanceof NotFound || err instanceof NoSuchKey)
|
|
107
|
+
return null
|
|
108
|
+
else
|
|
109
|
+
throw err
|
|
105
110
|
}
|
|
106
111
|
}
|
|
107
112
|
|
|
113
|
+
public async list (prefix: string): Promise<string[]> {
|
|
114
|
+
return (await this.listObjects(prefix))
|
|
115
|
+
.map(({ Key }) => posix.basename(Key!))
|
|
116
|
+
}
|
|
117
|
+
|
|
108
118
|
public async put (path: string, filename: string, stream: Readable): Promise<void> {
|
|
109
119
|
await new Upload({
|
|
110
120
|
client: this.client,
|
|
@@ -136,10 +146,7 @@ export class S3 extends Provider<S3Options> {
|
|
|
136
146
|
assert.ok(err instanceof NotFound || err instanceof NoSuchKey, err as Error)
|
|
137
147
|
}
|
|
138
148
|
|
|
139
|
-
const objectsToRemove: ObjectIdentifier[] =
|
|
140
|
-
|
|
141
|
-
for await (const page of paginateListObjectsV2({ client }, { Bucket, Prefix: Key }))
|
|
142
|
-
for (const { Key } of page.Contents ?? []) objectsToRemove.push({ Key })
|
|
149
|
+
const objectsToRemove: ObjectIdentifier[] = await this.listObjects(Key)
|
|
143
150
|
|
|
144
151
|
// Removing all objects in parallel in batches
|
|
145
152
|
await Promise.all((function * () {
|
|
@@ -163,4 +170,27 @@ export class S3 extends Provider<S3Options> {
|
|
|
163
170
|
|
|
164
171
|
await this.client.send(new DeleteObjectCommand({ Bucket: this.bucket, Key: from }))
|
|
165
172
|
}
|
|
173
|
+
|
|
174
|
+
public async moveDir (from: string, to: string): Promise<void> {
|
|
175
|
+
const objects: ObjectIdentifier[] = await this.listObjects(from)
|
|
176
|
+
|
|
177
|
+
await Promise.all(objects.map(async ({ Key }) => {
|
|
178
|
+
const ent = posix.basename(Key!)
|
|
179
|
+
const source = posix.join(from, ent)
|
|
180
|
+
const target = posix.join(to, ent)
|
|
181
|
+
|
|
182
|
+
return this.move(source, target)
|
|
183
|
+
}))
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
private async listObjects (Prefix: string): Promise<ObjectIdentifier[]> {
|
|
187
|
+
const { client, bucket: Bucket } = this
|
|
188
|
+
const objects: ObjectIdentifier[] = []
|
|
189
|
+
|
|
190
|
+
for await (const page of paginateListObjectsV2({ client }, { Bucket, Prefix }))
|
|
191
|
+
for (const { Key } of page.Contents ?? [])
|
|
192
|
+
objects.push({ Key })
|
|
193
|
+
|
|
194
|
+
return objects
|
|
195
|
+
}
|
|
166
196
|
}
|
|
@@ -69,6 +69,16 @@ describe.each(suites)('$provider', (suite) => {
|
|
|
69
69
|
expect(result).toBeNull()
|
|
70
70
|
})
|
|
71
71
|
|
|
72
|
+
it('should list files', async () => {
|
|
73
|
+
const stream = createReadStream('lenna.png')
|
|
74
|
+
|
|
75
|
+
await provider.put(dir, 'lenna.png', stream)
|
|
76
|
+
|
|
77
|
+
const result = await provider.list(dir)
|
|
78
|
+
|
|
79
|
+
expect(result).toContain('lenna.png')
|
|
80
|
+
})
|
|
81
|
+
|
|
72
82
|
describe('danger', () => {
|
|
73
83
|
/*
|
|
74
84
|
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
<svg width="160" height="160" viewBox="0 0 160 160" fill="none" xmlns="http://www.w3.org/2000/svg">
|
|
2
|
+
<rect width="160" height="160" rx="6" fill="white"/>
|
|
3
|
+
<path d="M83.8172 117.311H77.1506V75.6034C76.519 64.0461 71.2424 54.9753 62.4955 50.5487C55.8003 47.163 48.1655 47.3486 43.0436 51.0295C40.1688 53.0962 38.5807 55.5676 38.1898 58.5867C37.3617 64.9771 42.1922 72.4486 44.1231 74.7915L38.9812 79.0344C38.6222 78.5963 30.2131 68.2629 31.5783 57.7295C32.2107 52.8533 34.7593 48.7772 39.1526 45.6153C46.3817 40.42 56.4798 40.0343 65.5055 44.601C72.2494 48.0136 77.4112 53.7335 80.5225 60.9231C83.6337 53.7335 88.7954 48.0137 95.539 44.601C104.566 40.0343 114.664 40.42 121.892 45.6153C126.286 48.7772 128.834 52.8533 129.467 57.7295C130.832 68.2629 122.423 78.5963 122.064 79.0344L116.918 74.7963C116.988 74.7106 123.858 66.2201 122.853 58.5677C122.457 55.5582 120.87 53.0914 118.001 51.0295C112.879 47.3486 105.244 47.1582 98.5495 50.5487C89.5949 55.0803 84.2775 64.4793 83.8573 76.4304C83.8678 76.7653 83.8748 77.1016 83.8783 77.4392L83.8335 77.4396C83.8333 77.4617 83.833 77.4837 83.8328 77.5058L83.8172 77.5056V117.311Z" fill="#323232"/>
|
|
4
|
+
</svg>
|
|
Binary file
|
package/transpiled/Entry.d.ts
CHANGED
package/transpiled/Factory.js
CHANGED
|
@@ -5,6 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.Factory = void 0;
|
|
7
7
|
const node_assert_1 = __importDefault(require("node:assert"));
|
|
8
|
+
const openspan_1 = require("openspan");
|
|
8
9
|
const generic_1 = require("@toa.io/generic");
|
|
9
10
|
const providers_1 = require("./providers");
|
|
10
11
|
const Storage_1 = require("./Storage");
|
|
@@ -14,8 +15,8 @@ const Annotation_1 = require("./Annotation");
|
|
|
14
15
|
class Factory {
|
|
15
16
|
annotation;
|
|
16
17
|
constructor() {
|
|
17
|
-
const env = process.env.
|
|
18
|
-
node_assert_1.default.ok(env !== undefined,
|
|
18
|
+
const env = process.env[deployment_1.ENV_PREFIX];
|
|
19
|
+
node_assert_1.default.ok(env !== undefined, `${deployment_1.ENV_PREFIX} is not defined`);
|
|
19
20
|
this.annotation = (0, generic_1.decode)(env);
|
|
20
21
|
(0, Annotation_1.validateAnnotation)(this.annotation);
|
|
21
22
|
}
|
|
@@ -34,6 +35,7 @@ class Factory {
|
|
|
34
35
|
const Provider = providers_1.providers[providerId];
|
|
35
36
|
const secrets = this.resolveSecrets(name, Provider);
|
|
36
37
|
const provider = new Provider(options, secrets);
|
|
38
|
+
openspan_1.console.debug('Storage created', { name, provider: providerId, options, path: provider.path });
|
|
37
39
|
return new Storage_1.Storage(provider);
|
|
38
40
|
}
|
|
39
41
|
resolveSecrets(storageName, Class) {
|
|
@@ -41,7 +43,7 @@ class Factory {
|
|
|
41
43
|
return {};
|
|
42
44
|
const secrets = {};
|
|
43
45
|
for (const secret of Class.SECRETS) {
|
|
44
|
-
const variable = `${deployment_1.
|
|
46
|
+
const variable = `${deployment_1.ENV_PREFIX}_${storageName}_${secret.name}`.toUpperCase();
|
|
45
47
|
const value = process.env[variable];
|
|
46
48
|
node_assert_1.default.ok(secret.optional === true || value !== undefined, `'${variable}' is not defined`);
|
|
47
49
|
secrets[secret.name] = value;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Factory.js","sourceRoot":"","sources":["../source/Factory.ts"],"names":[],"mappings":";;;;;;AAAA,8DAAgC;AAChC,6CAAwC;AACxC,2CAAuC;AACvC,uCAAkD;AAClD,qCAAiC;AACjC,
|
|
1
|
+
{"version":3,"file":"Factory.js","sourceRoot":"","sources":["../source/Factory.ts"],"names":[],"mappings":";;;;;;AAAA,8DAAgC;AAChC,uCAAkC;AAClC,6CAAwC;AACxC,2CAAuC;AACvC,uCAAkD;AAClD,qCAAiC;AACjC,6CAAyC;AACzC,6CAAiD;AAKjD,MAAa,OAAO;IACD,UAAU,CAAY;IAEvC;QACE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,uBAAU,CAAC,CAAA;QAEnC,qBAAM,CAAC,EAAE,CAAC,GAAG,KAAK,SAAS,EAAE,GAAG,uBAAU,iBAAiB,CAAC,CAAA;QAE5D,IAAI,CAAC,UAAU,GAAG,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAA;QAE7B,IAAA,+BAAkB,EAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IACrC,CAAC;IAEM,MAAM;QACX,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,EAAE,CAAA;QAEtC,OAAO,IAAI,eAAM,CAAC,QAAQ,CAAC,CAAA;IAC7B,CAAC;IAEO,cAAc;QACpB,MAAM,QAAQ,GAAa,EAAE,CAAA;QAE7B,KAAK,MAAM,CAAC,IAAI,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC;YAC/D,QAAQ,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;QAExD,OAAO,QAAQ,CAAA;IACjB,CAAC;IAEO,aAAa,CAAE,IAAY,EAAE,WAAwB;QAC3D,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,OAAO,EAAE,GAAG,WAAW,CAAA;QACxD,MAAM,QAAQ,GAAwB,qBAAS,CAAC,UAAU,CAAC,CAAA;QAC3D,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;QACnD,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QAE/C,kBAAO,CAAC,KAAK,CAAC,iBAAiB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAA;QAE9F,OAAO,IAAI,iBAAO,CAAC,QAAQ,CAAC,CAAA;IAC9B,CAAC;IAEO,cAAc,CAAE,WAAmB,EACzC,KAA0B;QAC1B,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS;YAC7B,OAAO,EAAE,CAAA;QAEX,MAAM,OAAO,GAAuC,EAAE,CAAA;QAEtD,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,GAAG,uBAAU,IAAI,WAAW,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;YAC5E,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;YAEnC,qBAAM,CAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,EACvD,IAAI,QAAQ,kBAAkB,CAAC,CAAA;YAEjC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAA;QAC9B,CAAC;QAED,OAAO,OAAO,CAAA;IAChB,CAAC;CACF;AA1DD,0BA0DC"}
|
package/transpiled/Provider.d.ts
CHANGED
|
@@ -7,11 +7,14 @@ export interface ProviderSecret {
|
|
|
7
7
|
}
|
|
8
8
|
export declare abstract class Provider<Options = void> {
|
|
9
9
|
static readonly SECRETS: readonly ProviderSecret[];
|
|
10
|
+
readonly path: string | null;
|
|
10
11
|
constructor(_: Options, secrets?: ProviderSecrets);
|
|
11
12
|
abstract get(path: string): Promise<Readable | null>;
|
|
13
|
+
abstract list(path: string): Promise<string[]>;
|
|
12
14
|
abstract put(path: string, filename: string, stream: Readable): Promise<void>;
|
|
13
15
|
abstract delete(path: string): Promise<void>;
|
|
14
16
|
abstract move(from: string, to: string): Promise<void>;
|
|
17
|
+
abstract moveDir(from: string, to: string): Promise<void>;
|
|
15
18
|
}
|
|
16
19
|
export interface ProviderConstructor {
|
|
17
20
|
readonly SECRETS: readonly ProviderSecret[];
|
package/transpiled/Provider.js
CHANGED
|
@@ -27,6 +27,7 @@ exports.Provider = void 0;
|
|
|
27
27
|
const assert = __importStar(require("node:assert"));
|
|
28
28
|
class Provider {
|
|
29
29
|
static SECRETS = [];
|
|
30
|
+
path = null;
|
|
30
31
|
constructor(_, secrets) {
|
|
31
32
|
for (const { name, optional = false } of new.target.SECRETS)
|
|
32
33
|
assert.ok(optional || secrets?.[name] !== undefined, `Missing secret '${name}'`);
|