@toa.io/extensions.storages 1.0.0-alpha.7 → 1.0.0-alpha.78

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/package.json +8 -8
  2. package/readme.md +32 -22
  3. package/schemas/fs.cos.yaml +2 -0
  4. package/schemas/s3.cos.yaml +1 -0
  5. package/schemas/tmp.cos.yaml +0 -1
  6. package/source/Entry.ts +1 -0
  7. package/source/Factory.ts +7 -4
  8. package/source/Provider.ts +5 -0
  9. package/source/Scanner.ts +46 -17
  10. package/source/Storage.test.ts +126 -76
  11. package/source/Storage.ts +92 -91
  12. package/source/deployment.ts +38 -7
  13. package/source/providers/FileSystem.test.ts +6 -0
  14. package/source/providers/FileSystem.ts +15 -2
  15. package/source/providers/Memory.ts +21 -7
  16. package/source/providers/S3.test.ts +1 -0
  17. package/source/providers/S3.ts +36 -6
  18. package/source/providers/index.test.ts +10 -0
  19. package/source/test/arny.jpg +0 -0
  20. package/source/test/sample.avi +0 -0
  21. package/source/test/sample.svg +4 -0
  22. package/source/test/sample.wav +0 -0
  23. package/transpiled/Entry.d.ts +1 -0
  24. package/transpiled/Factory.js +5 -3
  25. package/transpiled/Factory.js.map +1 -1
  26. package/transpiled/Provider.d.ts +3 -0
  27. package/transpiled/Provider.js +1 -0
  28. package/transpiled/Provider.js.map +1 -1
  29. package/transpiled/Scanner.d.ts +3 -1
  30. package/transpiled/Scanner.js +36 -15
  31. package/transpiled/Scanner.js.map +1 -1
  32. package/transpiled/Storage.d.ts +6 -6
  33. package/transpiled/Storage.js +66 -78
  34. package/transpiled/Storage.js.map +1 -1
  35. package/transpiled/deployment.d.ts +1 -1
  36. package/transpiled/deployment.js +30 -7
  37. package/transpiled/deployment.js.map +1 -1
  38. package/transpiled/providers/FileSystem.d.ts +4 -1
  39. package/transpiled/providers/FileSystem.js +10 -1
  40. package/transpiled/providers/FileSystem.js.map +1 -1
  41. package/transpiled/providers/Memory.d.ts +2 -0
  42. package/transpiled/providers/Memory.js +13 -1
  43. package/transpiled/providers/Memory.js.map +1 -1
  44. package/transpiled/providers/S3.d.ts +3 -0
  45. package/transpiled/providers/S3.js +23 -4
  46. package/transpiled/providers/S3.js.map +1 -1
  47. package/transpiled/tsconfig.tsbuildinfo +1 -1
package/source/Storage.ts CHANGED
@@ -1,7 +1,8 @@
1
1
  import { Readable } from 'node:stream'
2
2
  import { posix } from 'node:path'
3
+ import { buffer } from 'node:stream/consumers'
3
4
  import { decode, encode } from 'msgpackr'
4
- import { buffer, newid } from '@toa.io/generic'
5
+ import { newid } from '@toa.io/generic'
5
6
  import { Err } from 'error-value'
6
7
  import { Scanner } from './Scanner'
7
8
  import type { ScanOptions } from './Scanner'
@@ -27,15 +28,33 @@ export class Storage {
27
28
 
28
29
  await this.persist(tempname, id)
29
30
 
30
- return await this.create(path, id, scanner.size, scanner.type, options?.meta)
31
+ const entry: Entry = {
32
+ id,
33
+ size: scanner.size,
34
+ type: scanner.type,
35
+ origin: options?.origin,
36
+ created: Date.now(),
37
+ variants: [],
38
+ meta: options?.meta ?? {}
39
+ }
40
+
41
+ return await this.create(path, entry)
31
42
  }
32
43
 
33
44
  public async get (path: string): Maybe<Entry> {
34
- const metapath = posix.join(ENTRIES, path, META)
35
- const result = await this.provider.get(metapath)
45
+ const paths = this.destruct(path)
46
+ const result = await this.provider.get(paths.metafile)
36
47
 
37
- if (result === null) return ERR_NOT_FOUND
38
- else return decode(await buffer(result))
48
+ if (result === null)
49
+ return ERR_NOT_FOUND
50
+ else
51
+ return decode(await buffer(result))
52
+ }
53
+
54
+ public async list (path: string): Promise<string[]> {
55
+ const dir = posix.join(ENTRIES_ROOT, path, ENTRIES_DIR)
56
+
57
+ return await this.provider.list(dir)
39
58
  }
40
59
 
41
60
  public async fetch (path: string): Maybe<Readable> {
@@ -50,12 +69,14 @@ export class Storage {
50
69
 
51
70
  const blob = variant === null
52
71
  ? posix.join(BLOBs, id)
53
- : posix.join(ENTRIES, rel, id, variant)
72
+ : posix.join(ENTRIES_ROOT, rel, id, variant)
54
73
 
55
74
  const stream = await this.provider.get(blob)
56
75
 
57
- if (stream === null) return ERR_NOT_FOUND
58
- else return stream
76
+ if (stream === null)
77
+ return ERR_NOT_FOUND
78
+ else
79
+ return stream
59
80
  }
60
81
 
61
82
  public async delete (path: string): Maybe<void> {
@@ -64,57 +85,38 @@ export class Storage {
64
85
  if (entry instanceof Error)
65
86
  return entry
66
87
 
67
- await this.conceal(path)
68
- await this.provider.delete(posix.join(ENTRIES, path))
69
- }
88
+ const paths = this.destruct(path)
70
89
 
71
- public async list (path: string): Promise<string[]> {
72
- const listfile = posix.join(ENTRIES, path, LIST)
73
- const stream = await this.provider.get(listfile)
74
-
75
- return stream === null ? [] : decode(await buffer(stream))
76
- }
77
-
78
- public async permute (path: string, ids: string[]): Maybe<void> {
79
- const unique = new Set(ids)
80
- const dir = posix.join(ENTRIES, path)
81
- const list = await this.list(path)
82
-
83
- if (list.length !== ids.length || unique.size !== ids.length)
84
- return ERR_PERMUTATION_MISMATCH
85
-
86
- for (const id of ids)
87
- if (!list.includes(id))
88
- return ERR_PERMUTATION_MISMATCH
89
-
90
- await this.provider.put(dir, LIST, Readable.from(encode(ids)))
90
+ await Promise.all([
91
+ this.provider.delete(paths.metafile),
92
+ this.provider.delete(paths.vardir)
93
+ ])
91
94
  }
92
95
 
93
- public async conceal (path: string): Maybe<void> {
94
- const { id, rel } = this.parse(path)
95
- const dir = posix.join(ENTRIES, rel)
96
- const list = await this.list(rel)
97
- const index = list.indexOf(id)
98
-
99
- if (index === -1)
100
- return ERR_NOT_FOUND
96
+ public async move (path: string, to: string): Maybe<void> {
97
+ const source = this.destruct(path)
98
+ const rel = to.startsWith('.')
99
+ const dir = to.endsWith('/')
101
100
 
102
- list.splice(index, 1)
101
+ if (rel)
102
+ to = posix.resolve(source.rel + '/', to)
103
103
 
104
- await this.provider.put(dir, LIST, Readable.from(encode(list)))
105
- }
104
+ if (dir)
105
+ to = posix.join(to, source.ent)
106
106
 
107
- public async reveal (path: string): Maybe<void> {
108
- const { id, rel } = this.parse(path)
107
+ const target = this.destruct(to)
109
108
 
110
- return await this.enroll(rel, id)
109
+ await Promise.all([
110
+ this.provider.move(source.metafile, target.metafile),
111
+ this.provider.moveDir(source.vardir, target.vardir)
112
+ ])
111
113
  }
112
114
 
113
115
  public async diversify (path: string, name: string, stream: Readable): Maybe<void> {
114
116
  const scanner = new Scanner()
115
117
  const pipe = stream.pipe(scanner)
116
118
 
117
- await this.provider.put(posix.join(ENTRIES, path), name, pipe)
119
+ await this.provider.put(posix.join(ENTRIES_ROOT, path), name, pipe)
118
120
 
119
121
  if (scanner.error !== null)
120
122
  return scanner.error
@@ -131,19 +133,30 @@ export class Storage {
131
133
  await this.save(path, entry)
132
134
  }
133
135
 
134
- public async annotate (path: string, key: string, value?: unknown): Maybe<void> {
136
+ public async annotate (path: string, key: string | Record<string, unknown>, value?: unknown): Maybe<void> {
135
137
  const entry = await this.get(path)
136
138
 
137
139
  if (entry instanceof Error)
138
140
  return entry
139
141
 
140
- // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
141
- if (value === undefined) delete entry.meta[key]
142
- else entry.meta[key] = value
142
+ const update = typeof key === 'string'
143
+ ? { [key]: value }
144
+ : key
145
+
146
+ Object.assign(entry.meta, update)
147
+
148
+ // filter undefined values
149
+ for (const key of Object.keys(entry.meta))
150
+ if (entry.meta[key] === undefined)
151
+ delete entry.meta[key]
143
152
 
144
153
  await this.save(path, entry)
145
154
  }
146
155
 
156
+ public path (): string | null {
157
+ return this.provider.path
158
+ }
159
+
147
160
  private async transit (stream: Readable): Promise<string> {
148
161
  const tempname = newid()
149
162
 
@@ -161,52 +174,21 @@ export class Storage {
161
174
 
162
175
  // eslint-disable-next-line max-params
163
176
  private async create
164
- (path: string, id: string, size: number, type: string, meta: Meta = {}): Promise<Entry> {
165
- const entry: Entry = {
166
- id,
167
- size,
168
- type,
169
- created: Date.now(),
170
- variants: [],
171
- meta
172
- }
173
-
177
+ (path: string, entry: Entry): Promise<Entry> {
174
178
  const metafile = posix.join(path, entry.id)
175
179
  const existing = await this.get(metafile)
176
180
 
177
181
  if (existing instanceof Error)
178
182
  await this.save(metafile, entry)
179
183
 
180
- await this.enroll(path, id, true)
181
-
182
184
  return entry
183
185
  }
184
186
 
185
- private async save (rel: string, entry: Entry): Promise<void> {
186
- const buffer = encode(entry)
187
- const stream = Readable.from(buffer)
188
-
189
- await this.provider.put(posix.join(ENTRIES, rel), META, stream)
190
- }
191
-
192
- private async enroll (path: string, id: string, addition: boolean = false): Maybe<void> {
193
- const dir = posix.join(ENTRIES, path)
194
- const list = await this.list(path)
195
- const index = list.indexOf(id)
187
+ private async save (path: string, entry: Entry): Promise<void> {
188
+ const paths = this.destruct(path)
189
+ const stream = Readable.from(encode(entry))
196
190
 
197
- if (index !== -1)
198
- if (addition) list.splice(index, 1)
199
- else return
200
- else if (!addition) {
201
- const entry = await this.get(posix.join(path, id))
202
-
203
- if (entry instanceof Error)
204
- return entry
205
- }
206
-
207
- list.push(id)
208
-
209
- await this.provider.put(dir, LIST, Readable.from(encode(list)))
191
+ await this.provider.put(paths.metadir, paths.ent, stream)
210
192
  }
211
193
 
212
194
  private parse (path: string): Path {
@@ -217,16 +199,25 @@ export class Storage {
217
199
 
218
200
  return { rel, id, variant }
219
201
  }
202
+
203
+ private destruct (path: string): Paths {
204
+ const rel = posix.dirname(path)
205
+ const dir = posix.join(ENTRIES_ROOT, rel)
206
+ const ent = posix.basename(path)
207
+ const metadir = posix.join(dir, ENTRIES_DIR)
208
+ const metafile = posix.join(metadir, ent)
209
+ const vardir = posix.join(dir, ent)
210
+
211
+ return { rel, dir, ent, metadir, metafile, vardir }
212
+ }
220
213
  }
221
214
 
222
215
  const ERR_NOT_FOUND = Err('NOT_FOUND')
223
- const ERR_PERMUTATION_MISMATCH = Err('PERMUTATION_MISMATCH')
224
216
 
225
217
  const TEMP = '/temp'
226
218
  const BLOBs = '/blobs'
227
- const ENTRIES = '/entries'
228
- const LIST = '.list'
229
- const META = '.meta'
219
+ const ENTRIES_ROOT = '/entries'
220
+ const ENTRIES_DIR = '.meta'
230
221
 
231
222
  type Maybe<T> = Promise<T | Error>
232
223
 
@@ -236,9 +227,19 @@ interface Path {
236
227
  variant: string | null
237
228
  }
238
229
 
230
+ interface Paths {
231
+ rel: string
232
+ dir: string
233
+ ent: string
234
+ metadir: string
235
+ metafile: string
236
+ vardir: string
237
+ }
238
+
239
239
  type Meta = Record<string, string>
240
240
 
241
241
  interface Options extends ScanOptions {
242
+ origin?: string
242
243
  meta?: Meta
243
244
  }
244
245
 
@@ -3,21 +3,25 @@ import { encode } from '@toa.io/generic'
3
3
  import { providers } from './providers'
4
4
  import { validateAnnotation } from './Annotation'
5
5
  import type { Annotation } from './Annotation'
6
- import type { Dependency, Variable } from '@toa.io/operations'
6
+ import type { Dependency, Variable, Mounts } from '@toa.io/operations'
7
7
  import type { context } from '@toa.io/norm'
8
8
 
9
- export const SERIALIZATION_PREFIX = 'TOA_STORAGES'
9
+ export const ENV_PREFIX = 'TOA_STORAGES'
10
10
 
11
11
  export function deployment (instances: Instance[], annotation: unknown): Dependency {
12
12
  validate(instances, annotation)
13
13
 
14
14
  const value = encode(annotation)
15
- const pointer: Variable = { name: SERIALIZATION_PREFIX, value }
15
+ const pointer: Variable = { name: ENV_PREFIX, value }
16
16
  const secrets = getSecrets(annotation)
17
+ const mounts = getMounts(instances, annotation)
17
18
 
18
- return {
19
- variables: { global: [pointer, ...secrets] }
20
- }
19
+ const dependency: Dependency = { variables: { global: [pointer, ...secrets] } }
20
+
21
+ if (mounts !== null)
22
+ dependency.mounts = mounts
23
+
24
+ return dependency
21
25
  }
22
26
 
23
27
  function validate (instances: Instance[], annotation: unknown): asserts annotation is Annotation {
@@ -42,7 +46,7 @@ function getSecrets (annotation: Annotation): Variable[] {
42
46
 
43
47
  for (const secret of Provider.SECRETS)
44
48
  secrets.push({
45
- name: `${SERIALIZATION_PREFIX}_${name}_${secret.name}`.toUpperCase(),
49
+ name: `${ENV_PREFIX}_${name}_${secret.name}`.toUpperCase(),
46
50
  secret: {
47
51
  name: `toa-storages-${name}`,
48
52
  key: secret.name,
@@ -54,4 +58,31 @@ function getSecrets (annotation: Annotation): Variable[] {
54
58
  return secrets
55
59
  }
56
60
 
61
+ function getMounts (instances: Instance[], annotation: Annotation): Mounts | null {
62
+ let mounts: Mounts | null = null
63
+
64
+ for (const { locator, manifest } of instances)
65
+ for (const name of manifest) {
66
+ const declaration = annotation[name]
67
+
68
+ // eslint-disable-next-line max-depth
69
+ if (declaration.provider !== 'fs')
70
+ continue
71
+
72
+ // eslint-disable-next-line max-depth
73
+ if (declaration.claim !== undefined) {
74
+ mounts ??= {}
75
+ mounts[locator.label] ??= []
76
+
77
+ mounts[locator.label].push({
78
+ name,
79
+ path: declaration.path,
80
+ claim: declaration.claim
81
+ })
82
+ }
83
+ }
84
+
85
+ return mounts
86
+ }
87
+
57
88
  export type Instance = context.Dependency<string[]>
@@ -3,3 +3,9 @@ import { FileSystem } from './FileSystem'
3
3
  it('should be ok', async () => {
4
4
  expect(() => new FileSystem({ path: 'path' })).not.toThrow()
5
5
  })
6
+
7
+ it('should expose path', () => {
8
+ const fs = new FileSystem({ path: '/tmp/foo' })
9
+
10
+ expect(fs.path).toBe('/tmp/foo')
11
+ })
@@ -6,10 +6,11 @@ import type { ProviderSecrets } from '../Provider'
6
6
 
7
7
  export interface FileSystemOptions {
8
8
  path: string
9
+ claim?: string
9
10
  }
10
11
 
11
12
  export class FileSystem extends Provider<FileSystemOptions> {
12
- protected readonly path: string
13
+ public override readonly path: string
13
14
 
14
15
  public constructor (options: FileSystemOptions, secrets?: ProviderSecrets) {
15
16
  super(options, secrets)
@@ -29,11 +30,19 @@ export class FileSystem extends Provider<FileSystemOptions> {
29
30
  }
30
31
  }
31
32
 
33
+ public async list (path: string): Promise<string[]> {
34
+ const dir = join(this.path, path)
35
+
36
+ return (await fs.readdir(dir, { withFileTypes: true }))
37
+ .filter((dirent) => dirent.isFile())
38
+ .map((dirent) => dirent.name)
39
+ }
40
+
32
41
  public async put (rel: string, filename: string, stream: Readable): Promise<void> {
33
42
  const dir = join(this.path, rel)
34
43
  const path = join(dir, filename)
35
44
 
36
- await fs.mkdir(dirname(path), { recursive: true })
45
+ await fs.mkdir(dir, { recursive: true })
37
46
  await fs.writeFile(path, stream)
38
47
  }
39
48
 
@@ -48,4 +57,8 @@ export class FileSystem extends Provider<FileSystemOptions> {
48
57
  await fs.mkdir(dirname(to), { recursive: true })
49
58
  await fs.rename(from, to)
50
59
  }
60
+
61
+ public async moveDir (from: string, to: string): Promise<void> {
62
+ await this.move(from, to).catch(() => null)
63
+ }
51
64
  }
@@ -1,5 +1,5 @@
1
1
  import { Readable } from 'node:stream'
2
- import { join } from 'node:path'
2
+ import { join, posix } from 'node:path'
3
3
  import { buffer } from 'node:stream/consumers'
4
4
  import * as assert from 'node:assert'
5
5
 
@@ -11,7 +11,7 @@ import { Provider } from '../Provider'
11
11
  export class InMemory extends Provider {
12
12
  private readonly storage = new Map<string, Buffer>()
13
13
 
14
- public override async get (path: string): Promise<Readable | null> {
14
+ public async get (path: string): Promise<Readable | null> {
15
15
  const data = this.storage.get(path)
16
16
 
17
17
  if (data === undefined) return null
@@ -19,18 +19,22 @@ export class InMemory extends Provider {
19
19
  return Readable.from(data)
20
20
  }
21
21
 
22
- public override async put (path: string, filename: string, stream: Readable): Promise<void> {
22
+ public async list (path: string): Promise<string[]> {
23
+ return Array.from(this.storage.keys())
24
+ .filter((f) => posix.dirname(f) === path)
25
+ .map((f) => posix.basename(f))
26
+ }
27
+
28
+ public async put (path: string, filename: string, stream: Readable): Promise<void> {
23
29
  this.storage.set(join(path, filename), await buffer(stream))
24
30
  }
25
31
 
26
- public override async delete (path: string): Promise<void> {
32
+ public async delete (path: string): Promise<void> {
27
33
  for (const f of this.storage.keys())
28
34
  if (f.startsWith(path)) this.storage.delete(f)
29
35
  }
30
36
 
31
- public override async move (from: string, to: string): Promise<void> {
32
- assert.notEqual(from, to, 'Source and destination are the same')
33
-
37
+ public async move (from: string, to: string): Promise<void> {
34
38
  const buf = this.storage.get(from)
35
39
 
36
40
  assert.ok(buf !== undefined, `File not found: ${from}`)
@@ -38,4 +42,14 @@ export class InMemory extends Provider {
38
42
  this.storage.set(to, buf)
39
43
  this.storage.delete(from)
40
44
  }
45
+
46
+ public async moveDir (from: string, to: string): Promise<void> {
47
+ for (const f of this.storage.keys())
48
+ if (f.startsWith(from)) {
49
+ const toPath = to + f.slice(from.length)
50
+
51
+ this.storage.set(toPath, this.storage.get(f)!)
52
+ this.storage.delete(f)
53
+ }
54
+ }
41
55
  }
@@ -105,6 +105,7 @@ describe('S3 storage provider', () => {
105
105
  deleteHandler))
106
106
 
107
107
  await provider.delete('/some/absolute/path_to_remove')
108
+
108
109
  expect(headHandler).toHaveBeenCalledTimes(1)
109
110
  expect(listHandler).toHaveBeenCalledTimes(2)
110
111
  expect(deleteHandler).toHaveBeenCalledTimes(1)
@@ -2,6 +2,7 @@ import { Readable } from 'node:stream'
2
2
  import { Blob } from 'node:buffer'
3
3
  import { join } from 'node:path/posix'
4
4
  import assert from 'node:assert'
5
+ import { posix } from 'node:path'
5
6
  import { Upload } from '@aws-sdk/lib-storage'
6
7
  import {
7
8
  S3Client,
@@ -70,10 +71,12 @@ export class S3 extends Provider<S3Options> {
70
71
  this.client.middlewareStack.add((next, _context) => async (args) => {
71
72
  if ('Key' in args.input && typeof args.input.Key === 'string')
72
73
  // removes leading slash
74
+
73
75
  args.input.Key = args.input.Key.replace(/^\//, '')
74
76
 
75
77
  if ('Prefix' in args.input && typeof args.input.Prefix === 'string')
76
78
  // removes leading slash and ensures finishing slash
79
+
77
80
  args.input.Prefix = args.input.Prefix.replace(/^\/|\/$/g, '') + '/'
78
81
 
79
82
  return next(args)
@@ -100,11 +103,18 @@ export class S3 extends Provider<S3Options> {
100
103
  ? fileResponse.Body.stream()
101
104
  : fileResponse.Body) as ReadableStream) // types mismatch between Node 20 and aws-sdk
102
105
  } catch (err) {
103
- if (err instanceof NotFound || err instanceof NoSuchKey) return null
104
- else throw err
106
+ if (err instanceof NotFound || err instanceof NoSuchKey)
107
+ return null
108
+ else
109
+ throw err
105
110
  }
106
111
  }
107
112
 
113
+ public async list (prefix: string): Promise<string[]> {
114
+ return (await this.listObjects(prefix))
115
+ .map(({ Key }) => posix.basename(Key!))
116
+ }
117
+
108
118
  public async put (path: string, filename: string, stream: Readable): Promise<void> {
109
119
  await new Upload({
110
120
  client: this.client,
@@ -136,10 +146,7 @@ export class S3 extends Provider<S3Options> {
136
146
  assert.ok(err instanceof NotFound || err instanceof NoSuchKey, err as Error)
137
147
  }
138
148
 
139
- const objectsToRemove: ObjectIdentifier[] = []
140
-
141
- for await (const page of paginateListObjectsV2({ client }, { Bucket, Prefix: Key }))
142
- for (const { Key } of page.Contents ?? []) objectsToRemove.push({ Key })
149
+ const objectsToRemove: ObjectIdentifier[] = await this.listObjects(Key)
143
150
 
144
151
  // Removing all objects in parallel in batches
145
152
  await Promise.all((function * () {
@@ -163,4 +170,27 @@ export class S3 extends Provider<S3Options> {
163
170
 
164
171
  await this.client.send(new DeleteObjectCommand({ Bucket: this.bucket, Key: from }))
165
172
  }
173
+
174
+ public async moveDir (from: string, to: string): Promise<void> {
175
+ const objects: ObjectIdentifier[] = await this.listObjects(from)
176
+
177
+ await Promise.all(objects.map(async ({ Key }) => {
178
+ const ent = posix.basename(Key!)
179
+ const source = posix.join(from, ent)
180
+ const target = posix.join(to, ent)
181
+
182
+ return this.move(source, target)
183
+ }))
184
+ }
185
+
186
+ private async listObjects (Prefix: string): Promise<ObjectIdentifier[]> {
187
+ const { client, bucket: Bucket } = this
188
+ const objects: ObjectIdentifier[] = []
189
+
190
+ for await (const page of paginateListObjectsV2({ client }, { Bucket, Prefix }))
191
+ for (const { Key } of page.Contents ?? [])
192
+ objects.push({ Key })
193
+
194
+ return objects
195
+ }
166
196
  }
@@ -69,6 +69,16 @@ describe.each(suites)('$provider', (suite) => {
69
69
  expect(result).toBeNull()
70
70
  })
71
71
 
72
+ it('should list files', async () => {
73
+ const stream = createReadStream('lenna.png')
74
+
75
+ await provider.put(dir, 'lenna.png', stream)
76
+
77
+ const result = await provider.list(dir)
78
+
79
+ expect(result).toContain('lenna.png')
80
+ })
81
+
72
82
  describe('danger', () => {
73
83
  /*
74
84
 
Binary file
Binary file
@@ -0,0 +1,4 @@
1
+ <svg width="160" height="160" viewBox="0 0 160 160" fill="none" xmlns="http://www.w3.org/2000/svg">
2
+ <rect width="160" height="160" rx="6" fill="white"/>
3
+ <path d="M83.8172 117.311H77.1506V75.6034C76.519 64.0461 71.2424 54.9753 62.4955 50.5487C55.8003 47.163 48.1655 47.3486 43.0436 51.0295C40.1688 53.0962 38.5807 55.5676 38.1898 58.5867C37.3617 64.9771 42.1922 72.4486 44.1231 74.7915L38.9812 79.0344C38.6222 78.5963 30.2131 68.2629 31.5783 57.7295C32.2107 52.8533 34.7593 48.7772 39.1526 45.6153C46.3817 40.42 56.4798 40.0343 65.5055 44.601C72.2494 48.0136 77.4112 53.7335 80.5225 60.9231C83.6337 53.7335 88.7954 48.0137 95.539 44.601C104.566 40.0343 114.664 40.42 121.892 45.6153C126.286 48.7772 128.834 52.8533 129.467 57.7295C130.832 68.2629 122.423 78.5963 122.064 79.0344L116.918 74.7963C116.988 74.7106 123.858 66.2201 122.853 58.5677C122.457 55.5582 120.87 53.0914 118.001 51.0295C112.879 47.3486 105.244 47.1582 98.5495 50.5487C89.5949 55.0803 84.2775 64.4793 83.8573 76.4304C83.8678 76.7653 83.8748 77.1016 83.8783 77.4392L83.8335 77.4396C83.8333 77.4617 83.833 77.4837 83.8328 77.5058L83.8172 77.5056V117.311Z" fill="#323232"/>
4
+ </svg>
Binary file
@@ -3,6 +3,7 @@ export interface Entry {
3
3
  size: number;
4
4
  type: string;
5
5
  created: number;
6
+ origin?: string;
6
7
  variants: Variant[];
7
8
  meta: Record<string, unknown>;
8
9
  }
@@ -5,6 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
6
  exports.Factory = void 0;
7
7
  const node_assert_1 = __importDefault(require("node:assert"));
8
+ const openspan_1 = require("openspan");
8
9
  const generic_1 = require("@toa.io/generic");
9
10
  const providers_1 = require("./providers");
10
11
  const Storage_1 = require("./Storage");
@@ -14,8 +15,8 @@ const Annotation_1 = require("./Annotation");
14
15
  class Factory {
15
16
  annotation;
16
17
  constructor() {
17
- const env = process.env.TOA_STORAGES;
18
- node_assert_1.default.ok(env !== undefined, 'TOA_STORAGES is not defined');
18
+ const env = process.env[deployment_1.ENV_PREFIX];
19
+ node_assert_1.default.ok(env !== undefined, `${deployment_1.ENV_PREFIX} is not defined`);
19
20
  this.annotation = (0, generic_1.decode)(env);
20
21
  (0, Annotation_1.validateAnnotation)(this.annotation);
21
22
  }
@@ -34,6 +35,7 @@ class Factory {
34
35
  const Provider = providers_1.providers[providerId];
35
36
  const secrets = this.resolveSecrets(name, Provider);
36
37
  const provider = new Provider(options, secrets);
38
+ openspan_1.console.debug('Storage created', { name, provider: providerId, options, path: provider.path });
37
39
  return new Storage_1.Storage(provider);
38
40
  }
39
41
  resolveSecrets(storageName, Class) {
@@ -41,7 +43,7 @@ class Factory {
41
43
  return {};
42
44
  const secrets = {};
43
45
  for (const secret of Class.SECRETS) {
44
- const variable = `${deployment_1.SERIALIZATION_PREFIX}_${storageName}_${secret.name}`.toUpperCase();
46
+ const variable = `${deployment_1.ENV_PREFIX}_${storageName}_${secret.name}`.toUpperCase();
45
47
  const value = process.env[variable];
46
48
  node_assert_1.default.ok(secret.optional === true || value !== undefined, `'${variable}' is not defined`);
47
49
  secrets[secret.name] = value;
@@ -1 +1 @@
1
- {"version":3,"file":"Factory.js","sourceRoot":"","sources":["../source/Factory.ts"],"names":[],"mappings":";;;;;;AAAA,8DAAgC;AAChC,6CAAwC;AACxC,2CAAuC;AACvC,uCAAkD;AAClD,qCAAiC;AACjC,6CAAmD;AACnD,6CAAiD;AAKjD,MAAa,OAAO;IACD,UAAU,CAAY;IAEvC;QACE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAA;QAEpC,qBAAM,CAAC,EAAE,CAAC,GAAG,KAAK,SAAS,EAAE,6BAA6B,CAAC,CAAA;QAE3D,IAAI,CAAC,UAAU,GAAG,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAA;QAE7B,IAAA,+BAAkB,EAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IACrC,CAAC;IAEM,MAAM;QACX,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,EAAE,CAAA;QAEtC,OAAO,IAAI,eAAM,CAAC,QAAQ,CAAC,CAAA;IAC7B,CAAC;IAEO,cAAc;QACpB,MAAM,QAAQ,GAAa,EAAE,CAAA;QAE7B,KAAK,MAAM,CAAC,IAAI,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC;YAC/D,QAAQ,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;QAExD,OAAO,QAAQ,CAAA;IACjB,CAAC;IAEO,aAAa,CAAE,IAAY,EAAE,WAAwB;QAC3D,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,OAAO,EAAE,GAAG,WAAW,CAAA;QACxD,MAAM,QAAQ,GAAwB,qBAAS,CAAC,UAAU,CAAC,CAAA;QAC3D,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;QACnD,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QAE/C,OAAO,IAAI,iBAAO,CAAC,QAAQ,CAAC,CAAA;IAC9B,CAAC;IAEO,cAAc,CAAE,WAAmB,EACzC,KAA0B;QAC1B,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS;YAC7B,OAAO,EAAE,CAAA;QAEX,MAAM,OAAO,GAAuC,EAAE,CAAA;QAEtD,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,GAAG,iCAAoB,IAAI,WAAW,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;YACtF,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;YAEnC,qBAAM,CAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,EACvD,IAAI,QAAQ,kBAAkB,CAAC,CAAA;YAEjC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAA;QAC9B,CAAC;QAED,OAAO,OAAO,CAAA;IAChB,CAAC;CACF;AAxDD,0BAwDC"}
1
+ {"version":3,"file":"Factory.js","sourceRoot":"","sources":["../source/Factory.ts"],"names":[],"mappings":";;;;;;AAAA,8DAAgC;AAChC,uCAAkC;AAClC,6CAAwC;AACxC,2CAAuC;AACvC,uCAAkD;AAClD,qCAAiC;AACjC,6CAAyC;AACzC,6CAAiD;AAKjD,MAAa,OAAO;IACD,UAAU,CAAY;IAEvC;QACE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,uBAAU,CAAC,CAAA;QAEnC,qBAAM,CAAC,EAAE,CAAC,GAAG,KAAK,SAAS,EAAE,GAAG,uBAAU,iBAAiB,CAAC,CAAA;QAE5D,IAAI,CAAC,UAAU,GAAG,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAA;QAE7B,IAAA,+BAAkB,EAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IACrC,CAAC;IAEM,MAAM;QACX,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,EAAE,CAAA;QAEtC,OAAO,IAAI,eAAM,CAAC,QAAQ,CAAC,CAAA;IAC7B,CAAC;IAEO,cAAc;QACpB,MAAM,QAAQ,GAAa,EAAE,CAAA;QAE7B,KAAK,MAAM,CAAC,IAAI,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC;YAC/D,QAAQ,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;QAExD,OAAO,QAAQ,CAAA;IACjB,CAAC;IAEO,aAAa,CAAE,IAAY,EAAE,WAAwB;QAC3D,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,OAAO,EAAE,GAAG,WAAW,CAAA;QACxD,MAAM,QAAQ,GAAwB,qBAAS,CAAC,UAAU,CAAC,CAAA;QAC3D,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;QACnD,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QAE/C,kBAAO,CAAC,KAAK,CAAC,iBAAiB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAA;QAE9F,OAAO,IAAI,iBAAO,CAAC,QAAQ,CAAC,CAAA;IAC9B,CAAC;IAEO,cAAc,CAAE,WAAmB,EACzC,KAA0B;QAC1B,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS;YAC7B,OAAO,EAAE,CAAA;QAEX,MAAM,OAAO,GAAuC,EAAE,CAAA;QAEtD,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,GAAG,uBAAU,IAAI,WAAW,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;YAC5E,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;YAEnC,qBAAM,CAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,EACvD,IAAI,QAAQ,kBAAkB,CAAC,CAAA;YAEjC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAA;QAC9B,CAAC;QAED,OAAO,OAAO,CAAA;IAChB,CAAC;CACF;AA1DD,0BA0DC"}
@@ -7,11 +7,14 @@ export interface ProviderSecret {
7
7
  }
8
8
  export declare abstract class Provider<Options = void> {
9
9
  static readonly SECRETS: readonly ProviderSecret[];
10
+ readonly path: string | null;
10
11
  constructor(_: Options, secrets?: ProviderSecrets);
11
12
  abstract get(path: string): Promise<Readable | null>;
13
+ abstract list(path: string): Promise<string[]>;
12
14
  abstract put(path: string, filename: string, stream: Readable): Promise<void>;
13
15
  abstract delete(path: string): Promise<void>;
14
16
  abstract move(from: string, to: string): Promise<void>;
17
+ abstract moveDir(from: string, to: string): Promise<void>;
15
18
  }
16
19
  export interface ProviderConstructor {
17
20
  readonly SECRETS: readonly ProviderSecret[];
@@ -27,6 +27,7 @@ exports.Provider = void 0;
27
27
  const assert = __importStar(require("node:assert"));
28
28
  class Provider {
29
29
  static SECRETS = [];
30
+ path = null;
30
31
  constructor(_, secrets) {
31
32
  for (const { name, optional = false } of new.target.SECRETS)
32
33
  assert.ok(optional || secrets?.[name] !== undefined, `Missing secret '${name}'`);