@3sln/trove 0.0.5 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/package.json +1 -1
- package/packages/core/src/collections/index.js +146 -1
- package/packages/core/src/encryption/envelope.js +444 -0
- package/packages/core/src/encryption/exposure.js +101 -0
- package/packages/core/src/encryption/keys.js +88 -0
- package/packages/core/src/encryption/policy.js +112 -0
- package/packages/core/src/encryption/rotation.js +313 -0
- package/packages/core/src/index.js +17 -1
- package/packages/core/src/links.js +85 -0
- package/packages/core/src/metadata/memory.js +3 -1
- package/packages/core/src/metadata/sqlite.js +27 -7
- package/packages/core/src/scan.js +35 -1
- package/packages/core/src/storage/cost.js +228 -0
- package/packages/core/src/storage/drivers.js +13 -4
- package/packages/core/src/storage/registry.js +37 -4
- package/packages/core/src/uploads.js +197 -15
- package/packages/core/src/vfs.js +148 -9
- package/packages/server/src/engine/providers/core.js +38 -3
- package/packages/server/src/index.js +8 -0
- package/packages/server/src/routes.js +29 -1
- package/packages/web/dist/assets/main-y778bpte.js +356 -0
- package/packages/web/dist/assets/{main-f0f2tfhp.js.map → main-y778bpte.js.map} +10 -8
- package/packages/web/dist/assets/styles-nfy8t3n1.css +1 -0
- package/packages/web/dist/index.html +9 -3
- package/packages/web/dist/sw.js +1 -1
- package/packages/web/src/bl/actions.js +20 -4
- package/packages/web/src/bl/services.js +64 -4
- package/packages/web/src/platform/api.js +134 -11
- package/packages/web/src/styles.css +3 -0
- package/packages/web/src/ui/components/overlays.js +13 -0
- package/packages/web/dist/assets/main-f0f2tfhp.js +0 -356
- package/packages/web/dist/assets/styles-d3cyysgp.css +0 -1
|
@@ -139,3 +139,88 @@ export function troveUrisFor(node) {
|
|
|
139
139
|
if (node?.collectionId && node?.id) out.push(troveUri(node, 'id'));
|
|
140
140
|
return out;
|
|
141
141
|
}
|
|
142
|
+
|
|
143
|
+
// --- shareable web links -------------------------------------------------------
|
|
144
|
+
//
|
|
145
|
+
// `trove:` addresses an item INSIDE the drive: it is what one document writes to link
|
|
146
|
+
// another, and it means nothing to a browser. A share link is the other half — a URL you
|
|
147
|
+
// can paste into a message so that someone else's browser opens the same item.
|
|
148
|
+
//
|
|
149
|
+
// They are deliberately the same addressing, not two competing schemes. A share link is a
|
|
150
|
+
// `trove:` URI wearing an http(s) coat: same collection, same explicit `name`-or-`id`
|
|
151
|
+
// selector, same refusal to infer which one you meant. So anything that can already
|
|
152
|
+
// resolve a `trove:` URI can resolve a share link by parsing it back, and a rename breaks
|
|
153
|
+
// both in the same visible way rather than one silently retargeting.
|
|
154
|
+
//
|
|
155
|
+
// The path form — /c/<collection>/i/<selector> — rather than the query form
|
|
156
|
+
// `?coll=&item=`. Both put the ids in the URL and therefore in server logs and browser
|
|
157
|
+
// history, so that is not the difference; the path reads as a location, survives being
|
|
158
|
+
// truncated in a chat client more gracefully, and leaves the query string free for the
|
|
159
|
+
// things that genuinely are parameters.
|
|
160
|
+
//
|
|
161
|
+
// Nothing secret ever rides in one. An encrypted collection's key is not in the link and
|
|
162
|
+
// must not be: a link is pasted into chats, logged by proxies, and kept in history
|
|
163
|
+
// forever. The recipient gets the item because they are allowed the collection, which is
|
|
164
|
+
// the same rule as everywhere else here.
|
|
165
|
+
|
|
166
|
+
/** The path a share link uses. Exported so a client router and the server agree. */
|
|
167
|
+
export const SHARE_PATH = '/c';
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* A URL that opens this item in a browser.
|
|
171
|
+
*
|
|
172
|
+
* @param {object} node
|
|
173
|
+
* @param {string} [origin] where the drive is served; omitted gives a root-relative link
|
|
174
|
+
* @param {'name'|'id'} [by] `name` reads better and breaks on rename; `id` is the reverse
|
|
175
|
+
*/
|
|
176
|
+
export function shareUrl(node, origin = '', by = 'name') {
|
|
177
|
+
if (!node?.collectionId) throw TroveError.invalid('An item needs a collection to be linked to');
|
|
178
|
+
const selector = by === 'id'
|
|
179
|
+
? `id:${node.id}`
|
|
180
|
+
: encodeURIComponent(node.name);
|
|
181
|
+
if (!selector || (by === 'id' && !node.id)) throw TroveError.invalid('Nothing to link to');
|
|
182
|
+
const path = `${SHARE_PATH}/${encodeURIComponent(node.collectionId)}/i/${selector}`;
|
|
183
|
+
return origin ? `${String(origin).replace(/\/$/, '')}${path}` : path;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Read a share link back, from a full URL or just a path.
|
|
188
|
+
*
|
|
189
|
+
* Returns the same shape `parseTroveUri` does, so a caller resolves either without caring
|
|
190
|
+
* which it was handed. Null rather than a throw: this parses whatever was in the address
|
|
191
|
+
* bar, and a URL that is not a share link is an ordinary page, not an error.
|
|
192
|
+
*/
|
|
193
|
+
export function parseShareUrl(url) {
|
|
194
|
+
let path;
|
|
195
|
+
try {
|
|
196
|
+
path = url.startsWith('/') ? url : new URL(url).pathname;
|
|
197
|
+
} catch {
|
|
198
|
+
return null;
|
|
199
|
+
}
|
|
200
|
+
const m = /^\/c\/([^/]+)\/i\/(.+)$/.exec(path);
|
|
201
|
+
if (!m) return null;
|
|
202
|
+
let collection;
|
|
203
|
+
let raw;
|
|
204
|
+
try {
|
|
205
|
+
collection = decodeURIComponent(m[1]);
|
|
206
|
+
raw = decodeURIComponent(m[2]);
|
|
207
|
+
} catch {
|
|
208
|
+
return null; // a malformed escape is not a link to anything
|
|
209
|
+
}
|
|
210
|
+
if (!COLLECTION_RE.test(collection)) return null;
|
|
211
|
+
// `id:` is the explicit selector, mirroring `?id=` — a name is never guessed at just
|
|
212
|
+
// because it happens to look like an id.
|
|
213
|
+
const byId = raw.startsWith('id:');
|
|
214
|
+
const value = byId ? raw.slice(3) : raw;
|
|
215
|
+
if (!value || value.length > MAX_SELECTOR) return null;
|
|
216
|
+
return { collection, by: byId ? 'id' : 'name', value };
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/** The `trove:` URI a share link denotes — the two are the same address. */
|
|
220
|
+
export function troveUriFromShareUrl(url) {
|
|
221
|
+
const parsed = parseShareUrl(url);
|
|
222
|
+
if (!parsed) return null;
|
|
223
|
+
return parsed.by === 'id'
|
|
224
|
+
? `${TROVE_SCHEME}${parsed.collection}?id=${encodeURIComponent(parsed.value)}`
|
|
225
|
+
: `${TROVE_SCHEME}${parsed.collection}?name=${encodeURIComponent(parsed.value)}`;
|
|
226
|
+
}
|
|
@@ -90,6 +90,8 @@ export class MemoryStore extends MetadataStore {
|
|
|
90
90
|
size: node.size ?? 0, contentType: node.contentType ?? null,
|
|
91
91
|
storageKey: node.storageKey ?? null, etag: node.etag ?? null,
|
|
92
92
|
createdAt: now, updatedAt: now, meta: node.meta ?? {}, facets: rawFacetsFromNode(node),
|
|
93
|
+
// Which key opens this object; null for anything stored in the clear.
|
|
94
|
+
encryption: node.encryption ?? null,
|
|
93
95
|
};
|
|
94
96
|
this.#index(full);
|
|
95
97
|
return clone(full);
|
|
@@ -98,7 +100,7 @@ export class MemoryStore extends MetadataStore {
|
|
|
98
100
|
async update(id, patch) {
|
|
99
101
|
const node = this.nodes.get(id);
|
|
100
102
|
if (!node) throw TroveError.notFound('Item');
|
|
101
|
-
for (const k of ['size', 'contentType', 'storageKey', 'etag', 'meta']) {
|
|
103
|
+
for (const k of ['size', 'contentType', 'storageKey', 'etag', 'meta', 'encryption']) {
|
|
102
104
|
if (k in patch) node[k] = patch[k];
|
|
103
105
|
}
|
|
104
106
|
node.updatedAt = Date.now();
|
|
@@ -46,7 +46,8 @@ export class SqliteStore extends MetadataStore {
|
|
|
46
46
|
createdAt INTEGER NOT NULL,
|
|
47
47
|
updatedAt INTEGER NOT NULL,
|
|
48
48
|
meta TEXT NOT NULL DEFAULT '{}',
|
|
49
|
-
facets TEXT NOT NULL DEFAULT '{}'
|
|
49
|
+
facets TEXT NOT NULL DEFAULT '{}',
|
|
50
|
+
encryption TEXT
|
|
50
51
|
);
|
|
51
52
|
CREATE INDEX IF NOT EXISTS idx_nodes_name ON nodes(name);
|
|
52
53
|
CREATE INDEX IF NOT EXISTS idx_nodes_updated ON nodes(updatedAt);
|
|
@@ -68,6 +69,11 @@ export class SqliteStore extends MetadataStore {
|
|
|
68
69
|
if (!cols.some((c) => c.name === 'deletedAt')) {
|
|
69
70
|
await this.db.exec('ALTER TABLE nodes ADD COLUMN deletedAt INTEGER');
|
|
70
71
|
}
|
|
72
|
+
// Which key opens this object, for drives that predate encryption. NULL means stored
|
|
73
|
+
// in the clear, which is what every existing row is.
|
|
74
|
+
if (!cols.some((c) => c.name === 'encryption')) {
|
|
75
|
+
await this.db.exec('ALTER TABLE nodes ADD COLUMN encryption TEXT');
|
|
76
|
+
}
|
|
71
77
|
// Recreating the index is cheap and idempotent; naming the new one differently is
|
|
72
78
|
// what makes "has this run?" answerable without a migrations table.
|
|
73
79
|
await this.db.exec(`
|
|
@@ -130,14 +136,16 @@ export class SqliteStore extends MetadataStore {
|
|
|
130
136
|
size: node.size ?? 0, contentType: node.contentType ?? null,
|
|
131
137
|
storageKey: node.storageKey ?? null, etag: node.etag ?? null,
|
|
132
138
|
createdAt: now, updatedAt: now, meta: node.meta ?? {}, facets: rawFacetsFromNode(node),
|
|
139
|
+
encryption: node.encryption ?? null,
|
|
133
140
|
};
|
|
134
141
|
try {
|
|
135
142
|
await this.db.run(
|
|
136
|
-
`INSERT INTO nodes (id,collectionId,name,size,contentType,storageKey,etag,createdAt,updatedAt,meta,facets)
|
|
137
|
-
VALUES (
|
|
143
|
+
`INSERT INTO nodes (id,collectionId,name,size,contentType,storageKey,etag,createdAt,updatedAt,meta,facets,encryption)
|
|
144
|
+
VALUES (?,?,?,?,?,?,?,?,?,?,?,?)`,
|
|
138
145
|
full.id, full.collectionId, full.name, full.size,
|
|
139
146
|
full.contentType, full.storageKey, full.etag, full.createdAt, full.updatedAt,
|
|
140
147
|
JSON.stringify(full.meta), JSON.stringify(full.facets),
|
|
148
|
+
full.encryption ? JSON.stringify(full.encryption) : null,
|
|
141
149
|
);
|
|
142
150
|
return full;
|
|
143
151
|
} catch (err) {
|
|
@@ -150,13 +158,17 @@ export class SqliteStore extends MetadataStore {
|
|
|
150
158
|
const node = await this.getById(id);
|
|
151
159
|
if (!node) throw TroveError.notFound('Node');
|
|
152
160
|
const next = { ...node };
|
|
153
|
-
|
|
161
|
+
// `encryption` is here because a key rotation rewrites the object and has to record
|
|
162
|
+
// which key the new one is sealed with. Without it a rotated item still claims the old
|
|
163
|
+
// key and becomes unreadable the moment that key is retired.
|
|
164
|
+
for (const k of ['size', 'contentType', 'storageKey', 'etag', 'meta', 'encryption']) {
|
|
154
165
|
if (k in patch) next[k] = patch[k];
|
|
155
166
|
}
|
|
156
167
|
next.updatedAt = Date.now();
|
|
157
168
|
await this.db.run(
|
|
158
|
-
`UPDATE nodes SET size=?, contentType=?, storageKey=?, etag=?, meta=?, updatedAt=? WHERE id=?`,
|
|
159
|
-
next.size, next.contentType, next.storageKey, next.etag, JSON.stringify(next.meta),
|
|
169
|
+
`UPDATE nodes SET size=?, contentType=?, storageKey=?, etag=?, meta=?, encryption=?, updatedAt=? WHERE id=?`,
|
|
170
|
+
next.size, next.contentType, next.storageKey, next.etag, JSON.stringify(next.meta),
|
|
171
|
+
next.encryption ? JSON.stringify(next.encryption) : null, next.updatedAt, id,
|
|
160
172
|
);
|
|
161
173
|
return next;
|
|
162
174
|
}
|
|
@@ -353,7 +365,15 @@ export class SqliteStore extends MetadataStore {
|
|
|
353
365
|
function row(r) {
|
|
354
366
|
if (!r) return null;
|
|
355
367
|
const { contributions, tags } = splitContributions(r.facets ? JSON.parse(r.facets) : {});
|
|
356
|
-
const out = {
|
|
368
|
+
const out = {
|
|
369
|
+
...r,
|
|
370
|
+
meta: r.meta ? JSON.parse(r.meta) : {},
|
|
371
|
+
// Null for everything stored in the clear, which is every row on a drive that predates
|
|
372
|
+
// encryption and every item a rule did not match.
|
|
373
|
+
encryption: r.encryption ? JSON.parse(r.encryption) : null,
|
|
374
|
+
contributions,
|
|
375
|
+
tags,
|
|
376
|
+
};
|
|
357
377
|
delete out.facets; // internal column name; exposed as contributions + tags
|
|
358
378
|
return out;
|
|
359
379
|
}
|
|
@@ -23,6 +23,8 @@
|
|
|
23
23
|
import { TroveError } from './errors.js';
|
|
24
24
|
import { extname } from './util.js';
|
|
25
25
|
import { PACKAGE_PREFIX } from './plugins/packageStore.js';
|
|
26
|
+
import { isEnvelope, decodeHeader, HEADER_BYTES } from './encryption/envelope.js';
|
|
27
|
+
import { toHex } from './encryption/keys.js';
|
|
26
28
|
|
|
27
29
|
/** Objects Trove wrote itself. Anything else in the store arrived some other way. */
|
|
28
30
|
const TROVE_KEY = /^obj_[0-9a-f]+$/i;
|
|
@@ -184,6 +186,26 @@ export class CollectionScanner {
|
|
|
184
186
|
return false;
|
|
185
187
|
}
|
|
186
188
|
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* Read an object's envelope header, or null if it does not have one.
|
|
192
|
+
*
|
|
193
|
+
* A fixed-size range read, so it costs one small request per adopted object and never
|
|
194
|
+
* pulls the object itself. A failure to read or parse means "not one of ours", which is
|
|
195
|
+
* the right answer for an ordinary file that happens to start with something odd.
|
|
196
|
+
*/
|
|
197
|
+
async #envelopeOf(collectionId, storageKey) {
|
|
198
|
+
try {
|
|
199
|
+
const storage = await this.vfs.storageFor(collectionId);
|
|
200
|
+
const head = await storage.get(storageKey, { range: { start: 0, end: HEADER_BYTES - 1 } });
|
|
201
|
+
const bytes = new Uint8Array(await new Response(head.stream).arrayBuffer());
|
|
202
|
+
if (!isEnvelope(bytes)) return null;
|
|
203
|
+
return decodeHeader(bytes);
|
|
204
|
+
} catch {
|
|
205
|
+
return null;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
187
209
|
/**
|
|
188
210
|
* Create an item for an object that arrived without us.
|
|
189
211
|
*
|
|
@@ -196,13 +218,25 @@ export class CollectionScanner {
|
|
|
196
218
|
async #adopt(collectionId, object) {
|
|
197
219
|
if (TROVE_KEY.test(object.key)) return null; // orphaned blob from an interrupted upload
|
|
198
220
|
const name = await this.#uniqueName(collectionId, object.key);
|
|
221
|
+
// Is this an encrypted object somebody copied in?
|
|
222
|
+
//
|
|
223
|
+
// The envelope says so, and says which key it wants, WITHOUT the key — which is the
|
|
224
|
+
// entire reason the header is readable. Without this an adopted object is recorded as
|
|
225
|
+
// plaintext, and every read of it hands back raw ciphertext with no error: the drive
|
|
226
|
+
// shows a file, and opening it gives you an unreadable blob. Sideloading is a named
|
|
227
|
+
// use case here, so it has to be the case that works.
|
|
228
|
+
const envelope = await this.#envelopeOf(collectionId, object.key);
|
|
199
229
|
const node = await this.vfs.metadata.create({
|
|
200
230
|
collectionId,
|
|
201
231
|
name,
|
|
202
232
|
storageKey: object.key,
|
|
203
|
-
size
|
|
233
|
+
// The size the file has, not the size the envelope occupies.
|
|
234
|
+
size: envelope ? envelope.plaintextSize : (object.size ?? 0),
|
|
204
235
|
etag: object.etag ?? null,
|
|
205
236
|
contentType: this.vfs.guessContentType(name),
|
|
237
|
+
encryption: envelope
|
|
238
|
+
? { fingerprint: toHex(envelope.fingerprint), chunkSize: envelope.chunkSize }
|
|
239
|
+
: null,
|
|
206
240
|
meta: { adopted: true, adoptedAt: Date.now() },
|
|
207
241
|
});
|
|
208
242
|
// Adopted files are indexed like any other, so they are findable immediately —
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
// What a bulk pass over a collection will cost at the storage provider.
|
|
2
|
+
//
|
|
3
|
+
// Re-encrypting a collection is the first operation here big enough for the answer to
|
|
4
|
+
// matter: the server reads every object and writes it back, so on a metered store it is a
|
|
5
|
+
// real bill, and on a large collection it can be a surprising one. Someone should be able
|
|
6
|
+
// to see the number before they press the button, not on an invoice.
|
|
7
|
+
//
|
|
8
|
+
// Two things shape how this is written.
|
|
9
|
+
//
|
|
10
|
+
// THE MODEL OUTLASTS THE NUMBERS. "R2 does not charge for egress, so this costs operations
|
|
11
|
+
// only" stays true long after any per-GB figure is stale. So every provider carries a
|
|
12
|
+
// plain-language description of WHAT is charged, and that is what leads; the rates are
|
|
13
|
+
// supporting detail, stamped with the date they were taken and linked to the source.
|
|
14
|
+
//
|
|
15
|
+
// WE WILL BE WRONG EVENTUALLY. Prices change, free tiers change, and a drive can be years
|
|
16
|
+
// old. Nothing here is presented as authoritative: an estimate says where its figures came
|
|
17
|
+
// from and when, and every answer links the provider's own pricing page. An unrecognised
|
|
18
|
+
// endpoint gets an honest description of how object stores usually bill rather than a
|
|
19
|
+
// fabricated number.
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* When these rates were last checked. Shown with every estimate, because a figure without
|
|
23
|
+
* a date invites being trusted longer than it deserves.
|
|
24
|
+
*/
|
|
25
|
+
export const RATES_AS_OF = '2026-07';
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* What we think we know, per provider.
|
|
29
|
+
*
|
|
30
|
+
* `egress` is the field that actually decides whether a rotation is cheap or expensive, and
|
|
31
|
+
* it is the one worth getting right: a store that does not charge for reads out is a store
|
|
32
|
+
* where this operation costs pennies regardless of size.
|
|
33
|
+
*/
|
|
34
|
+
const PROVIDERS = [
|
|
35
|
+
{
|
|
36
|
+
id: 'r2',
|
|
37
|
+
name: 'Cloudflare R2',
|
|
38
|
+
match: /\.r2\.cloudflarestorage\.com$/i,
|
|
39
|
+
docs: 'https://developers.cloudflare.com/r2/pricing/',
|
|
40
|
+
egress: 'none',
|
|
41
|
+
egressNote: 'R2 does not charge for data transferred out, which is the cost that would otherwise dominate this.',
|
|
42
|
+
// Class B is reads, Class A is writes.
|
|
43
|
+
readPerMillion: 0.36,
|
|
44
|
+
writePerMillion: 4.50,
|
|
45
|
+
egressPerGB: 0,
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
id: 'aws',
|
|
49
|
+
name: 'Amazon S3',
|
|
50
|
+
// Covers the regional forms (s3.eu-west-2, s3-us-west-2), the legacy global endpoint
|
|
51
|
+
// (s3.amazonaws.com), and virtual-host style (bucket.s3.region.amazonaws.com).
|
|
52
|
+
match: /(^|\.)s3([.-][a-z0-9-]+)?\.amazonaws\.com$/i,
|
|
53
|
+
docs: 'https://aws.amazon.com/s3/pricing/',
|
|
54
|
+
egress: 'metered',
|
|
55
|
+
// The nuance that changes the answer by orders of magnitude, and the one people are
|
|
56
|
+
// most often caught by.
|
|
57
|
+
egressNote:
|
|
58
|
+
'S3 charges for data transferred out to the internet. If this drive runs outside AWS '
|
|
59
|
+
+ '(on Cloudflare Workers, for example) every byte read is billed at that rate; if it runs '
|
|
60
|
+
+ 'in EC2 or Lambda in the same region as the bucket, transfer is normally free and this '
|
|
61
|
+
+ 'costs requests only.',
|
|
62
|
+
readPerMillion: 400,
|
|
63
|
+
writePerMillion: 5000,
|
|
64
|
+
egressPerGB: 0.09,
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
id: 'b2',
|
|
68
|
+
name: 'Backblaze B2',
|
|
69
|
+
match: /(^|\.)backblazeb2\.com$/i,
|
|
70
|
+
docs: 'https://www.backblaze.com/cloud-storage/pricing',
|
|
71
|
+
egress: 'allowance',
|
|
72
|
+
egressNote:
|
|
73
|
+
'B2 includes free egress up to a multiple of what you store, and charges beyond it. A '
|
|
74
|
+
+ 'one-off pass over a collection often falls inside that allowance.',
|
|
75
|
+
readPerMillion: 4,
|
|
76
|
+
writePerMillion: 0,
|
|
77
|
+
egressPerGB: 0.01,
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
id: 'wasabi',
|
|
81
|
+
name: 'Wasabi',
|
|
82
|
+
match: /(^|\.)wasabisys\.com$/i,
|
|
83
|
+
docs: 'https://wasabi.com/cloud-storage-pricing',
|
|
84
|
+
egress: 'none',
|
|
85
|
+
egressNote:
|
|
86
|
+
'Wasabi does not charge for egress or requests. Note its minimum storage duration, '
|
|
87
|
+
+ 'though: rewriting an object can restart the clock on what the original was charged for.',
|
|
88
|
+
readPerMillion: 0,
|
|
89
|
+
writePerMillion: 0,
|
|
90
|
+
egressPerGB: 0,
|
|
91
|
+
},
|
|
92
|
+
{
|
|
93
|
+
id: 'spaces',
|
|
94
|
+
name: 'DigitalOcean Spaces',
|
|
95
|
+
match: /(^|\.)digitaloceanspaces\.com$/i,
|
|
96
|
+
docs: 'https://www.digitalocean.com/pricing/spaces-object-storage',
|
|
97
|
+
egress: 'allowance',
|
|
98
|
+
egressNote: 'Spaces includes a monthly transfer allowance and charges per GB beyond it.',
|
|
99
|
+
readPerMillion: 0,
|
|
100
|
+
writePerMillion: 0,
|
|
101
|
+
egressPerGB: 0.01,
|
|
102
|
+
},
|
|
103
|
+
{
|
|
104
|
+
id: 'gcs',
|
|
105
|
+
name: 'Google Cloud Storage',
|
|
106
|
+
match: /(^|\.)storage\.googleapis\.com$/i,
|
|
107
|
+
docs: 'https://cloud.google.com/storage/pricing',
|
|
108
|
+
egress: 'metered',
|
|
109
|
+
egressNote:
|
|
110
|
+
'GCS charges for data read out to the internet, and for operations. Reading from within '
|
|
111
|
+
+ 'Google Cloud in the same region is normally free.',
|
|
112
|
+
readPerMillion: 400,
|
|
113
|
+
writePerMillion: 5000,
|
|
114
|
+
egressPerGB: 0.12,
|
|
115
|
+
},
|
|
116
|
+
];
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Stores where this question does not arise.
|
|
120
|
+
*
|
|
121
|
+
* A directory on a disk and a NATS object store are not metered by anyone, so offering a
|
|
122
|
+
* cost estimate would be inventing a concern.
|
|
123
|
+
*/
|
|
124
|
+
const UNMETERED_DRIVERS = new Set(['filesystem', 'memory', 'nats']);
|
|
125
|
+
|
|
126
|
+
/** Which provider an endpoint belongs to, or null if we do not recognise it. */
|
|
127
|
+
export function recognizeProvider(endpoint) {
|
|
128
|
+
if (!endpoint) return null;
|
|
129
|
+
let host;
|
|
130
|
+
try {
|
|
131
|
+
host = new URL(endpoint).hostname;
|
|
132
|
+
} catch {
|
|
133
|
+
return null;
|
|
134
|
+
}
|
|
135
|
+
return PROVIDERS.find((p) => p.match.test(host)) || null;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
const gb = (bytes) => bytes / 1024 ** 3;
|
|
139
|
+
const money = (n) => Math.round(n * 100) / 100;
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* What re-encrypting a collection will cost.
|
|
143
|
+
*
|
|
144
|
+
* The work is one read and one write per object, plus the bytes moving out of the bucket
|
|
145
|
+
* and back in. Ingress is free essentially everywhere, so it is named and not priced.
|
|
146
|
+
*
|
|
147
|
+
* @param {object} target
|
|
148
|
+
* @param {string} target.driver the collection's store driver
|
|
149
|
+
* @param {string} [target.endpoint] its S3 endpoint, if it has one
|
|
150
|
+
* @param {number} objects how many objects will be rewritten
|
|
151
|
+
* @param {number} bytes how many bytes they hold
|
|
152
|
+
*/
|
|
153
|
+
export function estimateRotationCost({ driver, endpoint } = {}, { objects = 0, bytes = 0 } = {}) {
|
|
154
|
+
if (UNMETERED_DRIVERS.has(driver)) {
|
|
155
|
+
return {
|
|
156
|
+
applicable: false,
|
|
157
|
+
provider: null,
|
|
158
|
+
summary: 'This collection is on storage nobody bills you for, so a rotation costs only the time it takes.',
|
|
159
|
+
lines: [],
|
|
160
|
+
total: null,
|
|
161
|
+
docs: null,
|
|
162
|
+
asOf: RATES_AS_OF,
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
const p = recognizeProvider(endpoint);
|
|
167
|
+
const work = [
|
|
168
|
+
{ label: 'Objects read and rewritten', value: `${objects.toLocaleString()}` },
|
|
169
|
+
{ label: 'Data moved out of the bucket', value: `${gb(bytes).toFixed(2)} GB` },
|
|
170
|
+
// Named rather than priced: essentially no object store charges to receive bytes, and
|
|
171
|
+
// an estimate that lists a zero invites the reader to wonder what it is hiding.
|
|
172
|
+
{ label: 'Data written back', value: `${gb(bytes).toFixed(2)} GB (ingress is not normally charged)` },
|
|
173
|
+
];
|
|
174
|
+
|
|
175
|
+
if (!p) {
|
|
176
|
+
// Honest rather than invented. Describing how these services usually bill is more use
|
|
177
|
+
// than a number we made up for a host we have never seen.
|
|
178
|
+
return {
|
|
179
|
+
applicable: true,
|
|
180
|
+
provider: null,
|
|
181
|
+
confidence: 'unknown',
|
|
182
|
+
summary:
|
|
183
|
+
'We do not recognise this storage endpoint, so we cannot estimate a price. Object stores '
|
|
184
|
+
+ 'usually charge for three things: the requests made, the data read back out, and what is '
|
|
185
|
+
+ 'stored. A rotation makes one read and one write per object and moves every byte out and '
|
|
186
|
+
+ 'back — so if your provider charges for egress, that is what will dominate.',
|
|
187
|
+
lines: work,
|
|
188
|
+
total: null,
|
|
189
|
+
docs: null,
|
|
190
|
+
asOf: RATES_AS_OF,
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
const egressCost = gb(bytes) * p.egressPerGB;
|
|
195
|
+
const readCost = (objects / 1e6) * p.readPerMillion;
|
|
196
|
+
const writeCost = (objects / 1e6) * p.writePerMillion;
|
|
197
|
+
const total = egressCost + readCost + writeCost;
|
|
198
|
+
|
|
199
|
+
const lines = [
|
|
200
|
+
...work,
|
|
201
|
+
{ label: 'Reads', value: `$${money(readCost).toFixed(2)}` },
|
|
202
|
+
{ label: 'Writes', value: `$${money(writeCost).toFixed(2)}` },
|
|
203
|
+
{
|
|
204
|
+
label: 'Egress',
|
|
205
|
+
value: p.egress === 'none' ? 'not charged' : `$${money(egressCost).toFixed(2)}`,
|
|
206
|
+
note: p.egressNote,
|
|
207
|
+
},
|
|
208
|
+
];
|
|
209
|
+
|
|
210
|
+
return {
|
|
211
|
+
applicable: true,
|
|
212
|
+
provider: p.name,
|
|
213
|
+
confidence: 'known',
|
|
214
|
+
// Leads with the model, because that is the part that stays true.
|
|
215
|
+
summary: p.egress === 'none'
|
|
216
|
+
? `${p.name} does not charge for data transferred out, so this costs operations only.`
|
|
217
|
+
: `${p.name} charges for data read out of the bucket, so the size of this collection is what drives the price.`,
|
|
218
|
+
lines,
|
|
219
|
+
total: { amount: money(total), currency: 'USD', approximate: true },
|
|
220
|
+
docs: p.docs,
|
|
221
|
+
asOf: RATES_AS_OF,
|
|
222
|
+
// Said in the estimate itself, not left to the UI to remember.
|
|
223
|
+
caveat:
|
|
224
|
+
`These figures are what we recorded in ${RATES_AS_OF} and prices change — treat this as an `
|
|
225
|
+
+ `order of magnitude, not a quote, and check ${p.name}'s own pricing page for what you will `
|
|
226
|
+
+ 'actually be billed. Free tiers and committed-use discounts are not accounted for here.',
|
|
227
|
+
};
|
|
228
|
+
}
|
|
@@ -49,9 +49,19 @@ export function portableDrivers() {
|
|
|
49
49
|
help: 'MinIO and most self-hosted endpoints need this.',
|
|
50
50
|
},
|
|
51
51
|
],
|
|
52
|
-
//
|
|
53
|
-
//
|
|
54
|
-
//
|
|
52
|
+
// Two config shapes reach here and both have to work.
|
|
53
|
+
//
|
|
54
|
+
// Flat — `{ driver: 's3', bucket: … }` — is what `fields` above describes and what
|
|
55
|
+
// the collection form posts. Nested under `s3` is what `configFromEnv` produces and
|
|
56
|
+
// what every collection record written before this driver existed holds, because the
|
|
57
|
+
// switch this replaced read `cfg.s3` and nothing else.
|
|
58
|
+
//
|
|
59
|
+
// Normalising rather than only handling it in `create` is the point: validation runs
|
|
60
|
+
// against the normalised shape, so a nested config is no longer refused for a missing
|
|
61
|
+
// top-level `bucket` that was about to be spread in anyway. That refusal broke every
|
|
62
|
+
// environment-configured S3 deployment at startup, which is as loud as a bug gets and
|
|
63
|
+
// still took a local run to see, because no test used the env shape.
|
|
64
|
+
normalize: (cfg) => ({ ...cfg, ...(cfg.s3 || {}) }),
|
|
55
65
|
create: (cfg) => new S3Storage({
|
|
56
66
|
bucket: cfg.bucket,
|
|
57
67
|
region: cfg.region || 'auto',
|
|
@@ -60,7 +70,6 @@ export function portableDrivers() {
|
|
|
60
70
|
secretAccessKey: cfg.secretAccessKey,
|
|
61
71
|
sessionToken: cfg.sessionToken,
|
|
62
72
|
forcePathStyle: cfg.forcePathStyle === true || cfg.forcePathStyle === 'true',
|
|
63
|
-
...(cfg.s3 || {}),
|
|
64
73
|
}),
|
|
65
74
|
},
|
|
66
75
|
{
|
|
@@ -56,6 +56,9 @@ export class StorageDriverRegistry {
|
|
|
56
56
|
* @param {string} driver.label for a form
|
|
57
57
|
* @param {string} [driver.description] one line about what it is
|
|
58
58
|
* @param {DriverField[]} [driver.fields]
|
|
59
|
+
* @param {(config: object) => object} [driver.normalize] accept an older or alternative
|
|
60
|
+
* config shape, returning the one `fields` describes. Runs before validation, so the
|
|
61
|
+
* shape checked is the shape built from.
|
|
59
62
|
* @param {(config: object) => StorageBackend} driver.create
|
|
60
63
|
*/
|
|
61
64
|
register(driver) {
|
|
@@ -82,6 +85,7 @@ export class StorageDriverRegistry {
|
|
|
82
85
|
placeholder: f.placeholder || '',
|
|
83
86
|
help: f.help || '',
|
|
84
87
|
})),
|
|
88
|
+
normalize: driver.normalize || null,
|
|
85
89
|
create: driver.create,
|
|
86
90
|
});
|
|
87
91
|
return this;
|
|
@@ -95,9 +99,30 @@ export class StorageDriverRegistry {
|
|
|
95
99
|
return [...this._drivers.keys()];
|
|
96
100
|
}
|
|
97
101
|
|
|
98
|
-
/**
|
|
102
|
+
/**
|
|
103
|
+
* One registered driver, `create` included — unlike `describe()`, which deliberately
|
|
104
|
+
* strips it because it answers a client.
|
|
105
|
+
*
|
|
106
|
+
* For copying a driver into another registry, which is how a deployment narrows the set
|
|
107
|
+
* it offers: a registry has no `unregister`, so narrowing rebuilds from what survived
|
|
108
|
+
* rather than removing from what did not. Keeping removal out of this class is the point
|
|
109
|
+
* — a driver disappearing from a live registry is a store that stops being buildable
|
|
110
|
+
* while collections still reference it.
|
|
111
|
+
*/
|
|
112
|
+
driver(key) {
|
|
113
|
+
return this._drivers.get(key);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* What a client needs to render a form.
|
|
118
|
+
*
|
|
119
|
+
* Neither `create` nor `normalize`: both are behaviour, not data. JSON.stringify would
|
|
120
|
+
* drop them anyway, which is exactly why they are removed here instead — a describe()
|
|
121
|
+
* whose result is only serialisable by accident is one that leaks the next function
|
|
122
|
+
* somebody adds into every in-process consumer.
|
|
123
|
+
*/
|
|
99
124
|
describe() {
|
|
100
|
-
return [...this._drivers.values()].map(({ create, ...rest }) => rest);
|
|
125
|
+
return [...this._drivers.values()].map(({ create, normalize, ...rest }) => rest);
|
|
101
126
|
}
|
|
102
127
|
|
|
103
128
|
/**
|
|
@@ -105,6 +130,13 @@ export class StorageDriverRegistry {
|
|
|
105
130
|
*
|
|
106
131
|
* An unknown driver throws, and says what IS available. This is the arm that used to
|
|
107
132
|
* return an in-memory store.
|
|
133
|
+
*
|
|
134
|
+
* `normalize` runs FIRST, so a driver that accepts more than one config shape validates
|
|
135
|
+
* the shape it will actually build from. Without it, required-field checks are performed
|
|
136
|
+
* against a config the driver was about to rewrite — which is precisely how S3 broke:
|
|
137
|
+
* `configFromEnv` nests its settings under `s3`, `create` spread that back out, and the
|
|
138
|
+
* check in between looked for a top-level `bucket` that was never going to be there and
|
|
139
|
+
* refused every environment-configured S3 deployment at startup.
|
|
108
140
|
*/
|
|
109
141
|
build(config) {
|
|
110
142
|
const key = config?.driver;
|
|
@@ -115,12 +147,13 @@ export class StorageDriverRegistry {
|
|
|
115
147
|
`Unknown storage driver "${key}" — this deployment has: ${this.keys().join(', ') || 'none'}`,
|
|
116
148
|
);
|
|
117
149
|
}
|
|
150
|
+
const cfg = driver.normalize ? driver.normalize(config) : config;
|
|
118
151
|
for (const f of driver.fields) {
|
|
119
|
-
if (f.required && (
|
|
152
|
+
if (f.required && (cfg[f.name] == null || cfg[f.name] === '')) {
|
|
120
153
|
throw TroveError.invalid(`Storage driver "${key}" requires "${f.name}"`);
|
|
121
154
|
}
|
|
122
155
|
}
|
|
123
|
-
const backend = driver.create(
|
|
156
|
+
const backend = driver.create(cfg);
|
|
124
157
|
if (!(backend instanceof StorageBackend)) {
|
|
125
158
|
throw TroveError.invalid(`Storage driver "${key}" did not return a StorageBackend`);
|
|
126
159
|
}
|