@crombolo/transfer 0.37.0 → 0.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -3
- package/src/check.ts +5 -3
- package/src/index.ts +1 -0
- package/src/load.ts +43 -5
- package/src/model.ts +8 -0
- package/src/package.ts +3 -0
- package/src/people-apply.ts +30 -23
- package/src/sanitise.ts +70 -2
- package/src/site-data.ts +108 -0
- package/src/write.ts +17 -4
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crombolo/transfer",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.39.0",
|
|
4
4
|
"description": "Internal to crombolo, with no compatibility promise: use @crombolo/crombolo. Content transfer for crombolo: bundles of content moved between environments.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Crombolo",
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
"access": "public"
|
|
29
29
|
},
|
|
30
30
|
"dependencies": {
|
|
31
|
-
"@crombolo/core": "0.
|
|
32
|
-
"@crombolo/data": "0.
|
|
31
|
+
"@crombolo/core": "0.39.0",
|
|
32
|
+
"@crombolo/data": "0.39.0"
|
|
33
33
|
}
|
|
34
34
|
}
|
package/src/check.ts
CHANGED
|
@@ -863,9 +863,11 @@ export async function checkBundle(
|
|
|
863
863
|
// pages, and names groups that may not exist here yet.
|
|
864
864
|
const groupsHere = new Set(
|
|
865
865
|
(
|
|
866
|
-
await db.
|
|
867
|
-
|
|
868
|
-
|
|
866
|
+
await db.q
|
|
867
|
+
.selectFrom('node')
|
|
868
|
+
.select('text')
|
|
869
|
+
.where('node_object_type', '=', ObjectTypes.MemberGroup)
|
|
870
|
+
.execute()
|
|
869
871
|
).map((row) => String(row.text)),
|
|
870
872
|
)
|
|
871
873
|
const carriedPeople = options.excludePeople ? undefined : set.people
|
package/src/index.ts
CHANGED
package/src/load.ts
CHANGED
|
@@ -29,6 +29,12 @@ import {
|
|
|
29
29
|
} from './model.ts'
|
|
30
30
|
import { type BundlePeople, openPeople, type PeopleBlock, parsePeopleBlock } from './people.ts'
|
|
31
31
|
import { openBytes, parseSealParameters, unlock } from './seal.ts'
|
|
32
|
+
import {
|
|
33
|
+
openSiteData,
|
|
34
|
+
parseSiteDataBlock,
|
|
35
|
+
type SiteDataBlock,
|
|
36
|
+
type SiteDataTable,
|
|
37
|
+
} from './site-data.ts'
|
|
32
38
|
import {
|
|
33
39
|
bundleIntegrity,
|
|
34
40
|
MANIFEST_FILE,
|
|
@@ -53,6 +59,8 @@ export interface LoadedBundle {
|
|
|
53
59
|
files: Map<string, string>
|
|
54
60
|
/** The sealed people, when the bundle carries any: opened by `openBundle`. */
|
|
55
61
|
people?: { block: PeopleBlock; files: Map<string, Uint8Array> }
|
|
62
|
+
/** The sealed site data, when the bundle carries any: opened by `openBundle` with the people. */
|
|
63
|
+
siteData?: { block: SiteDataBlock; files: Map<string, Uint8Array> }
|
|
56
64
|
/** Each node's sealed values, by path, when the content is sealed. */
|
|
57
65
|
sealedValues?: Map<string, Uint8Array>
|
|
58
66
|
/**
|
|
@@ -367,6 +375,9 @@ function parseManifest(text: string, problems: BundleProblem[]): BundleManifest
|
|
|
367
375
|
...(raw.people !== undefined && parsePeopleBlock(raw.people)
|
|
368
376
|
? { people: parsePeopleBlock(raw.people) as PeopleBlock }
|
|
369
377
|
: {}),
|
|
378
|
+
...(raw.siteData !== undefined && parseSiteDataBlock(raw.siteData)
|
|
379
|
+
? { siteData: parseSiteDataBlock(raw.siteData) as SiteDataBlock }
|
|
380
|
+
: {}),
|
|
370
381
|
...(parseSealParameters(raw.seal) ? { seal: parseSealParameters(raw.seal) } : {}),
|
|
371
382
|
...(raw.content === 'sealed' || raw.content === 'omitted' ? { content: raw.content } : {}),
|
|
372
383
|
...(raw.scope === 'site' ? { scope: 'site' as const } : {}),
|
|
@@ -573,7 +584,22 @@ export function loadBundle(dir: string): LoadedBundle {
|
|
|
573
584
|
read.push({ path, bytes })
|
|
574
585
|
}
|
|
575
586
|
|
|
576
|
-
|
|
587
|
+
const siteData = new Map<string, Uint8Array>()
|
|
588
|
+
for (const path of manifest.siteData?.paths ?? []) {
|
|
589
|
+
const file = join(dir, path)
|
|
590
|
+
if (!existsSync(file)) {
|
|
591
|
+
problems.push({
|
|
592
|
+
file: path,
|
|
593
|
+
message: 'the manifest says this file is carried, and it is not',
|
|
594
|
+
})
|
|
595
|
+
continue
|
|
596
|
+
}
|
|
597
|
+
const bytes = new Uint8Array(readFileSync(file))
|
|
598
|
+
siteData.set(path, bytes)
|
|
599
|
+
read.push({ path, bytes })
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
if ((manifest.content === 'sealed' || manifest.people || manifest.siteData) && !manifest.seal)
|
|
577
603
|
problems.push({
|
|
578
604
|
file: MANIFEST_FILE,
|
|
579
605
|
message: 'the bundle says it is sealed but carries no seal to open it with',
|
|
@@ -625,13 +651,17 @@ export function loadBundle(dir: string): LoadedBundle {
|
|
|
625
651
|
seen.add(node.key)
|
|
626
652
|
}
|
|
627
653
|
|
|
628
|
-
const isSealed =
|
|
654
|
+
const isSealed =
|
|
655
|
+
manifest.content === 'sealed' ||
|
|
656
|
+
manifest.people !== undefined ||
|
|
657
|
+
manifest.siteData !== undefined
|
|
629
658
|
return {
|
|
630
659
|
set: { manifest, nodes, ...(isSealed ? { sealed: true } : {}) },
|
|
631
660
|
problems,
|
|
632
661
|
blobs,
|
|
633
662
|
files,
|
|
634
663
|
...(manifest.people ? { people: { block: manifest.people, files: people } } : {}),
|
|
664
|
+
...(manifest.siteData ? { siteData: { block: manifest.siteData, files: siteData } } : {}),
|
|
635
665
|
...(manifest.content === 'sealed' ? { sealedValues: sealed } : {}),
|
|
636
666
|
}
|
|
637
667
|
}
|
|
@@ -640,6 +670,8 @@ export function loadBundle(dir: string): LoadedBundle {
|
|
|
640
670
|
export interface OpenedBundle {
|
|
641
671
|
set: ContentSet
|
|
642
672
|
people: BundlePeople | undefined
|
|
673
|
+
/** The site's and bundles' own tables; left sealed, as the people are, by `withoutPeople`. */
|
|
674
|
+
siteData: SiteDataTable[] | undefined
|
|
643
675
|
/** A carried media file's bytes as they were exported. */
|
|
644
676
|
openBlob: (key: string, bytes: Uint8Array) => Uint8Array
|
|
645
677
|
}
|
|
@@ -657,11 +689,12 @@ export function openBundle(
|
|
|
657
689
|
const set = loaded.set
|
|
658
690
|
if (!set) throw new Error('The bundle could not be read, so there is nothing to open.')
|
|
659
691
|
const params = set.manifest.seal
|
|
660
|
-
if (!set.sealed || !params)
|
|
692
|
+
if (!set.sealed || !params)
|
|
693
|
+
return { set, people: undefined, siteData: undefined, openBlob: (_key, bytes) => bytes }
|
|
661
694
|
// People left behind need no passphrase when nothing else is sealed.
|
|
662
695
|
if (options.withoutPeople && set.manifest.content !== 'sealed') {
|
|
663
696
|
const { sealed: _sealed, ...open } = set
|
|
664
|
-
return { set: open, people: undefined, openBlob: (_key, bytes) => bytes }
|
|
697
|
+
return { set: open, people: undefined, siteData: undefined, openBlob: (_key, bytes) => bytes }
|
|
665
698
|
}
|
|
666
699
|
const key = unlock(passphrase ?? '', params)
|
|
667
700
|
const nodes = set.nodes.map((node) => {
|
|
@@ -684,11 +717,16 @@ export function openBundle(
|
|
|
684
717
|
loaded.people && !options.withoutPeople
|
|
685
718
|
? openPeople(loaded.people.block, loaded.people.files, key)
|
|
686
719
|
: undefined
|
|
720
|
+
const siteData =
|
|
721
|
+
loaded.siteData && !options.withoutPeople
|
|
722
|
+
? openSiteData(loaded.siteData.block, loaded.siteData.files, key)
|
|
723
|
+
: undefined
|
|
687
724
|
const sealedBlobs = set.manifest.content === 'sealed'
|
|
688
725
|
const { sealed: _sealed, ...rest } = set
|
|
689
726
|
return {
|
|
690
|
-
set: { ...rest, nodes, ...(people ? { people } : {}) },
|
|
727
|
+
set: { ...rest, nodes, ...(people ? { people } : {}), ...(siteData ? { siteData } : {}) },
|
|
691
728
|
people,
|
|
729
|
+
siteData,
|
|
692
730
|
openBlob: (blob, bytes) => (sealedBlobs ? openBytes(key, mediaPath(blob), bytes) : bytes),
|
|
693
731
|
}
|
|
694
732
|
}
|
package/src/model.ts
CHANGED
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
|
|
11
11
|
import type { BundlePeople, PeopleBlock } from './people.ts'
|
|
12
12
|
import type { SealParameters } from './seal.ts'
|
|
13
|
+
import type { SiteDataBlock, SiteDataTable } from './site-data.ts'
|
|
13
14
|
|
|
14
15
|
/**
|
|
15
16
|
* Bumped when a change to the shapes below stops an older reader understanding
|
|
@@ -254,6 +255,11 @@ export interface BundleManifest {
|
|
|
254
255
|
* older reader refuses by name rather than half-reading.
|
|
255
256
|
*/
|
|
256
257
|
people?: PeopleBlock
|
|
258
|
+
/**
|
|
259
|
+
* The site's and bundles' own tables (story 189), sealed as people are:
|
|
260
|
+
* which tables and how many rows, never what they hold.
|
|
261
|
+
*/
|
|
262
|
+
siteData?: SiteDataBlock
|
|
257
263
|
/** How values, media files and people are sealed: the salt and cost, never the key. */
|
|
258
264
|
seal?: SealParameters
|
|
259
265
|
/**
|
|
@@ -296,6 +302,8 @@ export interface ContentSet {
|
|
|
296
302
|
sealed?: boolean
|
|
297
303
|
/** The people, once opened. */
|
|
298
304
|
people?: BundlePeople
|
|
305
|
+
/** The site's and bundles' own tables, once opened. */
|
|
306
|
+
siteData?: SiteDataTable[]
|
|
299
307
|
}
|
|
300
308
|
|
|
301
309
|
/** A problem with the bundle itself, as opposed to a finding about applying it. */
|
package/src/package.ts
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
* it a name, a version and a core it was written for.
|
|
8
8
|
*/
|
|
9
9
|
import { PEOPLE_DIR } from './people.ts'
|
|
10
|
+
import { SITE_DATA_DIR } from './site-data.ts'
|
|
10
11
|
import { type BundleFile, MANIFEST_FILE } from './write.ts'
|
|
11
12
|
|
|
12
13
|
/** The npm keyword a bundle publishes so the marketplace can find it. */
|
|
@@ -98,6 +99,8 @@ export function writeBundlePackage(
|
|
|
98
99
|
// and sealed content would be no use to anyone who installed it.
|
|
99
100
|
if (!info.private && payload.some((file) => file.path.startsWith(`${PEOPLE_DIR}/`)))
|
|
100
101
|
throw new Error('A bundle that carries members or users can only be a private package.')
|
|
102
|
+
if (!info.private && payload.some((file) => file.path.startsWith(`${SITE_DATA_DIR}/`)))
|
|
103
|
+
throw new Error("A bundle that carries the site's own data can only be a private package.")
|
|
101
104
|
if (!info.private && payload.some((file) => file.path.startsWith('values/')))
|
|
102
105
|
throw new Error('A bundle whose content is encrypted can only be a private package.')
|
|
103
106
|
return [
|
package/src/people-apply.ts
CHANGED
|
@@ -238,10 +238,12 @@ export async function planPeople(
|
|
|
238
238
|
}
|
|
239
239
|
|
|
240
240
|
async function memberGroupNamed(db: Db, name: string): Promise<string | undefined> {
|
|
241
|
-
const rows = await db.
|
|
242
|
-
'
|
|
243
|
-
|
|
244
|
-
|
|
241
|
+
const rows = await db.q
|
|
242
|
+
.selectFrom('node')
|
|
243
|
+
.select('unique_id')
|
|
244
|
+
.where('node_object_type', '=', ObjectTypes.MemberGroup)
|
|
245
|
+
.where('text', '=', name)
|
|
246
|
+
.execute()
|
|
245
247
|
return rows[0] ? normaliseUuid(String(rows[0].unique_id)) : undefined
|
|
246
248
|
}
|
|
247
249
|
|
|
@@ -250,10 +252,12 @@ async function memberKeyWhere(
|
|
|
250
252
|
column: 'email' | 'login_name',
|
|
251
253
|
value: string,
|
|
252
254
|
): Promise<string | undefined> {
|
|
253
|
-
const rows = await db.
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
255
|
+
const rows = await db.q
|
|
256
|
+
.selectFrom('member as m')
|
|
257
|
+
.innerJoin('node as n', 'n.id', 'm.node_id')
|
|
258
|
+
.select('n.unique_id')
|
|
259
|
+
.where((eb) => eb(eb.fn('lower', [`m.${column}`]), '=', eb.fn('lower', [eb.val(value)])))
|
|
260
|
+
.execute()
|
|
257
261
|
return rows[0] ? normaliseUuid(String(rows[0].unique_id)) : undefined
|
|
258
262
|
}
|
|
259
263
|
|
|
@@ -417,11 +421,11 @@ export async function applyPeople(
|
|
|
417
421
|
}
|
|
418
422
|
|
|
419
423
|
async function setUserPassword(tx: Db, key: string, user: BundleUser): Promise<void> {
|
|
420
|
-
await tx.
|
|
421
|
-
|
|
422
|
-
user.passwordConfig
|
|
423
|
-
normaliseUuid(key)
|
|
424
|
-
|
|
424
|
+
await tx.q
|
|
425
|
+
.updateTable('user_account')
|
|
426
|
+
.set({ password_hash: user.passwordHash, password_config: user.passwordConfig })
|
|
427
|
+
.where('key', '=', normaliseUuid(key))
|
|
428
|
+
.execute()
|
|
425
429
|
}
|
|
426
430
|
|
|
427
431
|
async function currentValues(members: MemberRepository, key: string) {
|
|
@@ -475,8 +479,8 @@ export async function removePerson(tx: Db, kind: PersonKind, key: string): Promi
|
|
|
475
479
|
const nodes = new NodeRepository(tx)
|
|
476
480
|
const node = await nodes.byKey(key)
|
|
477
481
|
if (!node || node.objectType !== ObjectTypes.MemberGroup) return false
|
|
478
|
-
await tx.
|
|
479
|
-
await tx.
|
|
482
|
+
await tx.q.deleteFrom('member_group_member').where('member_group_id', '=', node.id).execute()
|
|
483
|
+
await tx.q.deleteFrom('node').where('id', '=', node.id).execute()
|
|
480
484
|
return true
|
|
481
485
|
}
|
|
482
486
|
case 'user-group': {
|
|
@@ -495,10 +499,12 @@ export async function exportPeople(
|
|
|
495
499
|
const people: BundlePeople = { memberGroups: [], members: [], userGroups: [], users: [] }
|
|
496
500
|
const state = await currentSchemaState(db)
|
|
497
501
|
if (options.members) {
|
|
498
|
-
const groups = await db.
|
|
499
|
-
'
|
|
500
|
-
[
|
|
501
|
-
|
|
502
|
+
const groups = await db.q
|
|
503
|
+
.selectFrom('node')
|
|
504
|
+
.select(['unique_id', 'text'])
|
|
505
|
+
.where('node_object_type', '=', ObjectTypes.MemberGroup)
|
|
506
|
+
.orderBy('text')
|
|
507
|
+
.execute()
|
|
502
508
|
people.memberGroups = groups.map((g) => ({
|
|
503
509
|
key: normaliseUuid(String(g.unique_id)),
|
|
504
510
|
name: String(g.text),
|
|
@@ -551,10 +557,11 @@ export async function exportPeople(
|
|
|
551
557
|
const users = new UserRepository(db)
|
|
552
558
|
for (const user of await users.all()) {
|
|
553
559
|
if (user.kind !== 'Default') continue
|
|
554
|
-
const
|
|
555
|
-
|
|
556
|
-
password_config
|
|
557
|
-
|
|
560
|
+
const row = await db.q
|
|
561
|
+
.selectFrom('user_account')
|
|
562
|
+
.select(['password_hash', 'password_config'])
|
|
563
|
+
.where('id', '=', user.id)
|
|
564
|
+
.executeTakeFirst()
|
|
558
565
|
people.users.push({
|
|
559
566
|
key: user.key,
|
|
560
567
|
name: user.name,
|
package/src/sanitise.ts
CHANGED
|
@@ -10,6 +10,9 @@
|
|
|
10
10
|
* username and email replaced, the password and every property value gone.
|
|
11
11
|
* - Every email address in content, people and the dictionary is replaced, the
|
|
12
12
|
* same address by the same stand-in each time so references still agree.
|
|
13
|
+
* - The site's and bundles' own tables (story 189) are default-deny: a table
|
|
14
|
+
* travels only as its owner's rules say, and one without a rule travels
|
|
15
|
+
* empty. An `email` column takes the same stand-ins as everything else.
|
|
13
16
|
*
|
|
14
17
|
* Tokens, sessions, external logins and form entries are never exported, so
|
|
15
18
|
* there is nothing of theirs to remove. Code and schema are left alone: an
|
|
@@ -17,6 +20,12 @@
|
|
|
17
20
|
*/
|
|
18
21
|
import type { BundleNode, BundleValue, ContentSet } from './model.ts'
|
|
19
22
|
import type { BundleMember, BundlePeople } from './people.ts'
|
|
23
|
+
import {
|
|
24
|
+
type AnonymiseRules,
|
|
25
|
+
type SiteDataRow,
|
|
26
|
+
type SiteDataTable,
|
|
27
|
+
siteDataName,
|
|
28
|
+
} from './site-data.ts'
|
|
20
29
|
|
|
21
30
|
const EMAIL = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g
|
|
22
31
|
|
|
@@ -28,12 +37,25 @@ export interface SanitiseInput {
|
|
|
28
37
|
people?: BundlePeople
|
|
29
38
|
/** The dictionary `.udt`, whose translations may hold addresses. */
|
|
30
39
|
dictionary?: string
|
|
40
|
+
siteData?: SanitiseSiteData
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** The tables to anonymise, with each owner's rules. */
|
|
44
|
+
export interface SanitiseSiteData {
|
|
45
|
+
tables: readonly SiteDataTable[]
|
|
46
|
+
/** By owner: `site`, or a bundle's id. */
|
|
47
|
+
rules: Readonly<Record<string, AnonymiseRules | undefined>>
|
|
48
|
+
/** What a scrubbed column holds: null where it may, otherwise its type's empty value. */
|
|
49
|
+
empty: (owner: string, table: string, column: string) => unknown
|
|
31
50
|
}
|
|
32
51
|
|
|
33
52
|
export interface SanitiseResult {
|
|
34
53
|
set: ContentSet
|
|
35
54
|
people: BundlePeople | undefined
|
|
36
55
|
dictionary: string | undefined
|
|
56
|
+
siteData: SiteDataTable[] | undefined
|
|
57
|
+
/** The tables no rule lets travel, as `owner.table`: they travel empty. */
|
|
58
|
+
emptied: string[]
|
|
37
59
|
/** How many distinct email addresses were replaced. */
|
|
38
60
|
emails: number
|
|
39
61
|
}
|
|
@@ -112,10 +134,56 @@ export function sanitise(input: SanitiseInput): SanitiseResult {
|
|
|
112
134
|
: undefined
|
|
113
135
|
for (const user of input.people?.users ?? []) standInFor(user.email)
|
|
114
136
|
|
|
137
|
+
const set = { ...input.set, nodes: input.set.nodes.map(node) }
|
|
138
|
+
const dictionary = input.dictionary === undefined ? undefined : scrub(input.dictionary)
|
|
139
|
+
|
|
140
|
+
// A member column names a member only when that member travels, as their
|
|
141
|
+
// placeholder under the same key.
|
|
142
|
+
const travelling = new Set((people?.members ?? []).map((m) => m.key.toLowerCase()))
|
|
143
|
+
const emptied: string[] = []
|
|
144
|
+
const email = (value: unknown): unknown => {
|
|
145
|
+
if (typeof value !== 'string' || value === '') return value
|
|
146
|
+
const whole = value.trim()
|
|
147
|
+
// A column declared to hold an address is replaced even when it is not
|
|
148
|
+
// shaped like one, so nothing in it travels.
|
|
149
|
+
const matches = whole.match(EMAIL)
|
|
150
|
+
if (!matches || (matches.length === 1 && matches[0] === whole)) {
|
|
151
|
+
return whole.toLowerCase().endsWith(`@${ANONYMISED_DOMAIN}`) ? whole : standInFor(whole)
|
|
152
|
+
}
|
|
153
|
+
return scrub(value)
|
|
154
|
+
}
|
|
155
|
+
const siteData = input.siteData?.tables.map((t): SiteDataTable => {
|
|
156
|
+
const rule = input.siteData?.rules[t.owner]?.[t.table]
|
|
157
|
+
if (rule === undefined) {
|
|
158
|
+
emptied.push(siteDataName(t.owner, t.table))
|
|
159
|
+
return { ...t, rows: [] }
|
|
160
|
+
}
|
|
161
|
+
if (rule === 'keep') return t
|
|
162
|
+
const rows = t.rows.map((row) => {
|
|
163
|
+
const out: SiteDataRow = { ...row }
|
|
164
|
+
for (const [column, how] of Object.entries(rule)) {
|
|
165
|
+
if (!Object.hasOwn(out, column)) continue
|
|
166
|
+
const value = out[column]
|
|
167
|
+
if (how === 'email') out[column] = email(value)
|
|
168
|
+
else if (how === 'scrub')
|
|
169
|
+
out[column] = input.siteData?.empty(t.owner, t.table, column) ?? null
|
|
170
|
+
else
|
|
171
|
+
out[column] =
|
|
172
|
+
typeof value === 'string' && travelling.has(value.toLowerCase())
|
|
173
|
+
? value
|
|
174
|
+
: (input.siteData?.empty(t.owner, t.table, column) ?? null)
|
|
175
|
+
}
|
|
176
|
+
return out
|
|
177
|
+
})
|
|
178
|
+
return { ...t, rows }
|
|
179
|
+
})
|
|
180
|
+
|
|
115
181
|
return {
|
|
116
|
-
set
|
|
182
|
+
set,
|
|
117
183
|
people,
|
|
118
|
-
dictionary
|
|
184
|
+
dictionary,
|
|
185
|
+
siteData,
|
|
186
|
+
emptied,
|
|
119
187
|
emails: standIns.size,
|
|
120
188
|
}
|
|
121
189
|
}
|
package/src/site-data.ts
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The site's and bundles' own tables in a site file (story 189): one file per
|
|
3
|
+
* table at `data/<owner>/<table>.json`, a JSON array of rows, always sealed as
|
|
4
|
+
* people are (`seal.ts`). The manifest says which tables and how many rows,
|
|
5
|
+
* never what they hold. docs/content-transfer.md, "Site data".
|
|
6
|
+
*/
|
|
7
|
+
import { openBytes, type Seal, sealBytes } from './seal.ts'
|
|
8
|
+
|
|
9
|
+
/** What an anonymised export does with a column. */
|
|
10
|
+
export type AnonymiseColumn = 'member' | 'email' | 'scrub'
|
|
11
|
+
|
|
12
|
+
/** A table's rule: copied as it is, or with the listed columns rewritten. Unlisted tables travel empty. */
|
|
13
|
+
export type AnonymiseTable = 'keep' | Readonly<Record<string, AnonymiseColumn>>
|
|
14
|
+
|
|
15
|
+
export type AnonymiseRules = Readonly<Record<string, AnonymiseTable>>
|
|
16
|
+
|
|
17
|
+
export const ANONYMISE_COLUMNS: readonly AnonymiseColumn[] = ['member', 'email', 'scrub']
|
|
18
|
+
|
|
19
|
+
export type SiteDataRow = Record<string, unknown>
|
|
20
|
+
|
|
21
|
+
/** One owner's table: `site`, or a bundle's id. */
|
|
22
|
+
export interface SiteDataTable {
|
|
23
|
+
owner: string
|
|
24
|
+
table: string
|
|
25
|
+
rows: SiteDataRow[]
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** What the manifest says of the site data: tables and row counts, never what they hold. */
|
|
29
|
+
export interface SiteDataBlock {
|
|
30
|
+
tables: Array<{ owner: string; table: string; rows: number }>
|
|
31
|
+
/** Every sealed file, in path order. */
|
|
32
|
+
paths: string[]
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export const SITE_DATA_DIR = 'data'
|
|
36
|
+
|
|
37
|
+
const OWNER = /^(?:site|[a-z][a-z0-9-]{1,38})$/
|
|
38
|
+
const TABLE = /^[a-z][a-z0-9_]*$/
|
|
39
|
+
|
|
40
|
+
export const siteDataPath = (owner: string, table: string): string =>
|
|
41
|
+
`${SITE_DATA_DIR}/${owner}/${table}.json`
|
|
42
|
+
|
|
43
|
+
/** `owner.table`, as a person reads which table is meant. */
|
|
44
|
+
export const siteDataName = (owner: string, table: string): string => `${owner}.${table}`
|
|
45
|
+
|
|
46
|
+
function parsePath(path: string): { owner: string; table: string } | undefined {
|
|
47
|
+
const parts = path.split('/')
|
|
48
|
+
if (parts.length !== 3 || parts[0] !== SITE_DATA_DIR) return undefined
|
|
49
|
+
const [, owner, file] = parts as [string, string, string]
|
|
50
|
+
const table = file.endsWith('.json') ? file.slice(0, -'.json'.length) : ''
|
|
51
|
+
return OWNER.test(owner) && TABLE.test(table) ? { owner, table } : undefined
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** The tables sealed under the bundle's seal: the files, and the manifest's block. */
|
|
55
|
+
export function sealSiteData(
|
|
56
|
+
tables: readonly SiteDataTable[],
|
|
57
|
+
{ key }: Seal,
|
|
58
|
+
): { files: Array<{ path: string; bytes: Uint8Array }>; block: SiteDataBlock } {
|
|
59
|
+
const files = tables
|
|
60
|
+
.map((t) => {
|
|
61
|
+
const path = siteDataPath(t.owner, t.table)
|
|
62
|
+
return { path, bytes: sealBytes(key, path, Buffer.from(`${JSON.stringify(t.rows)}\n`)) }
|
|
63
|
+
})
|
|
64
|
+
.sort((a, b) => a.path.localeCompare(b.path))
|
|
65
|
+
return {
|
|
66
|
+
files,
|
|
67
|
+
block: {
|
|
68
|
+
tables: tables.map((t) => ({ owner: t.owner, table: t.table, rows: t.rows.length })),
|
|
69
|
+
paths: files.map((f) => f.path),
|
|
70
|
+
},
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** The tables opened with the bundle's key, in the order the block lists them. */
|
|
75
|
+
export function openSiteData(
|
|
76
|
+
block: SiteDataBlock,
|
|
77
|
+
files: ReadonlyMap<string, Uint8Array>,
|
|
78
|
+
key: Buffer,
|
|
79
|
+
): SiteDataTable[] {
|
|
80
|
+
return block.tables.map(({ owner, table, rows: count }) => {
|
|
81
|
+
const path = siteDataPath(owner, table)
|
|
82
|
+
const sealed = files.get(path)
|
|
83
|
+
if (!sealed) throw new Error(`${path} is missing from the bundle's site data.`)
|
|
84
|
+
const rows = JSON.parse(openBytes(key, path, sealed).toString('utf8')) as unknown
|
|
85
|
+
if (!Array.isArray(rows) || rows.length !== count)
|
|
86
|
+
throw new Error("The bundle's site data does not match the counts its manifest gives.")
|
|
87
|
+
return { owner, table, rows: rows as SiteDataRow[] }
|
|
88
|
+
})
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** A manifest's site data block read without trusting it: undefined when absent or malformed. Pure. */
|
|
92
|
+
export function parseSiteDataBlock(raw: unknown): SiteDataBlock | undefined {
|
|
93
|
+
if (!raw || typeof raw !== 'object') return undefined
|
|
94
|
+
const block = raw as Record<string, unknown>
|
|
95
|
+
if (!Array.isArray(block.tables)) return undefined
|
|
96
|
+
const tables: SiteDataBlock['tables'] = []
|
|
97
|
+
const seen = new Set<string>()
|
|
98
|
+
for (const entry of block.tables as unknown[]) {
|
|
99
|
+
const t = entry as Record<string, unknown> | null
|
|
100
|
+
if (!t || typeof t.owner !== 'string' || typeof t.table !== 'string') return undefined
|
|
101
|
+
const path = siteDataPath(t.owner, t.table)
|
|
102
|
+
if (!parsePath(path) || seen.has(path)) return undefined
|
|
103
|
+
if (!Number.isInteger(t.rows) || (t.rows as number) < 0) return undefined
|
|
104
|
+
seen.add(path)
|
|
105
|
+
tables.push({ owner: t.owner, table: t.table, rows: t.rows as number })
|
|
106
|
+
}
|
|
107
|
+
return { tables, paths: [...seen].sort() }
|
|
108
|
+
}
|
package/src/write.ts
CHANGED
|
@@ -25,6 +25,7 @@ import {
|
|
|
25
25
|
} from './model.ts'
|
|
26
26
|
import { type BundlePeople, sealPeople } from './people.ts'
|
|
27
27
|
import { type Seal, sealBytes } from './seal.ts'
|
|
28
|
+
import { type SiteDataTable, sealSiteData } from './site-data.ts'
|
|
28
29
|
|
|
29
30
|
/** One file of a written bundle, ready to be put on disk or into a store. */
|
|
30
31
|
export interface BundleFile {
|
|
@@ -202,6 +203,7 @@ export function writeManifest(manifest: BundleManifest): string {
|
|
|
202
203
|
? { carries: sortedCarries(manifest.carries) }
|
|
203
204
|
: {}),
|
|
204
205
|
...(manifest.people ? { people: manifest.people } : {}),
|
|
206
|
+
...(manifest.siteData ? { siteData: manifest.siteData } : {}),
|
|
205
207
|
...(manifest.seal ? { seal: manifest.seal } : {}),
|
|
206
208
|
...(manifest.content ? { content: manifest.content } : {}),
|
|
207
209
|
...(manifest.scope ? { scope: manifest.scope } : {}),
|
|
@@ -303,6 +305,8 @@ export interface WriteOptions {
|
|
|
303
305
|
seal?: Seal
|
|
304
306
|
/** Members and users: always sealed. */
|
|
305
307
|
people?: BundlePeople
|
|
308
|
+
/** The site's and bundles' own tables: always sealed. */
|
|
309
|
+
siteData?: readonly SiteDataTable[]
|
|
306
310
|
}
|
|
307
311
|
|
|
308
312
|
/** Without values, media files or usernames: what `omitted` writes. Pure. */
|
|
@@ -380,8 +384,8 @@ function planBundle(
|
|
|
380
384
|
): { manifest: BundleManifest; files: BundleFile[]; blobs: string[]; sealing: boolean } {
|
|
381
385
|
const seal = options.seal
|
|
382
386
|
const content = options.content ?? (seal ? 'sealed' : 'open')
|
|
383
|
-
if ((content === 'sealed' || options.people) && !seal)
|
|
384
|
-
throw new Error('Sealed content and
|
|
387
|
+
if ((content === 'sealed' || options.people || options.siteData) && !seal)
|
|
388
|
+
throw new Error('Sealed content, people and site data need a seal: give a passphrase.')
|
|
385
389
|
const sealing = content === 'sealed'
|
|
386
390
|
const source = content === 'omitted' ? set.nodes.map(withoutContent) : set.nodes
|
|
387
391
|
const carriedSizes: ReadonlyMap<string, number> = content === 'omitted' ? new Map() : blobSizes
|
|
@@ -411,6 +415,7 @@ function planBundle(
|
|
|
411
415
|
}
|
|
412
416
|
|
|
413
417
|
const people = options.people && seal ? sealPeople(options.people, seal) : undefined
|
|
418
|
+
const siteData = options.siteData && seal ? sealSiteData(options.siteData, seal) : undefined
|
|
414
419
|
const manifest: BundleManifest = {
|
|
415
420
|
...set.manifest,
|
|
416
421
|
formatVersion: BUNDLE_FORMAT_VERSION,
|
|
@@ -421,7 +426,8 @@ function planBundle(
|
|
|
421
426
|
}),
|
|
422
427
|
carries: extra.length > 0 ? carries : undefined,
|
|
423
428
|
people: people?.block,
|
|
424
|
-
|
|
429
|
+
siteData: siteData?.block,
|
|
430
|
+
seal: seal && (sealing || people || siteData) ? seal.params : undefined,
|
|
425
431
|
content: content === 'open' ? undefined : content,
|
|
426
432
|
integrity: '',
|
|
427
433
|
}
|
|
@@ -431,7 +437,14 @@ function planBundle(
|
|
|
431
437
|
: []
|
|
432
438
|
return {
|
|
433
439
|
manifest,
|
|
434
|
-
files: [
|
|
440
|
+
files: [
|
|
441
|
+
...nodes,
|
|
442
|
+
...sealedFiles,
|
|
443
|
+
...extra,
|
|
444
|
+
...(people?.files ?? []),
|
|
445
|
+
...(siteData?.files ?? []),
|
|
446
|
+
...media,
|
|
447
|
+
],
|
|
435
448
|
blobs: [...carriedSizes.keys()].sort((a, b) => a.localeCompare(b)),
|
|
436
449
|
sealing,
|
|
437
450
|
}
|