@cogenta/export 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +373 -0
- package/dist/backup.d.ts +64 -0
- package/dist/backup.d.ts.map +1 -0
- package/dist/backup.js +79 -0
- package/dist/backup.js.map +1 -0
- package/dist/content-export.d.ts +49 -0
- package/dist/content-export.d.ts.map +1 -0
- package/dist/content-export.js +197 -0
- package/dist/content-export.js.map +1 -0
- package/dist/content-import.d.ts +44 -0
- package/dist/content-import.d.ts.map +1 -0
- package/dist/content-import.js +221 -0
- package/dist/content-import.js.map +1 -0
- package/dist/crypto.d.ts +18 -0
- package/dist/crypto.d.ts.map +1 -0
- package/dist/crypto.js +151 -0
- package/dist/crypto.js.map +1 -0
- package/dist/format.d.ts +133 -0
- package/dist/format.d.ts.map +1 -0
- package/dist/format.js +71 -0
- package/dist/format.js.map +1 -0
- package/dist/gdpr.d.ts +79 -0
- package/dist/gdpr.d.ts.map +1 -0
- package/dist/gdpr.js +84 -0
- package/dist/gdpr.js.map +1 -0
- package/dist/index.d.ts +24 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +24 -0
- package/dist/index.js.map +1 -0
- package/dist/media-export.d.ts +33 -0
- package/dist/media-export.d.ts.map +1 -0
- package/dist/media-export.js +64 -0
- package/dist/media-export.js.map +1 -0
- package/dist/restore.d.ts +53 -0
- package/dist/restore.d.ts.map +1 -0
- package/dist/restore.js +173 -0
- package/dist/restore.js.map +1 -0
- package/dist/tables.d.ts +28 -0
- package/dist/tables.d.ts.map +1 -0
- package/dist/tables.js +26 -0
- package/dist/tables.js.map +1 -0
- package/dist/zip-reader.d.ts +13 -0
- package/dist/zip-reader.d.ts.map +1 -0
- package/dist/zip-reader.js +97 -0
- package/dist/zip-reader.js.map +1 -0
- package/dist/zip-writer.d.ts +16 -0
- package/dist/zip-writer.d.ts.map +1 -0
- package/dist/zip-writer.js +143 -0
- package/dist/zip-writer.js.map +1 -0
- package/package.json +44 -0
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
import { orderByDependency, } from '@cogenta/schema';
|
|
2
|
+
import { EXPORT_FORMAT, EXPORT_FORMAT_VERSION, encodeRecord, } from './format.js';
|
|
3
|
+
const PAGE_SIZE = 200;
|
|
4
|
+
function inSelection(selection, name) {
|
|
5
|
+
if (selection?.collections === undefined)
|
|
6
|
+
return true;
|
|
7
|
+
return selection.collections.includes(name);
|
|
8
|
+
}
|
|
9
|
+
function mediaIdsOf(value) {
|
|
10
|
+
if (typeof value === 'string')
|
|
11
|
+
return [value];
|
|
12
|
+
if (Array.isArray(value)) {
|
|
13
|
+
return value.filter((item) => typeof item === 'string');
|
|
14
|
+
}
|
|
15
|
+
return [];
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Streams a whole export as NDJSON text, one `ExportRecord` per line, and
|
|
19
|
+
* resolves to the exact counts and the set of referenced media ids once every
|
|
20
|
+
* record has been produced.
|
|
21
|
+
*
|
|
22
|
+
* The manifest is the **first** line rather than a sibling file: a two-file
|
|
23
|
+
* export can drift out of step (one gets copied, the other does not), one
|
|
24
|
+
* stream cannot. Its `counts` are written as zero and are informational only
|
|
25
|
+
* on this first pass — a reader that needs exact counts before it starts
|
|
26
|
+
* reads `exportContentToLines`'s returned `ExportResult` instead, which is
|
|
27
|
+
* what `cogenta export` writes back into the manifest once the file is
|
|
28
|
+
* seekable.
|
|
29
|
+
*/
|
|
30
|
+
export async function* exportContent(options) {
|
|
31
|
+
const selection = options.selection ?? {};
|
|
32
|
+
const canReadCollection = options.canReadCollection ?? (() => true);
|
|
33
|
+
const canReadTaxonomy = options.canReadTaxonomy ?? (() => true);
|
|
34
|
+
const now = options.now ?? (() => new Date());
|
|
35
|
+
const counts = { entries: 0, terms: 0, menus: 0, menuItems: 0, redirects: 0, mediaRefs: 0 };
|
|
36
|
+
const mediaSeen = new Set();
|
|
37
|
+
const manifest = {
|
|
38
|
+
kind: 'manifest',
|
|
39
|
+
format: EXPORT_FORMAT,
|
|
40
|
+
version: EXPORT_FORMAT_VERSION,
|
|
41
|
+
createdAt: now().toISOString(),
|
|
42
|
+
site: options.site,
|
|
43
|
+
selection,
|
|
44
|
+
counts,
|
|
45
|
+
};
|
|
46
|
+
yield encodeRecord(manifest);
|
|
47
|
+
// Taxonomies before collections, and collections in dependency order
|
|
48
|
+
// (`orderByDependency`, the same helper `createSchemaTables` uses to
|
|
49
|
+
// decide table creation order): a `f.taxonomy()` or `f.relation()` field
|
|
50
|
+
// is a foreign key, and `content-import.ts` replays this exact stream in a
|
|
51
|
+
// single forward pass rather than buffering it, so whatever it needs to
|
|
52
|
+
// already exist has to have been emitted already.
|
|
53
|
+
for (const taxonomy of options.taxonomies) {
|
|
54
|
+
if (!canReadTaxonomy(taxonomy))
|
|
55
|
+
continue;
|
|
56
|
+
const store = options.taxonomyStoreFor(taxonomy);
|
|
57
|
+
const terms = await store.list();
|
|
58
|
+
for (const term of terms) {
|
|
59
|
+
counts.terms += 1;
|
|
60
|
+
const record = {
|
|
61
|
+
kind: 'term',
|
|
62
|
+
taxonomy: taxonomy.name,
|
|
63
|
+
id: term.id,
|
|
64
|
+
slug: term.slug,
|
|
65
|
+
parent: term.parent,
|
|
66
|
+
position: term.position,
|
|
67
|
+
labels: term.labels,
|
|
68
|
+
};
|
|
69
|
+
yield encodeRecord(record);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
for (const collection of orderByDependency(options.collections)) {
|
|
73
|
+
if (!inSelection(selection, collection.name))
|
|
74
|
+
continue;
|
|
75
|
+
if (!canReadCollection(collection))
|
|
76
|
+
continue;
|
|
77
|
+
const mediaFields = Object.entries(collection.fields)
|
|
78
|
+
.filter(([, field]) => field.kind === 'media')
|
|
79
|
+
.map(([name]) => name);
|
|
80
|
+
const store = options.storeFor(collection);
|
|
81
|
+
let cursor;
|
|
82
|
+
for (;;) {
|
|
83
|
+
const page = await store.list({
|
|
84
|
+
state: 'working',
|
|
85
|
+
limit: PAGE_SIZE,
|
|
86
|
+
trashed: selection.includeTrashed === true ? 'include' : 'exclude',
|
|
87
|
+
...(cursor === undefined ? {} : { cursor }),
|
|
88
|
+
});
|
|
89
|
+
for (const entry of page.items) {
|
|
90
|
+
if (selection.statuses !== undefined && !selection.statuses.includes(entry.status)) {
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
if (selection.locales !== undefined && !selection.locales.includes(entry.locale)) {
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
96
|
+
if (selection.from !== undefined && entry.updatedAt < selection.from)
|
|
97
|
+
continue;
|
|
98
|
+
if (selection.to !== undefined && entry.updatedAt > selection.to)
|
|
99
|
+
continue;
|
|
100
|
+
counts.entries += 1;
|
|
101
|
+
yield encodeRecord({
|
|
102
|
+
kind: 'entry',
|
|
103
|
+
collection: collection.name,
|
|
104
|
+
id: entry.id,
|
|
105
|
+
locale: entry.locale,
|
|
106
|
+
translationOf: entry.translationOf,
|
|
107
|
+
status: entry.status,
|
|
108
|
+
deletedAt: entry.deletedAt,
|
|
109
|
+
version: entry.version,
|
|
110
|
+
provenance: entry.provenance,
|
|
111
|
+
provenanceDetail: entry.provenanceDetail ?? null,
|
|
112
|
+
createdAt: entry.createdAt,
|
|
113
|
+
updatedAt: entry.updatedAt,
|
|
114
|
+
createdBy: entry.createdBy,
|
|
115
|
+
updatedBy: entry.updatedBy,
|
|
116
|
+
publishedAt: entry.publishedAt,
|
|
117
|
+
values: entry.values,
|
|
118
|
+
blocks: entry.blocks,
|
|
119
|
+
});
|
|
120
|
+
for (const field of mediaFields) {
|
|
121
|
+
for (const id of mediaIdsOf(entry.values[field]))
|
|
122
|
+
mediaSeen.add(id);
|
|
123
|
+
}
|
|
124
|
+
if (selection.includeHistory === true) {
|
|
125
|
+
const history = await store.history(entry.id, { trashed: 'include' });
|
|
126
|
+
for (const version of history) {
|
|
127
|
+
const record = {
|
|
128
|
+
kind: 'version',
|
|
129
|
+
collection: collection.name,
|
|
130
|
+
entryId: entry.id,
|
|
131
|
+
version: version.version,
|
|
132
|
+
status: version.status,
|
|
133
|
+
createdAt: version.createdAt,
|
|
134
|
+
createdBy: version.createdBy,
|
|
135
|
+
};
|
|
136
|
+
yield encodeRecord(record);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
if (!page.hasMore || page.nextCursor === null)
|
|
141
|
+
break;
|
|
142
|
+
cursor = page.nextCursor;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
if (options.menus !== undefined) {
|
|
146
|
+
const menus = await options.menus.list();
|
|
147
|
+
for (const menu of menus) {
|
|
148
|
+
counts.menus += 1;
|
|
149
|
+
const record = {
|
|
150
|
+
kind: 'menu',
|
|
151
|
+
id: menu.id,
|
|
152
|
+
name: menu.name,
|
|
153
|
+
locale: menu.locale,
|
|
154
|
+
label: menu.label,
|
|
155
|
+
};
|
|
156
|
+
yield encodeRecord(record);
|
|
157
|
+
const items = await options.menus.listItems(menu.id);
|
|
158
|
+
for (const item of items) {
|
|
159
|
+
counts.menuItems += 1;
|
|
160
|
+
const itemRecord = {
|
|
161
|
+
kind: 'menu-item',
|
|
162
|
+
id: item.id,
|
|
163
|
+
menuId: menu.id,
|
|
164
|
+
parent: item.parent,
|
|
165
|
+
position: item.position,
|
|
166
|
+
label: item.label,
|
|
167
|
+
itemKind: item.kind,
|
|
168
|
+
url: item.url,
|
|
169
|
+
targetCollection: item.targetCollection,
|
|
170
|
+
targetEntryId: item.targetEntryId,
|
|
171
|
+
openInNewTab: item.openInNewTab,
|
|
172
|
+
};
|
|
173
|
+
yield encodeRecord(itemRecord);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
if (options.redirects !== undefined) {
|
|
178
|
+
const redirects = await options.redirects.list({ limit: 100_000 });
|
|
179
|
+
for (const redirect of redirects) {
|
|
180
|
+
counts.redirects += 1;
|
|
181
|
+
const record = {
|
|
182
|
+
kind: 'redirect',
|
|
183
|
+
from: redirect.from,
|
|
184
|
+
to: redirect.to,
|
|
185
|
+
status: redirect.status,
|
|
186
|
+
collection: redirect.collection,
|
|
187
|
+
entryId: redirect.entryId,
|
|
188
|
+
locale: redirect.locale,
|
|
189
|
+
reason: redirect.reason,
|
|
190
|
+
};
|
|
191
|
+
yield encodeRecord(record);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
counts.mediaRefs = mediaSeen.size;
|
|
195
|
+
return { counts, mediaIds: [...mediaSeen] };
|
|
196
|
+
}
|
|
197
|
+
//# sourceMappingURL=content-export.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"content-export.js","sourceRoot":"","sources":["../src/content-export.ts"],"names":[],"mappings":"AACA,OAAO,EAIL,iBAAiB,GAIlB,MAAM,iBAAiB,CAAA;AACxB,OAAO,EACL,aAAa,EACb,qBAAqB,EAQrB,YAAY,GACb,MAAM,aAAa,CAAA;AAgCpB,MAAM,SAAS,GAAG,GAAG,CAAA;AAErB,SAAS,WAAW,CAAC,SAAsC,EAAE,IAAY;IACvE,IAAI,SAAS,EAAE,WAAW,KAAK,SAAS;QAAE,OAAO,IAAI,CAAA;IACrD,OAAO,SAAS,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;AAC7C,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,CAAC,KAAK,CAAC,CAAA;IAC7C,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAA;IACzE,CAAC;IACD,OAAO,EAAE,CAAA;AACX,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,aAAa,CAClC,OAA6B;IAE7B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,EAAE,CAAA;IACzC,MAAM,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAA;IACnE,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAA;IAC/D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAA;IAE7C,MAAM,MAAM,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAA;IAC3F,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAA;IAEnC,MAAM,QAAQ,GAAyB;QACrC,IAAI,EAAE,UAAU;QAChB,MAAM,EAAE,aAAa;QACrB,OAAO,EAAE,qBAAqB;QAC9B,SAAS,EAAE,GAAG,EAAE,CAAC,WAAW,EAAE;QAC9B,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,SAAS;QACT,MAAM;KACP,CAAA;IACD,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAA;IAE5B,qEAAqE;IACrE,qEAAqE;IACrE,yEAAyE;IACzE,2EAA2E;IAC3E,wEAAwE;IACxE,kDAAkD;IAClD,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QAC1C,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC;YAAE,SAAQ;QAExC,MAAM,KAAK,GAAG,OAAO,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAA;QAChD,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,IAAI,EAAE,CAAA;QAChC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,CAAC,KAAK,IAAI,CAAC,CAAA;YACjB,MAAM,MAAM,GAAqB;gBAC/B,IAAI,EAAE,MAAM;gBACZ,QAAQ,EAAE,QAAQ,CAAC,IAAI;gBACvB,EAAE,EAAE,IAAI,CAAC,EAAE;gBACX,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,MAAM,EAAE,IAAI,CAAC,MAAM;aACpB,CAAA;YACD,MAAM,YAAY,CAAC,MAAM,CAAC,CAAA;QAC5B,CAAC;IACH,CAAC;IAED,KAAK,MAAM,UAAU,IAAI,iBAAiB,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;QAChE,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,UAAU,CAAC,IAAI,CAAC;YAAE,SAAQ;QACtD,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC;YAAE,SAAQ;QAE5C,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC;aAClD,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,OAAO,CAAC;aAC7C,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAA;QAExB,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAA;QAC1C,IAAI,MAA0B,CAAA;QAC9B,SAAS,CAAC;YACR,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC;gBAC5B,KAAK,EAAE,SAAS;gBAChB,KAAK,EAAE,SAAS;gBAChB,OAAO,EAAE,SAAS,CAAC,cAAc,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;gBAClE,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC;aAC5C,CAAC,CAAA;YAEF,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC/B,IAAI,SAAS,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;oBACnF,SAAQ;gBACV,CAAC;gBACD,IAAI,SAAS,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;oBACjF,SAAQ;gBACV,CAAC;gBACD,IAAI,SAAS,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,SAAS,GAAG,SAAS,CAAC,IAAI;oBAAE,SAAQ;gBAC9E,IAAI,SAAS,CAAC,EAAE,KAAK,SAAS,IAAI,KAAK,CAAC,SAAS,GAAG,SAAS,CAAC,EAAE;oBAAE,SAAQ;gBAE1E,MAAM,CAAC,OAAO,IAAI,CAAC,CAAA;gBACnB,MAAM,YAAY,CAAC;oBACjB,IAAI,EAAE,OAAO;oBACb,UAAU,EAAE,UAAU,CAAC,IAAI;oBAC3B,EAAE,EAAE,KAAK,CAAC,EAAE;oBACZ,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,aAAa,EAAE,KAAK,CAAC,aAAa;oBAClC,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,SAAS,EAAE,KAAK,CAAC,SAAS;oBAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,IAAI,IAAI;oBAChD,SAAS,EAAE,KAAK,CAAC,SAAS;oBAC1B,SAAS,EAAE,KAAK,CAAC,SAAS;oBAC1B,SAAS,EAAE,KAAK,CAAC,SAAS;oBAC1B,SAAS,EAAE,KAAK,CAAC,SAAS;oBAC1B,WAAW,EAAE,KAAK,CAAC,WAAW;oBAC9B,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,MAAM,EAAE,KAAK,CAAC,MAAM;iBACrB,CAAC,CAAA;gBAEF,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;oBAChC,KAAK,MAAM,EAAE,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;wBAAE,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;gBACrE,CAAC;gBAED,IAAI,SAAS,CAAC,cAAc,KAAK,IAAI,EAAE,CAAC;oBACtC,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAA;oBACrE,KAAK,MAAM,OAAO,IAAI,OAAO,EAAE,CAAC;wBAC9B,MAAM,MAAM,GAAwB;4BAClC,IAAI,EAAE,SAAS;4BACf,UAAU,EAAE,UAAU,CAAC,IAAI;4BAC3B,OAAO,EAAE,KAAK,CAAC,EAAE;4BACjB,OAAO,EAAE,OAAO,CAAC,OAAO;4BACxB,MAAM,EAAE,OAAO,CAAC,MAAM;4BACtB,SAAS,EAAE,OAAO,CAAC,SAAS;4BAC5B,SAAS,EAAE,OAAO,CAAC,SAAS;yBAC7B,CAAA;wBACD,MAAM,YAAY,CAAC,MAAM,CAAC,CAAA;oBAC5B,CAAC;gBACH,CAAC;YACH,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI;gBAAE,MAAK;YACpD,MAAM,GAAG,IAAI,CAAC,UAAU,CAAA;QAC1B,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAA;QACxC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,CAAC,KAAK,IAAI,CAAC,CAAA;YACjB,MAAM,MAAM,GAAqB;gBAC/B,IAAI,EAAE,MAAM;gBACZ,EAAE,EAAE,IAAI,CAAC,EAAE;gBACX,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;aAClB,CAAA;YACD,MAAM,YAAY,CAAC,MAAM,CAAC,CAAA;YAE1B,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACpD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,MAAM,CAAC,SAAS,IAAI,CAAC,CAAA;gBACrB,MAAM,UAAU,GAAyB;oBACvC,IAAI,EAAE,WAAW;oBACjB,EAAE,EAAE,IAAI,CAAC,EAAE;oBACX,MAAM,EAAE,IAAI,CAAC,EAAE;oBACf,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;oBACnB,GAAG,EAAE,IAAI,CAAC,GAAG;oBACb,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;oBACvC,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,YAAY,EAAE,IAAI,CAAC,YAAY;iBAChC,CAAA;gBACD,MAAM,YAAY,CAAC,UAAU,CAAC,CAAA;YAChC,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QACpC,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAA;QAClE,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,CAAC,SAAS,IAAI,CAAC,CAAA;YACrB,MAAM,MAAM,GAAyB;gBACnC,IAAI,EAAE,UAAU;gBAChB,IAAI,EAAE,QAAQ,CAAC,IAAI;gBACnB,EAAE,EAAE,QAAQ,CAAC,EAAE;gBACf,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,UAAU,EAAE,QAAQ,CAAC,UAAU;gBAC/B,OAAO,EAAE,QAAQ,CAAC,OAAO;gBACzB,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,MAAM,EAAE,QAAQ,CAAC,MAAM;aACxB,CAAA;YACD,MAAM,YAAY,CAAC,MAAM,CAAC,CAAA;QAC5B,CAAC;IACH,CAAC;IAED,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC,IAAI,CAAA;IACjC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,SAAS,CAAC,EAAE,CAAA;AAC7C,CAAC"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { CollectionDefinition, ContentStore, MenuStore, RedirectStore, TaxonomyDefinition, TaxonomyStore } from '@cogenta/schema';
|
|
2
|
+
export interface ImportContentOptions {
|
|
3
|
+
readonly collections: readonly CollectionDefinition[];
|
|
4
|
+
readonly taxonomies: readonly TaxonomyDefinition[];
|
|
5
|
+
readonly storeFor: (collection: CollectionDefinition) => ContentStore;
|
|
6
|
+
readonly taxonomyStoreFor: (taxonomy: TaxonomyDefinition) => TaxonomyStore;
|
|
7
|
+
readonly menus?: MenuStore;
|
|
8
|
+
readonly redirects?: RedirectStore;
|
|
9
|
+
/**
|
|
10
|
+
* `'skip'` (the default): a record whose id already exists in the target is
|
|
11
|
+
* left alone and counted in `report.skipped`. `'fail'` stops the whole
|
|
12
|
+
* import at the first collision — appropriate for "restore a content
|
|
13
|
+
* export into an empty site", where any collision means the site was not
|
|
14
|
+
* actually empty and the caller should know before more damage is done.
|
|
15
|
+
*/
|
|
16
|
+
readonly onConflict?: 'skip' | 'fail';
|
|
17
|
+
}
|
|
18
|
+
export interface ImportReport {
|
|
19
|
+
readonly entries: number;
|
|
20
|
+
readonly terms: number;
|
|
21
|
+
readonly menus: number;
|
|
22
|
+
readonly menuItems: number;
|
|
23
|
+
readonly redirects: number;
|
|
24
|
+
readonly skipped: number;
|
|
25
|
+
readonly errors: readonly {
|
|
26
|
+
readonly kind: string;
|
|
27
|
+
readonly id: string;
|
|
28
|
+
readonly message: string;
|
|
29
|
+
}[];
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Applies one line of an export stream at a time — the counterpart of
|
|
33
|
+
* `exportContent`, replaying the exact ordering it committed to (taxonomies,
|
|
34
|
+
* then collections in dependency order, then menus, then redirects) so a
|
|
35
|
+
* forward-only pass never needs the row it has not seen yet.
|
|
36
|
+
*
|
|
37
|
+
* The one exception is a translation whose source entry appears later than
|
|
38
|
+
* itself in the stream (possible when a store's default list order is not
|
|
39
|
+
* strictly creation order): such an entry is deferred to a small pending
|
|
40
|
+
* queue — bounded by how many translations are out of order, never by the
|
|
41
|
+
* size of the export — and retried once the stream ends.
|
|
42
|
+
*/
|
|
43
|
+
export declare function importContent(lines: AsyncIterable<string> | Iterable<string>, options: ImportContentOptions): Promise<ImportReport>;
|
|
44
|
+
//# sourceMappingURL=content-import.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"content-import.d.ts","sourceRoot":"","sources":["../src/content-import.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,oBAAoB,EACpB,YAAY,EACZ,SAAS,EACT,aAAa,EACb,kBAAkB,EAClB,aAAa,EACd,MAAM,iBAAiB,CAAA;AAGxB,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,WAAW,EAAE,SAAS,oBAAoB,EAAE,CAAA;IACrD,QAAQ,CAAC,UAAU,EAAE,SAAS,kBAAkB,EAAE,CAAA;IAClD,QAAQ,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,oBAAoB,KAAK,YAAY,CAAA;IACrE,QAAQ,CAAC,gBAAgB,EAAE,CAAC,QAAQ,EAAE,kBAAkB,KAAK,aAAa,CAAA;IAC1E,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,CAAA;IAC1B,QAAQ,CAAC,SAAS,CAAC,EAAE,aAAa,CAAA;IAClC;;;;;;OAMG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;CACtC;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,MAAM,EAAE,SAAS;QACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;QACrB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;QACnB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;KACzB,EAAE,CAAA;CACJ;AAcD;;;;;;;;;;;GAWG;AACH,wBAAsB,aAAa,CACjC,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,EAC/C,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,YAAY,CAAC,CAmNvB"}
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
import { CogentaError } from '@cogenta/core';
|
|
2
|
+
import { assertManifest, decodeRecord } from './format.js';
|
|
3
|
+
function emptyReport() {
|
|
4
|
+
return { entries: 0, terms: 0, menus: 0, menuItems: 0, redirects: 0, skipped: 0, errors: [] };
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* Applies one line of an export stream at a time — the counterpart of
|
|
8
|
+
* `exportContent`, replaying the exact ordering it committed to (taxonomies,
|
|
9
|
+
* then collections in dependency order, then menus, then redirects) so a
|
|
10
|
+
* forward-only pass never needs the row it has not seen yet.
|
|
11
|
+
*
|
|
12
|
+
* The one exception is a translation whose source entry appears later than
|
|
13
|
+
* itself in the stream (possible when a store's default list order is not
|
|
14
|
+
* strictly creation order): such an entry is deferred to a small pending
|
|
15
|
+
* queue — bounded by how many translations are out of order, never by the
|
|
16
|
+
* size of the export — and retried once the stream ends.
|
|
17
|
+
*/
|
|
18
|
+
export async function importContent(lines, options) {
|
|
19
|
+
const onConflict = options.onConflict ?? 'skip';
|
|
20
|
+
const report = emptyReport();
|
|
21
|
+
const collectionByName = new Map(options.collections.map((c) => [c.name, c]));
|
|
22
|
+
const taxonomyByName = new Map(options.taxonomies.map((t) => [t.name, t]));
|
|
23
|
+
const pendingTranslations = [];
|
|
24
|
+
let sawManifest = false;
|
|
25
|
+
let lineNumber = 0;
|
|
26
|
+
const applyEntry = async (record, allowDefer = true) => {
|
|
27
|
+
const collection = collectionByName.get(record.collection);
|
|
28
|
+
if (collection === undefined) {
|
|
29
|
+
report.errors.push({
|
|
30
|
+
kind: 'entry',
|
|
31
|
+
id: record.id,
|
|
32
|
+
message: `Collection "${record.collection}" does not exist in the target site.`,
|
|
33
|
+
});
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
const store = options.storeFor(collection);
|
|
37
|
+
const existing = await store.read(record.id, { state: 'working', trashed: 'include' });
|
|
38
|
+
if (existing !== null) {
|
|
39
|
+
if (onConflict === 'fail') {
|
|
40
|
+
throw new CogentaError({
|
|
41
|
+
code: 'RESTORE_CONFLICT',
|
|
42
|
+
message: `Entry "${record.id}" of "${record.collection}" already exists in the target.`,
|
|
43
|
+
hint: 'Import into an empty site, or pass onConflict: "skip".',
|
|
44
|
+
details: { collection: record.collection, id: record.id },
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
report.skipped += 1;
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
50
|
+
if (record.translationOf !== null) {
|
|
51
|
+
const source = await store.read(record.translationOf, {
|
|
52
|
+
state: 'working',
|
|
53
|
+
trashed: 'include',
|
|
54
|
+
});
|
|
55
|
+
if (source === null) {
|
|
56
|
+
if (allowDefer)
|
|
57
|
+
pendingTranslations.push(record);
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
await store.create({
|
|
62
|
+
id: record.id,
|
|
63
|
+
locale: record.locale,
|
|
64
|
+
translationOf: record.translationOf,
|
|
65
|
+
status: record.status,
|
|
66
|
+
createdBy: record.createdBy,
|
|
67
|
+
provenance: record.provenance,
|
|
68
|
+
...(record.provenanceDetail === null ? {} : { provenanceDetail: record.provenanceDetail }),
|
|
69
|
+
values: record.values,
|
|
70
|
+
blocks: record.blocks,
|
|
71
|
+
});
|
|
72
|
+
if (record.deletedAt !== null)
|
|
73
|
+
await store.delete(record.id);
|
|
74
|
+
report.entries += 1;
|
|
75
|
+
};
|
|
76
|
+
const applyRecord = async (record) => {
|
|
77
|
+
switch (record.kind) {
|
|
78
|
+
case 'manifest':
|
|
79
|
+
assertManifest(record);
|
|
80
|
+
sawManifest = true;
|
|
81
|
+
return;
|
|
82
|
+
case 'entry':
|
|
83
|
+
await applyEntry(record);
|
|
84
|
+
return;
|
|
85
|
+
case 'version':
|
|
86
|
+
// Version history is informational (task 1's "avec ou sans historique
|
|
87
|
+
// de versions"); it is exported for archival reading, not replayed —
|
|
88
|
+
// `ContentStore` has no "insert a past version" primitive, and
|
|
89
|
+
// fabricating one would misreport who wrote what, when.
|
|
90
|
+
return;
|
|
91
|
+
case 'term': {
|
|
92
|
+
const taxonomy = taxonomyByName.get(record.taxonomy);
|
|
93
|
+
if (taxonomy === undefined) {
|
|
94
|
+
report.errors.push({
|
|
95
|
+
kind: 'term',
|
|
96
|
+
id: record.id,
|
|
97
|
+
message: `Taxonomy "${record.taxonomy}" does not exist in the target site.`,
|
|
98
|
+
});
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
const store = options.taxonomyStoreFor(taxonomy);
|
|
102
|
+
const existing = await store.read(record.id);
|
|
103
|
+
if (existing !== null) {
|
|
104
|
+
if (onConflict === 'fail') {
|
|
105
|
+
throw new CogentaError({
|
|
106
|
+
code: 'RESTORE_CONFLICT',
|
|
107
|
+
message: `Term "${record.id}" of "${record.taxonomy}" already exists in the target.`,
|
|
108
|
+
hint: 'Import into an empty site, or pass onConflict: "skip".',
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
report.skipped += 1;
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
114
|
+
await store.create({
|
|
115
|
+
id: record.id,
|
|
116
|
+
slug: record.slug,
|
|
117
|
+
labels: record.labels,
|
|
118
|
+
parent: record.parent,
|
|
119
|
+
position: record.position,
|
|
120
|
+
});
|
|
121
|
+
report.terms += 1;
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
case 'menu': {
|
|
125
|
+
if (options.menus === undefined)
|
|
126
|
+
return;
|
|
127
|
+
const existing = await options.menus.read(record.id);
|
|
128
|
+
if (existing !== null) {
|
|
129
|
+
if (onConflict === 'fail') {
|
|
130
|
+
throw new CogentaError({
|
|
131
|
+
code: 'RESTORE_CONFLICT',
|
|
132
|
+
message: `Menu "${record.id}" already exists in the target.`,
|
|
133
|
+
hint: 'Import into an empty site, or pass onConflict: "skip".',
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
report.skipped += 1;
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
await options.menus.create({
|
|
140
|
+
id: record.id,
|
|
141
|
+
name: record.name,
|
|
142
|
+
locale: record.locale,
|
|
143
|
+
label: record.label,
|
|
144
|
+
});
|
|
145
|
+
report.menus += 1;
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
148
|
+
case 'menu-item': {
|
|
149
|
+
if (options.menus === undefined)
|
|
150
|
+
return;
|
|
151
|
+
const existing = await options.menus.readItem(record.id);
|
|
152
|
+
if (existing !== null) {
|
|
153
|
+
report.skipped += 1;
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
156
|
+
await options.menus.createItem(record.menuId, {
|
|
157
|
+
id: record.id,
|
|
158
|
+
label: record.label,
|
|
159
|
+
kind: record.itemKind,
|
|
160
|
+
parent: record.parent,
|
|
161
|
+
targetCollection: record.targetCollection,
|
|
162
|
+
targetEntryId: record.targetEntryId,
|
|
163
|
+
url: record.url,
|
|
164
|
+
position: record.position,
|
|
165
|
+
openInNewTab: record.openInNewTab,
|
|
166
|
+
});
|
|
167
|
+
report.menuItems += 1;
|
|
168
|
+
return;
|
|
169
|
+
}
|
|
170
|
+
case 'redirect': {
|
|
171
|
+
if (options.redirects === undefined)
|
|
172
|
+
return;
|
|
173
|
+
await options.redirects.add({
|
|
174
|
+
from: record.from,
|
|
175
|
+
to: record.to,
|
|
176
|
+
status: record.status,
|
|
177
|
+
reason: 'import',
|
|
178
|
+
...(record.collection === null ? {} : { collection: record.collection }),
|
|
179
|
+
...(record.entryId === null ? {} : { entryId: record.entryId }),
|
|
180
|
+
...(record.locale === null ? {} : { locale: record.locale }),
|
|
181
|
+
});
|
|
182
|
+
report.redirects += 1;
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
default:
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
188
|
+
};
|
|
189
|
+
for await (const line of lines) {
|
|
190
|
+
if (line.trim().length === 0)
|
|
191
|
+
continue;
|
|
192
|
+
lineNumber += 1;
|
|
193
|
+
const record = decodeRecord(line, lineNumber);
|
|
194
|
+
await applyRecord(record);
|
|
195
|
+
}
|
|
196
|
+
if (!sawManifest) {
|
|
197
|
+
throw new CogentaError({
|
|
198
|
+
code: 'EXPORT_FORMAT_INVALID',
|
|
199
|
+
message: 'The import stream never carried a manifest record.',
|
|
200
|
+
hint: 'The first line of a Cogenta export must be `{"kind":"manifest",…}`.',
|
|
201
|
+
});
|
|
202
|
+
}
|
|
203
|
+
// Retry deferred translations, in the order they were deferred. A source
|
|
204
|
+
// that is itself still missing after this pass is a genuinely broken
|
|
205
|
+
// export (a translation whose source was never included) rather than an
|
|
206
|
+
// ordering artefact, and is reported as an error instead of retried forever.
|
|
207
|
+
for (const record of [...pendingTranslations]) {
|
|
208
|
+
const entriesBefore = report.entries;
|
|
209
|
+
const skippedBefore = report.skipped;
|
|
210
|
+
await applyEntry(record, false);
|
|
211
|
+
if (report.entries === entriesBefore && report.skipped === skippedBefore) {
|
|
212
|
+
report.errors.push({
|
|
213
|
+
kind: 'entry',
|
|
214
|
+
id: record.id,
|
|
215
|
+
message: `Translation source "${record.translationOf}" was never found in the export.`,
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
return report;
|
|
220
|
+
}
|
|
221
|
+
//# sourceMappingURL=content-import.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"content-import.js","sourceRoot":"","sources":["../src/content-import.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAS5C,OAAO,EAAE,cAAc,EAAE,YAAY,EAAqB,MAAM,aAAa,CAAA;AAiC7E,SAAS,WAAW;IASlB,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAA;AAC/F,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,KAA+C,EAC/C,OAA6B;IAE7B,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,MAAM,CAAA;IAC/C,MAAM,MAAM,GAAG,WAAW,EAAE,CAAA;IAE5B,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IAC7E,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IAC1E,MAAM,mBAAmB,GAA+C,EAAE,CAAA;IAC1E,IAAI,WAAW,GAAG,KAAK,CAAA;IACvB,IAAI,UAAU,GAAG,CAAC,CAAA;IAElB,MAAM,UAAU,GAAG,KAAK,EACtB,MAAgD,EAChD,UAAU,GAAG,IAAI,EACF,EAAE;QACjB,MAAM,UAAU,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;QAC1D,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;gBACjB,IAAI,EAAE,OAAO;gBACb,EAAE,EAAE,MAAM,CAAC,EAAE;gBACb,OAAO,EAAE,eAAe,MAAM,CAAC,UAAU,sCAAsC;aAChF,CAAC,CAAA;YACF,OAAM;QACR,CAAC;QAED,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAA;QAC1C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAA;QACtF,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;YACtB,IAAI,UAAU,KAAK,MAAM,EAAE,CAAC;gBAC1B,MAAM,IAAI,YAAY,CAAC;oBACrB,IAAI,EAAE,kBAAkB;oBACxB,OAAO,EAAE,UAAU,MAAM,CAAC,EAAE,SAAS,MAAM,CAAC,UAAU,iCAAiC;oBACvF,IAAI,EAAE,wDAAwD;oBAC9D,OAAO,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE;iBAC1D,CAAC,CAAA;YACJ,CAAC;YACD,MAAM,CAAC,OAAO,IAAI,CAAC,CAAA;YACnB,OAAM;QACR,CAAC;QAED,IAAI,MAAM,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;YAClC,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE;gBACpD,KAAK,EAAE,SAAS;gBAChB,OAAO,EAAE,SAAS;aACnB,CAAC,CAAA;YACF,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;gBACpB,IAAI,UAAU;oBAAE,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;gBAChD,OAAM;YACR,CAAC;QACH,CAAC;QAED,MAAM,KAAK,CAAC,MAAM,CAAC;YACjB,EAAE,EAAE,MAAM,CAAC,EAAE;YACb,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,aAAa,EAAE,MAAM,CAAC,aAAa;YACnC,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,GAAG,CAAC,MAAM,CAAC,gBAAgB,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC1F,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB,CAAC,CAAA;QAEF,IAAI,MAAM,CAAC,SAAS,KAAK,IAAI;YAAE,MAAM,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QAE5D,MAAM,CAAC,OAAO,IAAI,CAAC,CAAA;IACrB,CAAC,CAAA;IAED,MAAM,WAAW,GAAG,KAAK,EAAE,MAAoB,EAAiB,EAAE;QAChE,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;YACpB,KAAK,UAAU;gBACb,cAAc,CAAC,MAAM,CAAC,CAAA;gBACtB,WAAW,GAAG,IAAI,CAAA;gBAClB,OAAM;YACR,KAAK,OAAO;gBACV,MAAM,UAAU,CAAC,MAAM,CAAC,CAAA;gBACxB,OAAM;YACR,KAAK,SAAS;gBACZ,sEAAsE;gBACtE,qEAAqE;gBACrE,+DAA+D;gBAC/D,wDAAwD;gBACxD,OAAM;YACR,KAAK,MAAM,EAAE,CAAC;gBACZ,MAAM,QAAQ,GAAG,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;gBACpD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;oBAC3B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;wBACjB,IAAI,EAAE,MAAM;wBACZ,EAAE,EAAE,MAAM,CAAC,EAAE;wBACb,OAAO,EAAE,aAAa,MAAM,CAAC,QAAQ,sCAAsC;qBAC5E,CAAC,CAAA;oBACF,OAAM;gBACR,CAAC;gBACD,MAAM,KAAK,GAAG,OAAO,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAA;gBAChD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;gBAC5C,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;oBACtB,IAAI,UAAU,KAAK,MAAM,EAAE,CAAC;wBAC1B,MAAM,IAAI,YAAY,CAAC;4BACrB,IAAI,EAAE,kBAAkB;4BACxB,OAAO,EAAE,SAAS,MAAM,CAAC,EAAE,SAAS,MAAM,CAAC,QAAQ,iCAAiC;4BACpF,IAAI,EAAE,wDAAwD;yBAC/D,CAAC,CAAA;oBACJ,CAAC;oBACD,MAAM,CAAC,OAAO,IAAI,CAAC,CAAA;oBACnB,OAAM;gBACR,CAAC;gBACD,MAAM,KAAK,CAAC,MAAM,CAAC;oBACjB,EAAE,EAAE,MAAM,CAAC,EAAE;oBACb,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,QAAQ,EAAE,MAAM,CAAC,QAAQ;iBAC1B,CAAC,CAAA;gBACF,MAAM,CAAC,KAAK,IAAI,CAAC,CAAA;gBACjB,OAAM;YACR,CAAC;YACD,KAAK,MAAM,EAAE,CAAC;gBACZ,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS;oBAAE,OAAM;gBACvC,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;gBACpD,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;oBACtB,IAAI,UAAU,KAAK,MAAM,EAAE,CAAC;wBAC1B,MAAM,IAAI,YAAY,CAAC;4BACrB,IAAI,EAAE,kBAAkB;4BACxB,OAAO,EAAE,SAAS,MAAM,CAAC,EAAE,iCAAiC;4BAC5D,IAAI,EAAE,wDAAwD;yBAC/D,CAAC,CAAA;oBACJ,CAAC;oBACD,MAAM,CAAC,OAAO,IAAI,CAAC,CAAA;oBACnB,OAAM;gBACR,CAAC;gBACD,MAAM,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC;oBACzB,EAAE,EAAE,MAAM,CAAC,EAAE;oBACb,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,KAAK,EAAE,MAAM,CAAC,KAAK;iBACpB,CAAC,CAAA;gBACF,MAAM,CAAC,KAAK,IAAI,CAAC,CAAA;gBACjB,OAAM;YACR,CAAC;YACD,KAAK,WAAW,EAAE,CAAC;gBACjB,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS;oBAAE,OAAM;gBACvC,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;gBACxD,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;oBACtB,MAAM,CAAC,OAAO,IAAI,CAAC,CAAA;oBACnB,OAAM;gBACR,CAAC;gBACD,MAAM,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,EAAE;oBAC5C,EAAE,EAAE,MAAM,CAAC,EAAE;oBACb,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,IAAI,EAAE,MAAM,CAAC,QAAmD;oBAChE,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;oBACzC,aAAa,EAAE,MAAM,CAAC,aAAa;oBACnC,GAAG,EAAE,MAAM,CAAC,GAAG;oBACf,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,YAAY,EAAE,MAAM,CAAC,YAAY;iBAClC,CAAC,CAAA;gBACF,MAAM,CAAC,SAAS,IAAI,CAAC,CAAA;gBACrB,OAAM;YACR,CAAC;YACD,KAAK,UAAU,EAAE,CAAC;gBAChB,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS;oBAAE,OAAM;gBAC3C,MAAM,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC;oBAC1B,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,EAAE,EAAE,MAAM,CAAC,EAAE;oBACb,MAAM,EAAE,MAAM,CAAC,MAAmB;oBAClC,MAAM,EAAE,QAAQ;oBAChB,GAAG,CAAC,MAAM,CAAC,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC;oBACxE,GAAG,CAAC,MAAM,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;oBAC/D,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;iBAC7D,CAAC,CAAA;gBACF,MAAM,CAAC,SAAS,IAAI,CAAC,CAAA;gBACrB,OAAM;YACR,CAAC;YACD;gBACE,OAAM;QACV,CAAC;IACH,CAAC,CAAA;IAED,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,SAAQ;QACtC,UAAU,IAAI,CAAC,CAAA;QACf,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,EAAE,UAAU,CAAC,CAAA;QAC7C,MAAM,WAAW,CAAC,MAAM,CAAC,CAAA;IAC3B,CAAC;IAED,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,IAAI,YAAY,CAAC;YACrB,IAAI,EAAE,uBAAuB;YAC7B,OAAO,EAAE,oDAAoD;YAC7D,IAAI,EAAE,qEAAqE;SAC5E,CAAC,CAAA;IACJ,CAAC;IAED,yEAAyE;IACzE,qEAAqE;IACrE,wEAAwE;IACxE,6EAA6E;IAC7E,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG,mBAAmB,CAAC,EAAE,CAAC;QAC9C,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,CAAA;QACpC,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,CAAA;QACpC,MAAM,UAAU,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;QAC/B,IAAI,MAAM,CAAC,OAAO,KAAK,aAAa,IAAI,MAAM,CAAC,OAAO,KAAK,aAAa,EAAE,CAAC;YACzE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;gBACjB,IAAI,EAAE,OAAO;gBACb,EAAE,EAAE,MAAM,CAAC,EAAE;gBACb,OAAO,EAAE,uBAAuB,MAAM,CAAC,aAAa,kCAAkC;aACvF,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC"}
|
package/dist/crypto.d.ts
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Encrypts a byte stream, never buffering more than one chunk at a time.
|
|
3
|
+
*
|
|
4
|
+
* Framing: `MAGIC(5) | salt(16) | iv(12) | ciphertext(…) | authTag(16)`. The
|
|
5
|
+
* tag trails the ciphertext because GCM only finalises it once every byte has
|
|
6
|
+
* passed through the cipher — putting it up front would require buffering the
|
|
7
|
+
* whole backup first, which is exactly what task 2's "never assembled in
|
|
8
|
+
* memory" rules out.
|
|
9
|
+
*/
|
|
10
|
+
export declare function encryptStream(input: AsyncIterable<Buffer> | Iterable<Buffer>, passphrase: string): AsyncGenerator<Buffer>;
|
|
11
|
+
/**
|
|
12
|
+
* Decrypts a stream produced by `encryptStream`. Holds at most one input
|
|
13
|
+
* chunk plus a `TAG_LENGTH`-byte lookback buffer at a time — the lookback is
|
|
14
|
+
* what lets the trailing auth tag be recognised without having read the whole
|
|
15
|
+
* file first.
|
|
16
|
+
*/
|
|
17
|
+
export declare function decryptStream(input: AsyncIterable<Buffer> | Iterable<Buffer>, passphrase: string): AsyncGenerator<Buffer>;
|
|
18
|
+
//# sourceMappingURL=crypto.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"crypto.d.ts","sourceRoot":"","sources":["../src/crypto.ts"],"names":[],"mappings":"AAuCA;;;;;;;;GAQG;AACH,wBAAuB,aAAa,CAClC,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,EAC/C,UAAU,EAAE,MAAM,GACjB,cAAc,CAAC,MAAM,CAAC,CAexB;AAED;;;;;GAKG;AACH,wBAAuB,aAAa,CAClC,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,EAC/C,UAAU,EAAE,MAAM,GACjB,cAAc,CAAC,MAAM,CAAC,CA2ExB"}
|
package/dist/crypto.js
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import { createCipheriv, createDecipheriv, randomBytes, scrypt as scryptCb } from 'node:crypto';
|
|
2
|
+
import { CogentaError } from '@cogenta/core';
|
|
3
|
+
function scrypt(passphrase, salt, keyLength, options) {
|
|
4
|
+
return new Promise((resolve, reject) => {
|
|
5
|
+
scryptCb(passphrase, salt, keyLength, options, (error, derivedKey) => {
|
|
6
|
+
if (error)
|
|
7
|
+
reject(error);
|
|
8
|
+
else
|
|
9
|
+
resolve(derivedKey);
|
|
10
|
+
});
|
|
11
|
+
});
|
|
12
|
+
}
|
|
13
|
+
/** `aes-256-gcm`: authenticated, so a tampered or truncated backup fails to decrypt rather than silently restoring garbage. */
|
|
14
|
+
const ALGORITHM = 'aes-256-gcm';
|
|
15
|
+
const KEY_LENGTH = 32;
|
|
16
|
+
const SALT_LENGTH = 16;
|
|
17
|
+
const IV_LENGTH = 12;
|
|
18
|
+
const TAG_LENGTH = 16;
|
|
19
|
+
/** `scrypt` cost parameters (RFC 7914's "interactive" profile, scaled up once): memory-hard, so a leaked backup cannot be brute-forced on a GPU as cheaply as a mere iterated hash would allow (rule of thumb: a backup carries every password hash on the site, per the plan's own piège). */
|
|
20
|
+
const SCRYPT_N = 2 ** 15;
|
|
21
|
+
const SCRYPT_R = 8;
|
|
22
|
+
const SCRYPT_P = 1;
|
|
23
|
+
const MAGIC = Buffer.from('CGEB1'); // "Cogenta Encrypted Backup, v1"
|
|
24
|
+
async function deriveKey(passphrase, salt) {
|
|
25
|
+
return scrypt(passphrase, salt, KEY_LENGTH, {
|
|
26
|
+
N: SCRYPT_N,
|
|
27
|
+
r: SCRYPT_R,
|
|
28
|
+
p: SCRYPT_P,
|
|
29
|
+
maxmem: 256 * 1024 * 1024,
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Encrypts a byte stream, never buffering more than one chunk at a time.
|
|
34
|
+
*
|
|
35
|
+
* Framing: `MAGIC(5) | salt(16) | iv(12) | ciphertext(…) | authTag(16)`. The
|
|
36
|
+
* tag trails the ciphertext because GCM only finalises it once every byte has
|
|
37
|
+
* passed through the cipher — putting it up front would require buffering the
|
|
38
|
+
* whole backup first, which is exactly what task 2's "never assembled in
|
|
39
|
+
* memory" rules out.
|
|
40
|
+
*/
|
|
41
|
+
export async function* encryptStream(input, passphrase) {
|
|
42
|
+
const salt = randomBytes(SALT_LENGTH);
|
|
43
|
+
const iv = randomBytes(IV_LENGTH);
|
|
44
|
+
const key = await deriveKey(passphrase, salt);
|
|
45
|
+
const cipher = createCipheriv(ALGORITHM, key, iv);
|
|
46
|
+
yield Buffer.concat([MAGIC, salt, iv]);
|
|
47
|
+
for await (const chunk of input) {
|
|
48
|
+
const encrypted = cipher.update(chunk);
|
|
49
|
+
if (encrypted.length > 0)
|
|
50
|
+
yield encrypted;
|
|
51
|
+
}
|
|
52
|
+
const final = cipher.final();
|
|
53
|
+
if (final.length > 0)
|
|
54
|
+
yield final;
|
|
55
|
+
yield cipher.getAuthTag();
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Decrypts a stream produced by `encryptStream`. Holds at most one input
|
|
59
|
+
* chunk plus a `TAG_LENGTH`-byte lookback buffer at a time — the lookback is
|
|
60
|
+
* what lets the trailing auth tag be recognised without having read the whole
|
|
61
|
+
* file first.
|
|
62
|
+
*/
|
|
63
|
+
export async function* decryptStream(input, passphrase) {
|
|
64
|
+
const iterator = normalise(input)[Symbol.asyncIterator]();
|
|
65
|
+
let buffered = Buffer.alloc(0);
|
|
66
|
+
const need = async (bytes) => {
|
|
67
|
+
while (buffered.length < bytes) {
|
|
68
|
+
const next = await iterator.next();
|
|
69
|
+
if (next.done === true)
|
|
70
|
+
return false;
|
|
71
|
+
buffered = Buffer.concat([buffered, next.value]);
|
|
72
|
+
}
|
|
73
|
+
return true;
|
|
74
|
+
};
|
|
75
|
+
if (!(await need(MAGIC.length + SALT_LENGTH + IV_LENGTH))) {
|
|
76
|
+
throw truncated();
|
|
77
|
+
}
|
|
78
|
+
const magic = buffered.subarray(0, MAGIC.length);
|
|
79
|
+
if (!magic.equals(MAGIC)) {
|
|
80
|
+
throw new CogentaError({
|
|
81
|
+
code: 'BACKUP_DECRYPTION_FAILED',
|
|
82
|
+
message: 'This file is not a Cogenta encrypted backup.',
|
|
83
|
+
hint: 'Decrypt only files produced by `cogenta backup --encrypt`.',
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
const salt = buffered.subarray(MAGIC.length, MAGIC.length + SALT_LENGTH);
|
|
87
|
+
const iv = buffered.subarray(MAGIC.length + SALT_LENGTH, MAGIC.length + SALT_LENGTH + IV_LENGTH);
|
|
88
|
+
buffered = buffered.subarray(MAGIC.length + SALT_LENGTH + IV_LENGTH);
|
|
89
|
+
const key = await deriveKey(passphrase, Buffer.from(salt));
|
|
90
|
+
const decipher = createDecipheriv(ALGORITHM, key, iv);
|
|
91
|
+
// Everything from here on is ciphertext, except the final TAG_LENGTH bytes
|
|
92
|
+
// of the whole stream. Only the boundary is uncertain until the stream
|
|
93
|
+
// ends, so at most TAG_LENGTH bytes are ever held back. `buffered` may
|
|
94
|
+
// already hold more than the header when the source handed everything to
|
|
95
|
+
// `need()` in a single chunk (a `Buffer[]` input, in particular) — released
|
|
96
|
+
// right away, rather than only after another `iterator.next()` that may
|
|
97
|
+
// never come.
|
|
98
|
+
const release = () => {
|
|
99
|
+
if (buffered.length <= TAG_LENGTH)
|
|
100
|
+
return null;
|
|
101
|
+
const releasable = buffered.subarray(0, buffered.length - TAG_LENGTH);
|
|
102
|
+
buffered = buffered.subarray(buffered.length - TAG_LENGTH);
|
|
103
|
+
return releasable;
|
|
104
|
+
};
|
|
105
|
+
const initial = release();
|
|
106
|
+
if (initial !== null) {
|
|
107
|
+
const decrypted = decipher.update(initial);
|
|
108
|
+
if (decrypted.length > 0)
|
|
109
|
+
yield decrypted;
|
|
110
|
+
}
|
|
111
|
+
for (;;) {
|
|
112
|
+
const next = await iterator.next();
|
|
113
|
+
if (next.done === true)
|
|
114
|
+
break;
|
|
115
|
+
buffered = Buffer.concat([buffered, next.value]);
|
|
116
|
+
const releasable = release();
|
|
117
|
+
if (releasable !== null) {
|
|
118
|
+
const decrypted = decipher.update(releasable);
|
|
119
|
+
if (decrypted.length > 0)
|
|
120
|
+
yield decrypted;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
if (buffered.length !== TAG_LENGTH)
|
|
124
|
+
throw truncated();
|
|
125
|
+
decipher.setAuthTag(buffered);
|
|
126
|
+
try {
|
|
127
|
+
const final = decipher.final();
|
|
128
|
+
if (final.length > 0)
|
|
129
|
+
yield final;
|
|
130
|
+
}
|
|
131
|
+
catch (cause) {
|
|
132
|
+
throw new CogentaError({
|
|
133
|
+
code: 'BACKUP_DECRYPTION_FAILED',
|
|
134
|
+
message: 'Decryption failed: the passphrase is wrong or the file was tampered with.',
|
|
135
|
+
hint: 'Re-enter the passphrase used to create this backup, or restore from a different file.',
|
|
136
|
+
cause,
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
function truncated() {
|
|
141
|
+
return new CogentaError({
|
|
142
|
+
code: 'BACKUP_DECRYPTION_FAILED',
|
|
143
|
+
message: 'The encrypted backup is truncated.',
|
|
144
|
+
hint: 'The file was not fully downloaded or copied. Retry the transfer.',
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
async function* normalise(input) {
|
|
148
|
+
for await (const chunk of input)
|
|
149
|
+
yield chunk;
|
|
150
|
+
}
|
|
151
|
+
//# sourceMappingURL=crypto.js.map
|