@starci/seed 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/LICENSE +21 -0
- package/README.md +51 -2
- package/dist/copies.d.ts +83 -0
- package/dist/copies.js +146 -0
- package/dist/driver.d.ts +145 -0
- package/dist/driver.js +21 -0
- package/dist/git/driver.d.ts +53 -0
- package/dist/git/driver.js +309 -0
- package/dist/git/files.d.ts +48 -0
- package/dist/git/files.js +78 -0
- package/dist/git/index.d.ts +4 -0
- package/dist/git/index.js +2 -0
- package/dist/index.d.ts +12 -0
- package/dist/index.js +6 -0
- package/dist/manifest.d.ts +108 -0
- package/dist/manifest.js +131 -0
- package/dist/postgres/index.d.ts +2 -0
- package/dist/postgres/index.js +1 -0
- package/dist/postgres/pipe.d.ts +11 -0
- package/dist/postgres/pipe.js +60 -0
- package/dist/postgres/sql.d.ts +42 -0
- package/dist/postgres/sql.js +54 -0
- package/dist/postgres/store.d.ts +33 -0
- package/dist/postgres/store.js +231 -0
- package/dist/protocol.d.ts +52 -0
- package/dist/protocol.js +101 -0
- package/dist/store.d.ts +97 -0
- package/dist/store.js +6 -0
- package/dist/testing/contract.d.ts +26 -0
- package/dist/testing/contract.js +219 -0
- package/dist/testing/index.d.ts +5 -0
- package/dist/testing/index.js +3 -0
- package/dist/testing/memory.d.ts +53 -0
- package/dist/testing/memory.js +0 -0
- package/dist/words.d.ts +33 -0
- package/dist/words.js +16 -0
- package/package.json +85 -4
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
import { spawnSync } from "node:child_process";
|
|
2
|
+
import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { dirname, join, relative, sep } from "node:path";
|
|
5
|
+
import { memorySpool, spooledSource } from "../copies.js";
|
|
6
|
+
import { DataRefusal, holdingFor } from "../driver.js";
|
|
7
|
+
import { ManifestBuilder, blobEntry, byteOrder, canonicalJson, manifestProblems, recordEntry, recordValue, sha256Hex } from "../manifest.js";
|
|
8
|
+
import { GIT_CAP_BYTES, GIT_SHARD_BYTES, GIT_WARN_BYTES } from "../words.js";
|
|
9
|
+
const SEALED_SUFFIX = ".sops.json";
|
|
10
|
+
/** Plaintext budget of one record shard, so the sealed file (base64 + envelope) stays under GIT_SHARD_BYTES. */
|
|
11
|
+
const SHARD_PLAIN_BYTES = Math.floor(GIT_SHARD_BYTES * 0.7);
|
|
12
|
+
const BLOB_CHUNK_BYTES = Math.floor(GIT_SHARD_BYTES * 0.7);
|
|
13
|
+
const COPY_ROOTS = ["data", "blobs", "index"];
|
|
14
|
+
const GIT_ATTRIBUTES = "* -text\n";
|
|
15
|
+
const defaultRunner = (program, args, options) => {
|
|
16
|
+
const result = spawnSync(program, [...args], { cwd: options.cwd, input: options.input, env: { ...process.env, ...options.env }, maxBuffer: 1 << 30 });
|
|
17
|
+
return { status: result.status, stdout: result.stdout, stderr: result.stderr };
|
|
18
|
+
};
|
|
19
|
+
function posix(path) {
|
|
20
|
+
return path.split(sep).join("/");
|
|
21
|
+
}
|
|
22
|
+
function recordShardLines(records) {
|
|
23
|
+
const lines = records.map((entry) => `${JSON.stringify(entry.key)}:${canonicalJson(recordValue(entry))}`);
|
|
24
|
+
return Buffer.from(`{\n${lines.join(",\n")}\n}\n`, "utf8");
|
|
25
|
+
}
|
|
26
|
+
function shardOf(key, width) {
|
|
27
|
+
return width === 0 ? "all" : sha256Hex(key).slice(0, width);
|
|
28
|
+
}
|
|
29
|
+
function shardWidth(records, previous) {
|
|
30
|
+
for (let width = previous; width < 8; width += 1) {
|
|
31
|
+
const sizes = new Map();
|
|
32
|
+
for (const entry of records)
|
|
33
|
+
sizes.set(shardOf(entry.key, width), (sizes.get(shardOf(entry.key, width)) ?? 0) + entry.body.byteLength + entry.key.length + 8);
|
|
34
|
+
if ([...sizes.values()].every((size) => size <= SHARD_PLAIN_BYTES))
|
|
35
|
+
return width;
|
|
36
|
+
}
|
|
37
|
+
return 8;
|
|
38
|
+
}
|
|
39
|
+
/** Builds the git driver over a local working copy. */
|
|
40
|
+
export function createGitDriver(options) {
|
|
41
|
+
const run = options.run ?? defaultRunner;
|
|
42
|
+
const branch = options.branch ?? "main";
|
|
43
|
+
const cap = options.capBytes ?? GIT_CAP_BYTES;
|
|
44
|
+
const warn = options.warnBytes ?? GIT_WARN_BYTES;
|
|
45
|
+
const env = options.env ?? {};
|
|
46
|
+
const limits = { forms: ["record", "blob"], maxFileBytes: GIT_SHARD_BYTES, maxCopyBytes: cap, warnCopyBytes: warn };
|
|
47
|
+
const git = (args) => {
|
|
48
|
+
const result = run("git", args, { cwd: options.checkout, env });
|
|
49
|
+
if (result.status !== 0)
|
|
50
|
+
throw new DataRefusal("DATA_DRIVER_UNAVAILABLE", "transient", { driver: options.id, step: `git ${args.slice(0, 1).join("")}` });
|
|
51
|
+
return result.stdout;
|
|
52
|
+
};
|
|
53
|
+
const sops = (args, input, identityEnv) => {
|
|
54
|
+
const result = run("sops", args, { cwd: tmpdir(), input, env: { ...env, ...identityEnv } });
|
|
55
|
+
if (result.status !== 0)
|
|
56
|
+
throw new DataRefusal("DATA_SEAL_FAILED", "configuration", { driver: options.id, step: `sops ${args.slice(0, 1).join("")}` });
|
|
57
|
+
return result.stdout;
|
|
58
|
+
};
|
|
59
|
+
const seal = (path, plain, recipients) => sops(["encrypt", "--input-type", "binary", "--output-type", "json", "--filename-override", path, "--age", recipients.join(",")], plain, {});
|
|
60
|
+
const open = (path, sealed, context) => sops(["decrypt", "--input-type", "json", "--output-type", "binary", "--filename-override", path, "--decryption-order", "age"], sealed, context.identity.env);
|
|
61
|
+
const prepare = () => {
|
|
62
|
+
if (!existsSync(join(options.checkout, ".git"))) {
|
|
63
|
+
mkdirSync(options.checkout, { recursive: true });
|
|
64
|
+
const cloned = options.remote !== undefined && run("git", ["clone", "-q", "-b", branch, options.remote, "."], { cwd: options.checkout, env }).status === 0;
|
|
65
|
+
if (!cloned) {
|
|
66
|
+
git(["init", "-q", "-b", branch]);
|
|
67
|
+
if (options.remote !== undefined)
|
|
68
|
+
git(["remote", "add", "origin", options.remote]);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
else if (options.remote !== undefined) {
|
|
72
|
+
const fetched = run("git", ["pull", "-q", "--ff-only", "origin", branch], { cwd: options.checkout, env });
|
|
73
|
+
if (fetched.status !== 0 && run("git", ["rev-parse", "--verify", "-q", "HEAD"], { cwd: options.checkout, env }).status === 0)
|
|
74
|
+
throw new DataRefusal("DATA_DRIVER_UNAVAILABLE", "transient", { driver: options.id, step: "git pull" });
|
|
75
|
+
}
|
|
76
|
+
};
|
|
77
|
+
/** Copy files are bytes, never text: no line-ending conversion may touch a sealed envelope or a digest. */
|
|
78
|
+
const pinBytes = () => {
|
|
79
|
+
const file = join(options.checkout, ".gitattributes");
|
|
80
|
+
if (!existsSync(file) || readFileSync(file, "utf8") !== GIT_ATTRIBUTES)
|
|
81
|
+
writeFileSync(file, GIT_ATTRIBUTES);
|
|
82
|
+
};
|
|
83
|
+
const headManifest = () => {
|
|
84
|
+
const file = join(options.checkout, "manifest.json");
|
|
85
|
+
return existsSync(file) ? JSON.parse(readFileSync(file, "utf8")) : null;
|
|
86
|
+
};
|
|
87
|
+
const showAt = (commit, path) => git(["show", `${commit}:${path}`]);
|
|
88
|
+
const manifestAt = (commit) => JSON.parse(showAt(commit, "manifest.json").toString("utf8"));
|
|
89
|
+
const copyBytesOnDisk = () => {
|
|
90
|
+
const found = [];
|
|
91
|
+
for (const root of COPY_ROOTS) {
|
|
92
|
+
const dir = join(options.checkout, root);
|
|
93
|
+
if (!existsSync(dir))
|
|
94
|
+
continue;
|
|
95
|
+
for (const name of readdirSync(dir, { recursive: true, encoding: "utf8" })) {
|
|
96
|
+
const full = join(dir, name);
|
|
97
|
+
if (statSync(full).isFile())
|
|
98
|
+
found.push(posix(relative(options.checkout, full)));
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
return found;
|
|
102
|
+
};
|
|
103
|
+
/** Every file of the new snapshot: path (sealed name when sealed) -> plaintext. */
|
|
104
|
+
const planFiles = async (source, previous) => {
|
|
105
|
+
const bySet = new Map();
|
|
106
|
+
for await (const item of source.entries()) {
|
|
107
|
+
const id = `${item.store}/${item.entry.set}`;
|
|
108
|
+
bySet.set(id, [...(bySet.get(id) ?? []), item.entry]);
|
|
109
|
+
}
|
|
110
|
+
const files = new Map();
|
|
111
|
+
const shards = {};
|
|
112
|
+
for (const set of source.manifest.sets) {
|
|
113
|
+
const id = `${set.store}/${set.set}`;
|
|
114
|
+
const entries = [...(bySet.get(id) ?? [])].sort((left, right) => byteOrder(left.key, right.key));
|
|
115
|
+
if (set.form === "record") {
|
|
116
|
+
const width = shardWidth(entries, previous?.shards[id] ?? 0);
|
|
117
|
+
shards[id] = width;
|
|
118
|
+
const groups = new Map();
|
|
119
|
+
for (const entry of entries)
|
|
120
|
+
groups.set(shardOf(entry.key, width), [...(groups.get(shardOf(entry.key, width)) ?? []), entry]);
|
|
121
|
+
for (const [shard, records] of groups)
|
|
122
|
+
files.set(`data/${id}/${shard}.json`, recordShardLines(records));
|
|
123
|
+
continue;
|
|
124
|
+
}
|
|
125
|
+
const index = {};
|
|
126
|
+
for (const entry of entries) {
|
|
127
|
+
const blobId = sha256Hex(`${entry.set}\u0000${entry.key}`);
|
|
128
|
+
const chunks = Math.max(1, Math.ceil(entry.body.byteLength / BLOB_CHUNK_BYTES));
|
|
129
|
+
for (let at = 0; at < chunks; at += 1) {
|
|
130
|
+
const name = chunks === 1 ? `${blobId}.bin` : `${blobId}.${String(at)}.bin`;
|
|
131
|
+
files.set(`blobs/${id}/${blobId.slice(0, 2)}/${name}`, Buffer.from(entry.body.subarray(at * BLOB_CHUNK_BYTES, (at + 1) * BLOB_CHUNK_BYTES)));
|
|
132
|
+
}
|
|
133
|
+
index[entry.key] = { media: entry.media, sha256: entry.sha256, chunks };
|
|
134
|
+
}
|
|
135
|
+
files.set(`index/${id}.json`, Buffer.from(`${canonicalJson(index)}\n`, "utf8"));
|
|
136
|
+
}
|
|
137
|
+
return { files, shards };
|
|
138
|
+
};
|
|
139
|
+
const fileText = (commit, manifest, path, context) => {
|
|
140
|
+
const stored = manifest.sealing === "none" ? path : `${path}${SEALED_SUFFIX}`;
|
|
141
|
+
const bytes = showAt(commit, stored);
|
|
142
|
+
if (manifest.sealing === "none")
|
|
143
|
+
return bytes;
|
|
144
|
+
if (context === null)
|
|
145
|
+
throw new DataRefusal("DATA_SEAL_FAILED", "configuration", { driver: options.id, reason: "an identity is needed to open a sealed copy" });
|
|
146
|
+
return open(stored, bytes, context);
|
|
147
|
+
};
|
|
148
|
+
const entriesAt = (commit, manifest, context) => {
|
|
149
|
+
const out = [];
|
|
150
|
+
for (const set of manifest.sets) {
|
|
151
|
+
const id = `${set.store}/${set.set}`;
|
|
152
|
+
if (set.form === "record") {
|
|
153
|
+
const shards = Object.keys(manifest.files).filter((path) => path.startsWith(`data/${id}/`)).sort(byteOrder);
|
|
154
|
+
const records = [];
|
|
155
|
+
for (const path of shards) {
|
|
156
|
+
const values = JSON.parse(fileText(commit, manifest, path, context).toString("utf8"));
|
|
157
|
+
for (const [key, value] of Object.entries(values))
|
|
158
|
+
records.push(recordEntry(set.set, key, value));
|
|
159
|
+
}
|
|
160
|
+
for (const entry of records.sort((left, right) => byteOrder(left.key, right.key)))
|
|
161
|
+
out.push({ store: set.store, entry });
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
const index = JSON.parse(fileText(commit, manifest, `index/${id}.json`, context).toString("utf8"));
|
|
165
|
+
for (const [key, item] of Object.entries(index).sort(([left], [right]) => byteOrder(left, right))) {
|
|
166
|
+
const blobId = sha256Hex(`${set.set}\u0000${key}`);
|
|
167
|
+
const parts = [];
|
|
168
|
+
for (let at = 0; at < item.chunks; at += 1) {
|
|
169
|
+
const name = item.chunks === 1 ? `${blobId}.bin` : `${blobId}.${String(at)}.bin`;
|
|
170
|
+
parts.push(fileText(commit, manifest, `blobs/${id}/${blobId.slice(0, 2)}/${name}`, context));
|
|
171
|
+
}
|
|
172
|
+
out.push({ store: set.store, entry: blobEntry(set.set, key, new Uint8Array(Buffer.concat(parts)), item.media) });
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
return out;
|
|
176
|
+
};
|
|
177
|
+
const refOf = (commit, manifest) => ({ driver: options.id, kind: "git", locator: commit, exportDigest: manifest.exportDigest, takenAt: manifest.takenAt });
|
|
178
|
+
return {
|
|
179
|
+
id: options.id,
|
|
180
|
+
kind: "git",
|
|
181
|
+
limits: () => limits,
|
|
182
|
+
plan: (manifest) => holdingFor(manifest, limits.forms),
|
|
183
|
+
async write(source, context) {
|
|
184
|
+
const sealed = source.manifest.purpose === "seed-backup";
|
|
185
|
+
if (sealed && context.recipients.length === 0)
|
|
186
|
+
throw new DataRefusal("DATA_SEAL_FAILED", "configuration", { driver: options.id, reason: "a seed-backup is sealed to its environment's recipients and none were given" });
|
|
187
|
+
if (!context.dryRun) {
|
|
188
|
+
prepare();
|
|
189
|
+
pinBytes();
|
|
190
|
+
}
|
|
191
|
+
const previous = headManifest();
|
|
192
|
+
const { files, shards } = await planFiles(source, previous);
|
|
193
|
+
const facts = {};
|
|
194
|
+
const writes = new Map();
|
|
195
|
+
let total = 0;
|
|
196
|
+
for (const [path, plain] of [...files.entries()].sort(([left], [right]) => byteOrder(left, right))) {
|
|
197
|
+
const stored = sealed ? `${path}${SEALED_SUFFIX}` : path;
|
|
198
|
+
const plainSha = sha256Hex(plain);
|
|
199
|
+
const before = previous?.files[path];
|
|
200
|
+
const onDisk = join(options.checkout, stored);
|
|
201
|
+
const keep = before?.plain === plainSha && previous?.sealing === (sealed ? "sops-age" : "none") && existsSync(onDisk);
|
|
202
|
+
const bytes = keep ? statSync(onDisk).size : (() => {
|
|
203
|
+
const out = sealed ? seal(stored, plain, context.recipients) : plain;
|
|
204
|
+
writes.set(stored, out);
|
|
205
|
+
return out.byteLength;
|
|
206
|
+
})();
|
|
207
|
+
if (bytes > GIT_SHARD_BYTES)
|
|
208
|
+
throw new DataRefusal("DATA_GIT_CAP_EXCEEDED", "configuration", { driver: options.id, file: path, bytes, limit: GIT_SHARD_BYTES });
|
|
209
|
+
facts[path] = { plain: plainSha, bytes };
|
|
210
|
+
total += bytes;
|
|
211
|
+
}
|
|
212
|
+
if (total > cap)
|
|
213
|
+
throw new DataRefusal("DATA_GIT_CAP_EXCEEDED", "configuration", { driver: options.id, bytes: total, cap, next: "s3" });
|
|
214
|
+
const warnings = total > warn ? ["DATA_GIT_CAP_NEAR"] : [];
|
|
215
|
+
const holding = holdingFor(source.manifest, limits.forms);
|
|
216
|
+
const at = new Date().toISOString();
|
|
217
|
+
const manifest = { ...source.manifest, sealing: sealed ? "sops-age" : "none", recipients: sealed ? [...context.recipients].sort() : [], shards, files: facts };
|
|
218
|
+
const keepPaths = new Set(Object.keys(facts).map((path) => (sealed ? `${path}${SEALED_SUFFIX}` : path)));
|
|
219
|
+
const stale = copyBytesOnDisk().filter((path) => !keepPaths.has(path));
|
|
220
|
+
if (context.dryRun) {
|
|
221
|
+
const ref = { driver: options.id, kind: "git", locator: "dry-run", exportDigest: manifest.exportDigest, takenAt: manifest.takenAt };
|
|
222
|
+
return { schema: "starci/data-copy@1", ref, holding, bytes: total, changed: writes.size + stale.length, warnings, at };
|
|
223
|
+
}
|
|
224
|
+
for (const path of stale)
|
|
225
|
+
rmSync(join(options.checkout, path));
|
|
226
|
+
for (const [path, bytes] of writes) {
|
|
227
|
+
mkdirSync(dirname(join(options.checkout, path)), { recursive: true });
|
|
228
|
+
writeFileSync(join(options.checkout, path), bytes);
|
|
229
|
+
}
|
|
230
|
+
writeFileSync(join(options.checkout, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`);
|
|
231
|
+
git(["add", "-A", "--", ".gitattributes", "manifest.json", ...COPY_ROOTS.filter((root) => existsSync(join(options.checkout, root)))]);
|
|
232
|
+
const staged = run("git", ["diff", "--cached", "--quiet"], { cwd: options.checkout, env }).status !== 0;
|
|
233
|
+
const unchanged = !staged || previous?.exportDigest === manifest.exportDigest && writes.size === 0 && stale.length === 0;
|
|
234
|
+
if (staged && !unchanged)
|
|
235
|
+
git(["commit", "-q", "-m", `${manifest.format} ${manifest.exportDigest.slice(0, 12)} ${manifest.takenAt}`]);
|
|
236
|
+
else if (staged)
|
|
237
|
+
git(["reset", "-q", "--hard", "HEAD"]);
|
|
238
|
+
if (options.remote !== undefined)
|
|
239
|
+
git(["push", "-q", "origin", `HEAD:${branch}`]);
|
|
240
|
+
const commit = git(["rev-parse", "HEAD"]).toString("utf8").trim();
|
|
241
|
+
return { schema: "starci/data-copy@1", ref: refOf(commit, headManifest() ?? manifest), holding, bytes: total, changed: unchanged ? 0 : writes.size + stale.length, warnings, at };
|
|
242
|
+
},
|
|
243
|
+
list() {
|
|
244
|
+
if (!existsSync(join(options.checkout, ".git")))
|
|
245
|
+
return Promise.resolve([]);
|
|
246
|
+
const log = run("git", ["log", "-n", "30", "--format=%H", "--", "manifest.json"], { cwd: options.checkout, env });
|
|
247
|
+
if (log.status !== 0)
|
|
248
|
+
return Promise.resolve([]);
|
|
249
|
+
const commits = log.stdout.toString("utf8").split("\n").filter((line) => line.trim() !== "");
|
|
250
|
+
return Promise.resolve(commits.map((commit) => refOf(commit, manifestAt(commit))));
|
|
251
|
+
},
|
|
252
|
+
read(ref, context) {
|
|
253
|
+
const manifest = manifestAt(ref.locator);
|
|
254
|
+
const spool = memorySpool();
|
|
255
|
+
const entries = entriesAt(ref.locator, manifest, context);
|
|
256
|
+
return Promise.all(entries.map((item) => spool.append(item))).then(() => spooledSource(manifest, spool));
|
|
257
|
+
},
|
|
258
|
+
verify(ref, context) {
|
|
259
|
+
const checks = [];
|
|
260
|
+
let manifest;
|
|
261
|
+
try {
|
|
262
|
+
manifest = manifestAt(ref.locator);
|
|
263
|
+
}
|
|
264
|
+
catch {
|
|
265
|
+
return Promise.resolve({ ref, healthy: false, checks: [{ name: "manifest", ok: false, detail: "manifest.json cannot be read at the commit" }], manifest: null });
|
|
266
|
+
}
|
|
267
|
+
const problems = manifestProblems(manifest);
|
|
268
|
+
checks.push({ name: "manifest", ok: problems.length === 0, detail: problems.length === 0 ? "starci/data-manifest@1, exportDigest matches the sets" : problems.join("; ") });
|
|
269
|
+
const sealedPolicy = manifest.purpose === "seed-backup" ? "sops-age" : "none";
|
|
270
|
+
checks.push({ name: "sealing", ok: manifest.sealing === sealedPolicy && (sealedPolicy === "none" || manifest.recipients.length > 0), detail: `purpose ${manifest.purpose} requires ${sealedPolicy}, the copy says ${manifest.sealing}` });
|
|
271
|
+
const tree = git(["ls-tree", "-r", "-l", ref.locator]).toString("utf8").split("\n").filter((line) => line.trim() !== "");
|
|
272
|
+
const listed = tree.flatMap((line) => {
|
|
273
|
+
const match = /^\S+ \S+ \S+\s+(\d+)\t(.+)$/u.exec(line);
|
|
274
|
+
return match === null ? [] : [{ path: String(match[2]), bytes: Number(match[1]) }];
|
|
275
|
+
});
|
|
276
|
+
const sizes = new Set(listed.map((item) => item.path));
|
|
277
|
+
const copyListed = listed.filter((item) => COPY_ROOTS.some((root) => item.path.startsWith(`${root}/`)));
|
|
278
|
+
const copyFiles = copyListed.map((item) => item.path);
|
|
279
|
+
const expected = new Set(Object.keys(manifest.files).map((path) => (manifest.sealing === "none" ? path : `${path}${SEALED_SUFFIX}`)));
|
|
280
|
+
const stray = copyFiles.filter((path) => !expected.has(path));
|
|
281
|
+
const missing = [...expected].filter((path) => !sizes.has(path));
|
|
282
|
+
const unsealed = manifest.sealing === "none" ? [] : copyFiles.filter((path) => !path.endsWith(SEALED_SUFFIX));
|
|
283
|
+
checks.push({ name: "files", ok: stray.length === 0 && missing.length === 0 && unsealed.length === 0, detail: `${String(copyFiles.length)} files; stray ${String(stray.length)}, missing ${String(missing.length)}, unsealed ${String(unsealed.length)}` });
|
|
284
|
+
if (manifest.sealing !== "none") {
|
|
285
|
+
const notEnvelope = copyFiles.filter((path) => path.endsWith(SEALED_SUFFIX)).filter((path) => {
|
|
286
|
+
const text = showAt(ref.locator, path).toString("utf8");
|
|
287
|
+
return !text.includes('"sops"') || !text.includes("ENC[AES256_GCM");
|
|
288
|
+
});
|
|
289
|
+
checks.push({ name: "sealed", ok: notEnvelope.length === 0, detail: notEnvelope.length === 0 ? "every data file is a sops envelope" : `${String(notEnvelope.length)} files are not sops envelopes` });
|
|
290
|
+
}
|
|
291
|
+
const total = copyListed.reduce((sum, item) => sum + item.bytes, 0);
|
|
292
|
+
checks.push({ name: "cap", ok: total <= cap, detail: `${String(total)} bytes of ${String(cap)}` });
|
|
293
|
+
if (context !== null || manifest.sealing === "none") {
|
|
294
|
+
try {
|
|
295
|
+
const builder = new ManifestBuilder();
|
|
296
|
+
for (const item of entriesAt(ref.locator, manifest, context))
|
|
297
|
+
builder.add(item.store, item.entry);
|
|
298
|
+
const rebuilt = builder.build(manifest);
|
|
299
|
+
const wrong = manifest.sets.filter((set) => rebuilt.sets.find((other) => other.store === set.store && other.set === set.set)?.digest !== set.digest);
|
|
300
|
+
checks.push({ name: "digests", ok: wrong.length === 0, detail: wrong.length === 0 ? "every set matches its digest and count" : `differ: ${wrong.map((set) => `${set.store}/${set.set}`).join(", ")}` });
|
|
301
|
+
}
|
|
302
|
+
catch (cause) {
|
|
303
|
+
checks.push({ name: "digests", ok: false, detail: cause instanceof DataRefusal ? cause.code : "a file cannot be read or parsed" });
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
return Promise.resolve({ ref, healthy: checks.every((check) => check.ok), checks, manifest });
|
|
307
|
+
},
|
|
308
|
+
};
|
|
309
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import type { OpenContext } from "../driver.js";
|
|
2
|
+
import type { Runner } from "./driver.js";
|
|
3
|
+
/**
|
|
4
|
+
* File-level reads of a copy repository at one pinned commit, and writes into a working tree for review: what a
|
|
5
|
+
* project's own extension of the standard (a kind it defines on top, with its own document) reads through, so no app
|
|
6
|
+
* writes git or sops code. The data driver (`createGitDriver`) stays the one writer of the generic layout.
|
|
7
|
+
*/
|
|
8
|
+
export interface PinnedRepository {
|
|
9
|
+
/** The commit every read is of (40 hex digits, verified to exist). */
|
|
10
|
+
readonly commit: string;
|
|
11
|
+
/** The bytes of a repository-relative file at the commit, or null when the commit holds no such file. */
|
|
12
|
+
readBytes(path: string): Buffer | null;
|
|
13
|
+
/** The text of a repository-relative file at the commit, or null. */
|
|
14
|
+
readText(path: string): string | null;
|
|
15
|
+
/** Opens a sops-sealed file of the commit IN MEMORY with the identity (sops on stdin/stdout); the plaintext never reaches a file. */
|
|
16
|
+
openSealed(path: string, context: OpenContext): Buffer | null;
|
|
17
|
+
}
|
|
18
|
+
/** Where a pinned repository is fetched from and into. */
|
|
19
|
+
export interface PinOptions {
|
|
20
|
+
/** A git remote URL (never one holding a credential) or a local repository path. */
|
|
21
|
+
readonly repository: string;
|
|
22
|
+
/** The commit, 40 hex digits. */
|
|
23
|
+
readonly commit: string;
|
|
24
|
+
/** A cache directory outside every code repository; one bare mirror per repository name lives under it. */
|
|
25
|
+
readonly cache: string;
|
|
26
|
+
/** The mirror's folder name under the cache (the repository's name). */
|
|
27
|
+
readonly name: string;
|
|
28
|
+
/** Extra environment for git and sops. */
|
|
29
|
+
readonly env?: Readonly<Record<string, string>>;
|
|
30
|
+
/** The program runner; spawnSync by default. */
|
|
31
|
+
readonly run?: Runner;
|
|
32
|
+
}
|
|
33
|
+
/** Fetches the repository into its bare mirror under the cache and answers it at the pinned commit; refuses a commit it cannot find. */
|
|
34
|
+
export declare function pinRepository(options: PinOptions): PinnedRepository;
|
|
35
|
+
/** A working tree a person reviews and commits: writes text, and seals values through sops with the tree's own `.sops.yaml` rules. */
|
|
36
|
+
export interface ReviewTree {
|
|
37
|
+
/** The text of a tree-relative file, or null. */
|
|
38
|
+
readText(path: string): string | null;
|
|
39
|
+
/** Writes a tree-relative text file; true when its content changed. */
|
|
40
|
+
writeText(path: string, text: string): boolean;
|
|
41
|
+
/** Seals a plaintext document (yaml or json by extension) into `path` with the tree's `.sops.yaml` creation rule for that path; the plaintext is sops's stdin only. True when it wrote. */
|
|
42
|
+
seal(path: string, plaintext: Buffer): boolean;
|
|
43
|
+
}
|
|
44
|
+
/** Opens a git working tree for review writes; refuses a folder that is not one. */
|
|
45
|
+
export declare function openReviewTree(root: string, options?: {
|
|
46
|
+
readonly env?: Readonly<Record<string, string>>;
|
|
47
|
+
readonly run?: Runner;
|
|
48
|
+
}): ReviewTree;
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { spawnSync } from "node:child_process";
|
|
2
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { dirname, join, normalize, posix, win32 } from "node:path";
|
|
4
|
+
import { DataRefusal } from "../driver.js";
|
|
5
|
+
const defaultRunner = (program, args, options) => {
|
|
6
|
+
const result = spawnSync(program, [...args], { cwd: options.cwd, input: options.input, env: { ...process.env, ...options.env }, maxBuffer: 1 << 30 });
|
|
7
|
+
return { status: result.status, stdout: result.stdout, stderr: result.stderr };
|
|
8
|
+
};
|
|
9
|
+
function safePath(path) {
|
|
10
|
+
const clean = normalize(path).split("\\").join("/");
|
|
11
|
+
// A repository path is the same on every machine: a drive or root prefix of either platform is refused on both.
|
|
12
|
+
if (posix.isAbsolute(path) || win32.isAbsolute(path) || clean.startsWith("../") || clean === "..")
|
|
13
|
+
throw new DataRefusal("DATA_COPY_MISSING", "configuration", { path });
|
|
14
|
+
return clean;
|
|
15
|
+
}
|
|
16
|
+
/** Fetches the repository into its bare mirror under the cache and answers it at the pinned commit; refuses a commit it cannot find. */
|
|
17
|
+
export function pinRepository(options) {
|
|
18
|
+
const run = options.run ?? defaultRunner;
|
|
19
|
+
const env = options.env ?? {};
|
|
20
|
+
if (!/^[0-9a-f]{40}$/u.test(options.commit))
|
|
21
|
+
throw new DataRefusal("DATA_COPY_MISSING", "configuration", { reason: "the pin is not a 40-hex commit", name: options.name });
|
|
22
|
+
const mirror = join(options.cache, `${options.name}.git`);
|
|
23
|
+
mkdirSync(options.cache, { recursive: true });
|
|
24
|
+
const git = (args) => run("git", args, { cwd: mirror, env });
|
|
25
|
+
const has = () => git(["cat-file", "-e", `${options.commit}^{commit}`]).status === 0;
|
|
26
|
+
if (!existsSync(mirror) && run("git", ["clone", "-q", "--bare", options.repository, mirror], { cwd: options.cache, env }).status !== 0)
|
|
27
|
+
throw new DataRefusal("DATA_DRIVER_UNAVAILABLE", "transient", { step: "git clone", name: options.name });
|
|
28
|
+
if (!has())
|
|
29
|
+
git(["fetch", "-q", "origin", "+refs/heads/*:refs/heads/*"]);
|
|
30
|
+
if (!has())
|
|
31
|
+
throw new DataRefusal("DATA_COPY_MISSING", "configuration", { reason: "the pinned commit is not in the repository", name: options.name });
|
|
32
|
+
const readBytes = (path) => {
|
|
33
|
+
const shown = git(["show", `${options.commit}:${safePath(path)}`]);
|
|
34
|
+
return shown.status === 0 ? shown.stdout : null;
|
|
35
|
+
};
|
|
36
|
+
return {
|
|
37
|
+
commit: options.commit,
|
|
38
|
+
readBytes,
|
|
39
|
+
readText: (path) => readBytes(path)?.toString("utf8") ?? null,
|
|
40
|
+
openSealed(path, context) {
|
|
41
|
+
const sealed = readBytes(path);
|
|
42
|
+
if (sealed === null)
|
|
43
|
+
return null;
|
|
44
|
+
const type = path.endsWith(".json") ? "json" : path.endsWith(".yaml") || path.endsWith(".yml") ? "yaml" : "binary";
|
|
45
|
+
const opened = run("sops", ["decrypt", "--input-type", type, "--output-type", type, "--filename-override", safePath(path), "--decryption-order", "age"], { cwd: options.cache, env: { ...env, ...context.identity.env }, input: sealed });
|
|
46
|
+
if (opened.status !== 0)
|
|
47
|
+
throw new DataRefusal("DATA_SEAL_FAILED", "configuration", { reason: "the sealed file does not open with this identity", path: safePath(path) });
|
|
48
|
+
return opened.stdout;
|
|
49
|
+
},
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
/** Opens a git working tree for review writes; refuses a folder that is not one. */
|
|
53
|
+
export function openReviewTree(root, options = {}) {
|
|
54
|
+
const run = options.run ?? defaultRunner;
|
|
55
|
+
if (!existsSync(join(root, ".git")))
|
|
56
|
+
throw new DataRefusal("DATA_COPY_MISSING", "configuration", { reason: "not a git working tree", root });
|
|
57
|
+
const full = (path) => join(root, safePath(path));
|
|
58
|
+
const readText = (path) => (existsSync(full(path)) ? readFileSync(full(path), "utf8") : null);
|
|
59
|
+
return {
|
|
60
|
+
readText,
|
|
61
|
+
writeText(path, text) {
|
|
62
|
+
if (readText(path) === text)
|
|
63
|
+
return false;
|
|
64
|
+
mkdirSync(dirname(full(path)), { recursive: true });
|
|
65
|
+
writeFileSync(full(path), text);
|
|
66
|
+
return true;
|
|
67
|
+
},
|
|
68
|
+
seal(path, plaintext) {
|
|
69
|
+
const type = path.endsWith(".json") ? "json" : "yaml";
|
|
70
|
+
const sealed = run("sops", ["encrypt", "--input-type", type, "--output-type", type, "--filename-override", safePath(path)], { cwd: root, env: options.env ?? {}, input: plaintext });
|
|
71
|
+
if (sealed.status !== 0)
|
|
72
|
+
throw new DataRefusal("DATA_SEAL_FAILED", "configuration", { reason: "sops could not seal with the tree's rules", path: safePath(path) });
|
|
73
|
+
mkdirSync(dirname(full(path)), { recursive: true });
|
|
74
|
+
writeFileSync(full(path), sealed.stdout);
|
|
75
|
+
return true;
|
|
76
|
+
},
|
|
77
|
+
};
|
|
78
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export { DATA_PURPOSES, DRIVER_KINDS, GIT_CAP_BYTES, GIT_SHARD_BYTES, GIT_WARN_BYTES, STORE_KINDS } from "./words.js";
|
|
2
|
+
export type { DataPurpose, DriverKind, EntryForm, StoreKind, StoreLevel } from "./words.js";
|
|
3
|
+
export { ManifestBuilder, blobEntry, byteOrder, canonicalJson, entryIntact, exportDigestOf, manifestProblems, recordEntry, recordValue, setDigest, setSchemaOf, sha256Hex, } from "./manifest.js";
|
|
4
|
+
export type { DataEntry, DataManifest, DataSetSummary, DataSource, JsonValue, ManifestHeader, Selection } from "./manifest.js";
|
|
5
|
+
export { importOutcome } from "./store.js";
|
|
6
|
+
export type { ImportOptions, ImportReport, Inability, RefusedKey, SetImport, StoreAdapter, StoreDescription, StoreSet } from "./store.js";
|
|
7
|
+
export { DataRefusal, holdingFor } from "./driver.js";
|
|
8
|
+
export type { CopyReceipt, CopyRef, CopyVerdict, DataRefusalCode, DriverAdapter, DriverLimits, ExportSource, Holding, OmittedSet, OpenContext, VerifyCheck, WriteContext, } from "./driver.js";
|
|
9
|
+
export { causeLabel, chooseCopy, compareCopies, exportOnce, memorySpool, restoreFrom, spooledSource, unchangedSince, writeEverywhere } from "./copies.js";
|
|
10
|
+
export type { CopyDifference, CopyLeg, CopyRun, Spool, StoredEntry } from "./copies.js";
|
|
11
|
+
export { DATA_PROTOCOL, commandStore, decodeEntry, encodeEntry, serveStore } from "./protocol.js";
|
|
12
|
+
export type { CommandRunner, EntryLine, ServeIo } from "./protocol.js";
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export { DATA_PURPOSES, DRIVER_KINDS, GIT_CAP_BYTES, GIT_SHARD_BYTES, GIT_WARN_BYTES, STORE_KINDS } from "./words.js";
|
|
2
|
+
export { ManifestBuilder, blobEntry, byteOrder, canonicalJson, entryIntact, exportDigestOf, manifestProblems, recordEntry, recordValue, setDigest, setSchemaOf, sha256Hex, } from "./manifest.js";
|
|
3
|
+
export { importOutcome } from "./store.js";
|
|
4
|
+
export { DataRefusal, holdingFor } from "./driver.js";
|
|
5
|
+
export { causeLabel, chooseCopy, compareCopies, exportOnce, memorySpool, restoreFrom, spooledSource, unchangedSince, writeEverywhere } from "./copies.js";
|
|
6
|
+
export { DATA_PROTOCOL, commandStore, decodeEntry, encodeEntry, serveStore } from "./protocol.js";
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import type { DataPurpose, EntryForm } from "./words.js";
|
|
2
|
+
/** One unit of content. The same content always yields the same entry, byte for byte. */
|
|
3
|
+
export interface DataEntry {
|
|
4
|
+
/** The set it belongs to: an entity of a database store, a bucket of a storage store, a part of a dump. kebab-case. */
|
|
5
|
+
readonly set: string;
|
|
6
|
+
/** The natural key, unique in its set; never a database id or a workspace id. Entries are ordered by (set, key) in UTF-8 byte order. */
|
|
7
|
+
readonly key: string;
|
|
8
|
+
/** `record` (canonical JSON) or `blob` (opaque bytes). */
|
|
9
|
+
readonly form: EntryForm;
|
|
10
|
+
/** The media type: `application/json` for a record, the stored content type for a blob. */
|
|
11
|
+
readonly media: string;
|
|
12
|
+
/** The content. */
|
|
13
|
+
readonly body: Uint8Array;
|
|
14
|
+
/** Lower-case hex sha256 of `body`. */
|
|
15
|
+
readonly sha256: string;
|
|
16
|
+
}
|
|
17
|
+
/** What one set of an export holds. Two copies hold the same set exactly when their digests are equal. */
|
|
18
|
+
export interface DataSetSummary {
|
|
19
|
+
/** The store the set was read from. */
|
|
20
|
+
readonly store: string;
|
|
21
|
+
/** The set. */
|
|
22
|
+
readonly set: string;
|
|
23
|
+
/** `<format>/<set>@<major>` for a record set, `blob@1` for a blob set. */
|
|
24
|
+
readonly schema: string;
|
|
25
|
+
/** The form of every entry of the set. */
|
|
26
|
+
readonly form: EntryForm;
|
|
27
|
+
/** How many entries. */
|
|
28
|
+
readonly items: number;
|
|
29
|
+
/** The plaintext bytes of all entries. */
|
|
30
|
+
readonly bytes: number;
|
|
31
|
+
/** sha256 over the sorted lines `<key>\t<sha256>\n`: independent of driver, sealing and sharding. */
|
|
32
|
+
readonly digest: string;
|
|
33
|
+
}
|
|
34
|
+
/** Where an export was taken. */
|
|
35
|
+
export interface DataSource {
|
|
36
|
+
/** The project `hfs.json` names. */
|
|
37
|
+
readonly project: string;
|
|
38
|
+
/** The environment of the stack manifest the stores belong to. */
|
|
39
|
+
readonly environment: string;
|
|
40
|
+
/** The release the environment ran, when known. */
|
|
41
|
+
readonly release?: string;
|
|
42
|
+
}
|
|
43
|
+
/** The plaintext manifest of one export. It holds no value of any entry. */
|
|
44
|
+
export interface DataManifest {
|
|
45
|
+
/** The schema of this document. */
|
|
46
|
+
readonly schema: "starci/data-manifest@1";
|
|
47
|
+
/** `<copy name>@<major>`, e.g. `nivo-lite-seed-standard@1`, the repository name and the format major. */
|
|
48
|
+
readonly format: string;
|
|
49
|
+
/** Why it was taken. */
|
|
50
|
+
readonly purpose: DataPurpose;
|
|
51
|
+
/** ISO time it was taken. */
|
|
52
|
+
readonly takenAt: string;
|
|
53
|
+
/** Where it was taken. */
|
|
54
|
+
readonly source: DataSource;
|
|
55
|
+
/** Sorted by (store, set). */
|
|
56
|
+
readonly sets: readonly DataSetSummary[];
|
|
57
|
+
/** sha256 of the canonical JSON of `sets` (takenAt and source excluded): the identity copies are compared on. */
|
|
58
|
+
readonly exportDigest: string;
|
|
59
|
+
}
|
|
60
|
+
/** The fields of a manifest the caller states; the sets and the digest are computed from the entries. */
|
|
61
|
+
export type ManifestHeader = Omit<DataManifest, "exportDigest" | "sets" | "schema">;
|
|
62
|
+
/** What a selection reads: whole sets, optionally narrowed by a scope the store understands (a workspace for a seed). */
|
|
63
|
+
export interface Selection {
|
|
64
|
+
/** The sets to read; every set of the store when absent. */
|
|
65
|
+
readonly sets?: readonly string[];
|
|
66
|
+
/** A narrowing the store understands, e.g. `{ workspace: "nivo-web" }`. */
|
|
67
|
+
readonly scope?: Readonly<Record<string, string>>;
|
|
68
|
+
}
|
|
69
|
+
/** The JSON a record body may hold. */
|
|
70
|
+
export type JsonValue = string | number | boolean | null | readonly JsonValue[] | {
|
|
71
|
+
readonly [key: string]: JsonValue;
|
|
72
|
+
};
|
|
73
|
+
/** Lower-case hex sha256 of bytes or of a UTF-8 string. */
|
|
74
|
+
export declare function sha256Hex(content: Uint8Array | string): string;
|
|
75
|
+
/** Compares two strings by their UTF-8 bytes, the one order every set and key list uses. */
|
|
76
|
+
export declare function byteOrder(left: string, right: string): number;
|
|
77
|
+
/** The canonical JSON text of a value: keys sorted by byte order, no insignificant whitespace, undefined fields dropped. */
|
|
78
|
+
export declare function canonicalJson(value: unknown): string;
|
|
79
|
+
/** A record entry of `value`, its body the canonical JSON. */
|
|
80
|
+
export declare function recordEntry(set: string, key: string, value: JsonValue): DataEntry;
|
|
81
|
+
/** A blob entry of `body`. */
|
|
82
|
+
export declare function blobEntry(set: string, key: string, body: Uint8Array, media: string): DataEntry;
|
|
83
|
+
/** The value of a record entry. */
|
|
84
|
+
export declare function recordValue(entry: DataEntry): JsonValue;
|
|
85
|
+
/** True when the entry's sha256 is the hash of its body. */
|
|
86
|
+
export declare function entryIntact(entry: DataEntry): boolean;
|
|
87
|
+
/** The digest of a set: sha256 over the sorted lines `<key>\t<sha256>\n`. */
|
|
88
|
+
export declare function setDigest(items: readonly {
|
|
89
|
+
readonly key: string;
|
|
90
|
+
readonly sha256: string;
|
|
91
|
+
}[]): string;
|
|
92
|
+
/** The identity of an export: sha256 of the canonical JSON of its sets. */
|
|
93
|
+
export declare function exportDigestOf(sets: readonly DataSetSummary[]): string;
|
|
94
|
+
/** The schema id of a set: `<format without @major>/<set>@<major>` for records, `blob@1` for blobs. */
|
|
95
|
+
export declare function setSchemaOf(format: string, set: string, form: EntryForm): string;
|
|
96
|
+
/**
|
|
97
|
+
* Builds summaries from entries as they pass, refusing an entry whose hash does not match its body, a key repeated in
|
|
98
|
+
* its set, or a set whose entries mix forms.
|
|
99
|
+
*/
|
|
100
|
+
export declare class ManifestBuilder {
|
|
101
|
+
private readonly tallies;
|
|
102
|
+
/** Counts one entry of `store`. */
|
|
103
|
+
add(store: string, entry: DataEntry): void;
|
|
104
|
+
/** The finished manifest. */
|
|
105
|
+
build(header: ManifestHeader): DataManifest;
|
|
106
|
+
}
|
|
107
|
+
/** The reasons a manifest document is not a valid `starci/data-manifest@1`, empty when it is. */
|
|
108
|
+
export declare function manifestProblems(document: unknown): readonly string[];
|