@starci/seed 0.0.0-stage → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,309 @@
1
+ import { spawnSync } from "node:child_process";
2
+ import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs";
3
+ import { tmpdir } from "node:os";
4
+ import { dirname, join, relative, sep } from "node:path";
5
+ import { memorySpool, spooledSource } from "../copies.js";
6
+ import { DataRefusal, holdingFor } from "../driver.js";
7
+ import { ManifestBuilder, blobEntry, byteOrder, canonicalJson, manifestProblems, recordEntry, recordValue, sha256Hex } from "../manifest.js";
8
+ import { GIT_CAP_BYTES, GIT_SHARD_BYTES, GIT_WARN_BYTES } from "../words.js";
9
+ const SEALED_SUFFIX = ".sops.json";
10
+ /** Plaintext budget of one record shard, so the sealed file (base64 + envelope) stays under GIT_SHARD_BYTES. */
11
+ const SHARD_PLAIN_BYTES = Math.floor(GIT_SHARD_BYTES * 0.7);
12
+ const BLOB_CHUNK_BYTES = Math.floor(GIT_SHARD_BYTES * 0.7);
13
+ const COPY_ROOTS = ["data", "blobs", "index"];
14
+ const GIT_ATTRIBUTES = "* -text\n";
15
+ const defaultRunner = (program, args, options) => {
16
+ const result = spawnSync(program, [...args], { cwd: options.cwd, input: options.input, env: { ...process.env, ...options.env }, maxBuffer: 1 << 30 });
17
+ return { status: result.status, stdout: result.stdout, stderr: result.stderr };
18
+ };
19
+ function posix(path) {
20
+ return path.split(sep).join("/");
21
+ }
22
+ function recordShardLines(records) {
23
+ const lines = records.map((entry) => `${JSON.stringify(entry.key)}:${canonicalJson(recordValue(entry))}`);
24
+ return Buffer.from(`{\n${lines.join(",\n")}\n}\n`, "utf8");
25
+ }
26
+ function shardOf(key, width) {
27
+ return width === 0 ? "all" : sha256Hex(key).slice(0, width);
28
+ }
29
+ function shardWidth(records, previous) {
30
+ for (let width = previous; width < 8; width += 1) {
31
+ const sizes = new Map();
32
+ for (const entry of records)
33
+ sizes.set(shardOf(entry.key, width), (sizes.get(shardOf(entry.key, width)) ?? 0) + entry.body.byteLength + entry.key.length + 8);
34
+ if ([...sizes.values()].every((size) => size <= SHARD_PLAIN_BYTES))
35
+ return width;
36
+ }
37
+ return 8;
38
+ }
39
+ /** Builds the git driver over a local working copy. */
40
+ export function createGitDriver(options) {
41
+ const run = options.run ?? defaultRunner;
42
+ const branch = options.branch ?? "main";
43
+ const cap = options.capBytes ?? GIT_CAP_BYTES;
44
+ const warn = options.warnBytes ?? GIT_WARN_BYTES;
45
+ const env = options.env ?? {};
46
+ const limits = { forms: ["record", "blob"], maxFileBytes: GIT_SHARD_BYTES, maxCopyBytes: cap, warnCopyBytes: warn };
47
+ const git = (args) => {
48
+ const result = run("git", args, { cwd: options.checkout, env });
49
+ if (result.status !== 0)
50
+ throw new DataRefusal("DATA_DRIVER_UNAVAILABLE", "transient", { driver: options.id, step: `git ${args.slice(0, 1).join("")}` });
51
+ return result.stdout;
52
+ };
53
+ const sops = (args, input, identityEnv) => {
54
+ const result = run("sops", args, { cwd: tmpdir(), input, env: { ...env, ...identityEnv } });
55
+ if (result.status !== 0)
56
+ throw new DataRefusal("DATA_SEAL_FAILED", "configuration", { driver: options.id, step: `sops ${args.slice(0, 1).join("")}` });
57
+ return result.stdout;
58
+ };
59
+ const seal = (path, plain, recipients) => sops(["encrypt", "--input-type", "binary", "--output-type", "json", "--filename-override", path, "--age", recipients.join(",")], plain, {});
60
+ const open = (path, sealed, context) => sops(["decrypt", "--input-type", "json", "--output-type", "binary", "--filename-override", path, "--decryption-order", "age"], sealed, context.identity.env);
61
+ const prepare = () => {
62
+ if (!existsSync(join(options.checkout, ".git"))) {
63
+ mkdirSync(options.checkout, { recursive: true });
64
+ const cloned = options.remote !== undefined && run("git", ["clone", "-q", "-b", branch, options.remote, "."], { cwd: options.checkout, env }).status === 0;
65
+ if (!cloned) {
66
+ git(["init", "-q", "-b", branch]);
67
+ if (options.remote !== undefined)
68
+ git(["remote", "add", "origin", options.remote]);
69
+ }
70
+ }
71
+ else if (options.remote !== undefined) {
72
+ const fetched = run("git", ["pull", "-q", "--ff-only", "origin", branch], { cwd: options.checkout, env });
73
+ if (fetched.status !== 0 && run("git", ["rev-parse", "--verify", "-q", "HEAD"], { cwd: options.checkout, env }).status === 0)
74
+ throw new DataRefusal("DATA_DRIVER_UNAVAILABLE", "transient", { driver: options.id, step: "git pull" });
75
+ }
76
+ };
77
+ /** Copy files are bytes, never text: no line-ending conversion may touch a sealed envelope or a digest. */
78
+ const pinBytes = () => {
79
+ const file = join(options.checkout, ".gitattributes");
80
+ if (!existsSync(file) || readFileSync(file, "utf8") !== GIT_ATTRIBUTES)
81
+ writeFileSync(file, GIT_ATTRIBUTES);
82
+ };
83
+ const headManifest = () => {
84
+ const file = join(options.checkout, "manifest.json");
85
+ return existsSync(file) ? JSON.parse(readFileSync(file, "utf8")) : null;
86
+ };
87
+ const showAt = (commit, path) => git(["show", `${commit}:${path}`]);
88
+ const manifestAt = (commit) => JSON.parse(showAt(commit, "manifest.json").toString("utf8"));
89
+ const copyBytesOnDisk = () => {
90
+ const found = [];
91
+ for (const root of COPY_ROOTS) {
92
+ const dir = join(options.checkout, root);
93
+ if (!existsSync(dir))
94
+ continue;
95
+ for (const name of readdirSync(dir, { recursive: true, encoding: "utf8" })) {
96
+ const full = join(dir, name);
97
+ if (statSync(full).isFile())
98
+ found.push(posix(relative(options.checkout, full)));
99
+ }
100
+ }
101
+ return found;
102
+ };
103
+ /** Every file of the new snapshot: path (sealed name when sealed) -> plaintext. */
104
+ const planFiles = async (source, previous) => {
105
+ const bySet = new Map();
106
+ for await (const item of source.entries()) {
107
+ const id = `${item.store}/${item.entry.set}`;
108
+ bySet.set(id, [...(bySet.get(id) ?? []), item.entry]);
109
+ }
110
+ const files = new Map();
111
+ const shards = {};
112
+ for (const set of source.manifest.sets) {
113
+ const id = `${set.store}/${set.set}`;
114
+ const entries = [...(bySet.get(id) ?? [])].sort((left, right) => byteOrder(left.key, right.key));
115
+ if (set.form === "record") {
116
+ const width = shardWidth(entries, previous?.shards[id] ?? 0);
117
+ shards[id] = width;
118
+ const groups = new Map();
119
+ for (const entry of entries)
120
+ groups.set(shardOf(entry.key, width), [...(groups.get(shardOf(entry.key, width)) ?? []), entry]);
121
+ for (const [shard, records] of groups)
122
+ files.set(`data/${id}/${shard}.json`, recordShardLines(records));
123
+ continue;
124
+ }
125
+ const index = {};
126
+ for (const entry of entries) {
127
+ const blobId = sha256Hex(`${entry.set}\u0000${entry.key}`);
128
+ const chunks = Math.max(1, Math.ceil(entry.body.byteLength / BLOB_CHUNK_BYTES));
129
+ for (let at = 0; at < chunks; at += 1) {
130
+ const name = chunks === 1 ? `${blobId}.bin` : `${blobId}.${String(at)}.bin`;
131
+ files.set(`blobs/${id}/${blobId.slice(0, 2)}/${name}`, Buffer.from(entry.body.subarray(at * BLOB_CHUNK_BYTES, (at + 1) * BLOB_CHUNK_BYTES)));
132
+ }
133
+ index[entry.key] = { media: entry.media, sha256: entry.sha256, chunks };
134
+ }
135
+ files.set(`index/${id}.json`, Buffer.from(`${canonicalJson(index)}\n`, "utf8"));
136
+ }
137
+ return { files, shards };
138
+ };
139
+ const fileText = (commit, manifest, path, context) => {
140
+ const stored = manifest.sealing === "none" ? path : `${path}${SEALED_SUFFIX}`;
141
+ const bytes = showAt(commit, stored);
142
+ if (manifest.sealing === "none")
143
+ return bytes;
144
+ if (context === null)
145
+ throw new DataRefusal("DATA_SEAL_FAILED", "configuration", { driver: options.id, reason: "an identity is needed to open a sealed copy" });
146
+ return open(stored, bytes, context);
147
+ };
148
+ const entriesAt = (commit, manifest, context) => {
149
+ const out = [];
150
+ for (const set of manifest.sets) {
151
+ const id = `${set.store}/${set.set}`;
152
+ if (set.form === "record") {
153
+ const shards = Object.keys(manifest.files).filter((path) => path.startsWith(`data/${id}/`)).sort(byteOrder);
154
+ const records = [];
155
+ for (const path of shards) {
156
+ const values = JSON.parse(fileText(commit, manifest, path, context).toString("utf8"));
157
+ for (const [key, value] of Object.entries(values))
158
+ records.push(recordEntry(set.set, key, value));
159
+ }
160
+ for (const entry of records.sort((left, right) => byteOrder(left.key, right.key)))
161
+ out.push({ store: set.store, entry });
162
+ continue;
163
+ }
164
+ const index = JSON.parse(fileText(commit, manifest, `index/${id}.json`, context).toString("utf8"));
165
+ for (const [key, item] of Object.entries(index).sort(([left], [right]) => byteOrder(left, right))) {
166
+ const blobId = sha256Hex(`${set.set}\u0000${key}`);
167
+ const parts = [];
168
+ for (let at = 0; at < item.chunks; at += 1) {
169
+ const name = item.chunks === 1 ? `${blobId}.bin` : `${blobId}.${String(at)}.bin`;
170
+ parts.push(fileText(commit, manifest, `blobs/${id}/${blobId.slice(0, 2)}/${name}`, context));
171
+ }
172
+ out.push({ store: set.store, entry: blobEntry(set.set, key, new Uint8Array(Buffer.concat(parts)), item.media) });
173
+ }
174
+ }
175
+ return out;
176
+ };
177
+ const refOf = (commit, manifest) => ({ driver: options.id, kind: "git", locator: commit, exportDigest: manifest.exportDigest, takenAt: manifest.takenAt });
178
+ return {
179
+ id: options.id,
180
+ kind: "git",
181
+ limits: () => limits,
182
+ plan: (manifest) => holdingFor(manifest, limits.forms),
183
+ async write(source, context) {
184
+ const sealed = source.manifest.purpose === "seed-backup";
185
+ if (sealed && context.recipients.length === 0)
186
+ throw new DataRefusal("DATA_SEAL_FAILED", "configuration", { driver: options.id, reason: "a seed-backup is sealed to its environment's recipients and none were given" });
187
+ if (!context.dryRun) {
188
+ prepare();
189
+ pinBytes();
190
+ }
191
+ const previous = headManifest();
192
+ const { files, shards } = await planFiles(source, previous);
193
+ const facts = {};
194
+ const writes = new Map();
195
+ let total = 0;
196
+ for (const [path, plain] of [...files.entries()].sort(([left], [right]) => byteOrder(left, right))) {
197
+ const stored = sealed ? `${path}${SEALED_SUFFIX}` : path;
198
+ const plainSha = sha256Hex(plain);
199
+ const before = previous?.files[path];
200
+ const onDisk = join(options.checkout, stored);
201
+ const keep = before?.plain === plainSha && previous?.sealing === (sealed ? "sops-age" : "none") && existsSync(onDisk);
202
+ const bytes = keep ? statSync(onDisk).size : (() => {
203
+ const out = sealed ? seal(stored, plain, context.recipients) : plain;
204
+ writes.set(stored, out);
205
+ return out.byteLength;
206
+ })();
207
+ if (bytes > GIT_SHARD_BYTES)
208
+ throw new DataRefusal("DATA_GIT_CAP_EXCEEDED", "configuration", { driver: options.id, file: path, bytes, limit: GIT_SHARD_BYTES });
209
+ facts[path] = { plain: plainSha, bytes };
210
+ total += bytes;
211
+ }
212
+ if (total > cap)
213
+ throw new DataRefusal("DATA_GIT_CAP_EXCEEDED", "configuration", { driver: options.id, bytes: total, cap, next: "s3" });
214
+ const warnings = total > warn ? ["DATA_GIT_CAP_NEAR"] : [];
215
+ const holding = holdingFor(source.manifest, limits.forms);
216
+ const at = new Date().toISOString();
217
+ const manifest = { ...source.manifest, sealing: sealed ? "sops-age" : "none", recipients: sealed ? [...context.recipients].sort() : [], shards, files: facts };
218
+ const keepPaths = new Set(Object.keys(facts).map((path) => (sealed ? `${path}${SEALED_SUFFIX}` : path)));
219
+ const stale = copyBytesOnDisk().filter((path) => !keepPaths.has(path));
220
+ if (context.dryRun) {
221
+ const ref = { driver: options.id, kind: "git", locator: "dry-run", exportDigest: manifest.exportDigest, takenAt: manifest.takenAt };
222
+ return { schema: "starci/data-copy@1", ref, holding, bytes: total, changed: writes.size + stale.length, warnings, at };
223
+ }
224
+ for (const path of stale)
225
+ rmSync(join(options.checkout, path));
226
+ for (const [path, bytes] of writes) {
227
+ mkdirSync(dirname(join(options.checkout, path)), { recursive: true });
228
+ writeFileSync(join(options.checkout, path), bytes);
229
+ }
230
+ writeFileSync(join(options.checkout, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`);
231
+ git(["add", "-A", "--", ".gitattributes", "manifest.json", ...COPY_ROOTS.filter((root) => existsSync(join(options.checkout, root)))]);
232
+ const staged = run("git", ["diff", "--cached", "--quiet"], { cwd: options.checkout, env }).status !== 0;
233
+ const unchanged = !staged || previous?.exportDigest === manifest.exportDigest && writes.size === 0 && stale.length === 0;
234
+ if (staged && !unchanged)
235
+ git(["commit", "-q", "-m", `${manifest.format} ${manifest.exportDigest.slice(0, 12)} ${manifest.takenAt}`]);
236
+ else if (staged)
237
+ git(["reset", "-q", "--hard", "HEAD"]);
238
+ if (options.remote !== undefined)
239
+ git(["push", "-q", "origin", `HEAD:${branch}`]);
240
+ const commit = git(["rev-parse", "HEAD"]).toString("utf8").trim();
241
+ return { schema: "starci/data-copy@1", ref: refOf(commit, headManifest() ?? manifest), holding, bytes: total, changed: unchanged ? 0 : writes.size + stale.length, warnings, at };
242
+ },
243
+ list() {
244
+ if (!existsSync(join(options.checkout, ".git")))
245
+ return Promise.resolve([]);
246
+ const log = run("git", ["log", "-n", "30", "--format=%H", "--", "manifest.json"], { cwd: options.checkout, env });
247
+ if (log.status !== 0)
248
+ return Promise.resolve([]);
249
+ const commits = log.stdout.toString("utf8").split("\n").filter((line) => line.trim() !== "");
250
+ return Promise.resolve(commits.map((commit) => refOf(commit, manifestAt(commit))));
251
+ },
252
+ read(ref, context) {
253
+ const manifest = manifestAt(ref.locator);
254
+ const spool = memorySpool();
255
+ const entries = entriesAt(ref.locator, manifest, context);
256
+ return Promise.all(entries.map((item) => spool.append(item))).then(() => spooledSource(manifest, spool));
257
+ },
258
+ verify(ref, context) {
259
+ const checks = [];
260
+ let manifest;
261
+ try {
262
+ manifest = manifestAt(ref.locator);
263
+ }
264
+ catch {
265
+ return Promise.resolve({ ref, healthy: false, checks: [{ name: "manifest", ok: false, detail: "manifest.json cannot be read at the commit" }], manifest: null });
266
+ }
267
+ const problems = manifestProblems(manifest);
268
+ checks.push({ name: "manifest", ok: problems.length === 0, detail: problems.length === 0 ? "starci/data-manifest@1, exportDigest matches the sets" : problems.join("; ") });
269
+ const sealedPolicy = manifest.purpose === "seed-backup" ? "sops-age" : "none";
270
+ checks.push({ name: "sealing", ok: manifest.sealing === sealedPolicy && (sealedPolicy === "none" || manifest.recipients.length > 0), detail: `purpose ${manifest.purpose} requires ${sealedPolicy}, the copy says ${manifest.sealing}` });
271
+ const tree = git(["ls-tree", "-r", "-l", ref.locator]).toString("utf8").split("\n").filter((line) => line.trim() !== "");
272
+ const listed = tree.flatMap((line) => {
273
+ const match = /^\S+ \S+ \S+\s+(\d+)\t(.+)$/u.exec(line);
274
+ return match === null ? [] : [{ path: String(match[2]), bytes: Number(match[1]) }];
275
+ });
276
+ const sizes = new Set(listed.map((item) => item.path));
277
+ const copyListed = listed.filter((item) => COPY_ROOTS.some((root) => item.path.startsWith(`${root}/`)));
278
+ const copyFiles = copyListed.map((item) => item.path);
279
+ const expected = new Set(Object.keys(manifest.files).map((path) => (manifest.sealing === "none" ? path : `${path}${SEALED_SUFFIX}`)));
280
+ const stray = copyFiles.filter((path) => !expected.has(path));
281
+ const missing = [...expected].filter((path) => !sizes.has(path));
282
+ const unsealed = manifest.sealing === "none" ? [] : copyFiles.filter((path) => !path.endsWith(SEALED_SUFFIX));
283
+ checks.push({ name: "files", ok: stray.length === 0 && missing.length === 0 && unsealed.length === 0, detail: `${String(copyFiles.length)} files; stray ${String(stray.length)}, missing ${String(missing.length)}, unsealed ${String(unsealed.length)}` });
284
+ if (manifest.sealing !== "none") {
285
+ const notEnvelope = copyFiles.filter((path) => path.endsWith(SEALED_SUFFIX)).filter((path) => {
286
+ const text = showAt(ref.locator, path).toString("utf8");
287
+ return !text.includes('"sops"') || !text.includes("ENC[AES256_GCM");
288
+ });
289
+ checks.push({ name: "sealed", ok: notEnvelope.length === 0, detail: notEnvelope.length === 0 ? "every data file is a sops envelope" : `${String(notEnvelope.length)} files are not sops envelopes` });
290
+ }
291
+ const total = copyListed.reduce((sum, item) => sum + item.bytes, 0);
292
+ checks.push({ name: "cap", ok: total <= cap, detail: `${String(total)} bytes of ${String(cap)}` });
293
+ if (context !== null || manifest.sealing === "none") {
294
+ try {
295
+ const builder = new ManifestBuilder();
296
+ for (const item of entriesAt(ref.locator, manifest, context))
297
+ builder.add(item.store, item.entry);
298
+ const rebuilt = builder.build(manifest);
299
+ const wrong = manifest.sets.filter((set) => rebuilt.sets.find((other) => other.store === set.store && other.set === set.set)?.digest !== set.digest);
300
+ checks.push({ name: "digests", ok: wrong.length === 0, detail: wrong.length === 0 ? "every set matches its digest and count" : `differ: ${wrong.map((set) => `${set.store}/${set.set}`).join(", ")}` });
301
+ }
302
+ catch (cause) {
303
+ checks.push({ name: "digests", ok: false, detail: cause instanceof DataRefusal ? cause.code : "a file cannot be read or parsed" });
304
+ }
305
+ }
306
+ return Promise.resolve({ ref, healthy: checks.every((check) => check.ok), checks, manifest });
307
+ },
308
+ };
309
+ }
@@ -0,0 +1,48 @@
1
+ import type { OpenContext } from "../driver.js";
2
+ import type { Runner } from "./driver.js";
3
+ /**
4
+ * File-level reads of a copy repository at one pinned commit, and writes into a working tree for review: what a
5
+ * project's own extension of the standard (a kind it defines on top, with its own document) reads through, so no app
6
+ * writes git or sops code. The data driver (`createGitDriver`) stays the one writer of the generic layout.
7
+ */
8
+ export interface PinnedRepository {
9
+ /** The commit every read is of (40 hex digits, verified to exist). */
10
+ readonly commit: string;
11
+ /** The bytes of a repository-relative file at the commit, or null when the commit holds no such file. */
12
+ readBytes(path: string): Buffer | null;
13
+ /** The text of a repository-relative file at the commit, or null. */
14
+ readText(path: string): string | null;
15
+ /** Opens a sops-sealed file of the commit IN MEMORY with the identity (sops on stdin/stdout); the plaintext never reaches a file. */
16
+ openSealed(path: string, context: OpenContext): Buffer | null;
17
+ }
18
+ /** Where a pinned repository is fetched from and into. */
19
+ export interface PinOptions {
20
+ /** A git remote URL (never one holding a credential) or a local repository path. */
21
+ readonly repository: string;
22
+ /** The commit, 40 hex digits. */
23
+ readonly commit: string;
24
+ /** A cache directory outside every code repository; one bare mirror per repository name lives under it. */
25
+ readonly cache: string;
26
+ /** The mirror's folder name under the cache (the repository's name). */
27
+ readonly name: string;
28
+ /** Extra environment for git and sops. */
29
+ readonly env?: Readonly<Record<string, string>>;
30
+ /** The program runner; spawnSync by default. */
31
+ readonly run?: Runner;
32
+ }
33
+ /** Fetches the repository into its bare mirror under the cache and answers it at the pinned commit; refuses a commit it cannot find. */
34
+ export declare function pinRepository(options: PinOptions): PinnedRepository;
35
+ /** A working tree a person reviews and commits: writes text, and seals values through sops with the tree's own `.sops.yaml` rules. */
36
+ export interface ReviewTree {
37
+ /** The text of a tree-relative file, or null. */
38
+ readText(path: string): string | null;
39
+ /** Writes a tree-relative text file; true when its content changed. */
40
+ writeText(path: string, text: string): boolean;
41
+ /** Seals a plaintext document (yaml or json by extension) into `path` with the tree's `.sops.yaml` creation rule for that path; the plaintext is sops's stdin only. True when it wrote. */
42
+ seal(path: string, plaintext: Buffer): boolean;
43
+ }
44
+ /** Opens a git working tree for review writes; refuses a folder that is not one. */
45
+ export declare function openReviewTree(root: string, options?: {
46
+ readonly env?: Readonly<Record<string, string>>;
47
+ readonly run?: Runner;
48
+ }): ReviewTree;
@@ -0,0 +1,78 @@
1
+ import { spawnSync } from "node:child_process";
2
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
3
+ import { dirname, join, normalize, posix, win32 } from "node:path";
4
+ import { DataRefusal } from "../driver.js";
5
+ const defaultRunner = (program, args, options) => {
6
+ const result = spawnSync(program, [...args], { cwd: options.cwd, input: options.input, env: { ...process.env, ...options.env }, maxBuffer: 1 << 30 });
7
+ return { status: result.status, stdout: result.stdout, stderr: result.stderr };
8
+ };
9
+ function safePath(path) {
10
+ const clean = normalize(path).split("\\").join("/");
11
+ // A repository path is the same on every machine: a drive or root prefix of either platform is refused on both.
12
+ if (posix.isAbsolute(path) || win32.isAbsolute(path) || clean.startsWith("../") || clean === "..")
13
+ throw new DataRefusal("DATA_COPY_MISSING", "configuration", { path });
14
+ return clean;
15
+ }
16
+ /** Fetches the repository into its bare mirror under the cache and answers it at the pinned commit; refuses a commit it cannot find. */
17
+ export function pinRepository(options) {
18
+ const run = options.run ?? defaultRunner;
19
+ const env = options.env ?? {};
20
+ if (!/^[0-9a-f]{40}$/u.test(options.commit))
21
+ throw new DataRefusal("DATA_COPY_MISSING", "configuration", { reason: "the pin is not a 40-hex commit", name: options.name });
22
+ const mirror = join(options.cache, `${options.name}.git`);
23
+ mkdirSync(options.cache, { recursive: true });
24
+ const git = (args) => run("git", args, { cwd: mirror, env });
25
+ const has = () => git(["cat-file", "-e", `${options.commit}^{commit}`]).status === 0;
26
+ if (!existsSync(mirror) && run("git", ["clone", "-q", "--bare", options.repository, mirror], { cwd: options.cache, env }).status !== 0)
27
+ throw new DataRefusal("DATA_DRIVER_UNAVAILABLE", "transient", { step: "git clone", name: options.name });
28
+ if (!has())
29
+ git(["fetch", "-q", "origin", "+refs/heads/*:refs/heads/*"]);
30
+ if (!has())
31
+ throw new DataRefusal("DATA_COPY_MISSING", "configuration", { reason: "the pinned commit is not in the repository", name: options.name });
32
+ const readBytes = (path) => {
33
+ const shown = git(["show", `${options.commit}:${safePath(path)}`]);
34
+ return shown.status === 0 ? shown.stdout : null;
35
+ };
36
+ return {
37
+ commit: options.commit,
38
+ readBytes,
39
+ readText: (path) => readBytes(path)?.toString("utf8") ?? null,
40
+ openSealed(path, context) {
41
+ const sealed = readBytes(path);
42
+ if (sealed === null)
43
+ return null;
44
+ const type = path.endsWith(".json") ? "json" : path.endsWith(".yaml") || path.endsWith(".yml") ? "yaml" : "binary";
45
+ const opened = run("sops", ["decrypt", "--input-type", type, "--output-type", type, "--filename-override", safePath(path), "--decryption-order", "age"], { cwd: options.cache, env: { ...env, ...context.identity.env }, input: sealed });
46
+ if (opened.status !== 0)
47
+ throw new DataRefusal("DATA_SEAL_FAILED", "configuration", { reason: "the sealed file does not open with this identity", path: safePath(path) });
48
+ return opened.stdout;
49
+ },
50
+ };
51
+ }
52
+ /** Opens a git working tree for review writes; refuses a folder that is not one. */
53
+ export function openReviewTree(root, options = {}) {
54
+ const run = options.run ?? defaultRunner;
55
+ if (!existsSync(join(root, ".git")))
56
+ throw new DataRefusal("DATA_COPY_MISSING", "configuration", { reason: "not a git working tree", root });
57
+ const full = (path) => join(root, safePath(path));
58
+ const readText = (path) => (existsSync(full(path)) ? readFileSync(full(path), "utf8") : null);
59
+ return {
60
+ readText,
61
+ writeText(path, text) {
62
+ if (readText(path) === text)
63
+ return false;
64
+ mkdirSync(dirname(full(path)), { recursive: true });
65
+ writeFileSync(full(path), text);
66
+ return true;
67
+ },
68
+ seal(path, plaintext) {
69
+ const type = path.endsWith(".json") ? "json" : "yaml";
70
+ const sealed = run("sops", ["encrypt", "--input-type", type, "--output-type", type, "--filename-override", safePath(path)], { cwd: root, env: options.env ?? {}, input: plaintext });
71
+ if (sealed.status !== 0)
72
+ throw new DataRefusal("DATA_SEAL_FAILED", "configuration", { reason: "sops could not seal with the tree's rules", path: safePath(path) });
73
+ mkdirSync(dirname(full(path)), { recursive: true });
74
+ writeFileSync(full(path), sealed.stdout);
75
+ return true;
76
+ },
77
+ };
78
+ }
@@ -0,0 +1,6 @@
1
+ export { createGitDriver } from "./driver.js";
2
+ export type { GitDriverOptions, RunResult, Runner } from "./driver.js";
3
+ export { openReviewTree, pinRepository } from "./files.js";
4
+ export type { PinOptions, PinnedRepository, ReviewTree } from "./files.js";
5
+ export { STANDARD_ENVIRONMENT, STANDARD_OUTPUTS, STANDARD_RAW, STANDARD_TAKEN_AT, renderStandard, runStandardAdapter } from "./standard.js";
6
+ export type { AdapterIo, RenderOptions, StandardContent, StandardConvert } from "./standard.js";
@@ -0,0 +1,3 @@
1
+ export { createGitDriver } from "./driver.js";
2
+ export { openReviewTree, pinRepository } from "./files.js";
3
+ export { STANDARD_ENVIRONMENT, STANDARD_OUTPUTS, STANDARD_RAW, STANDARD_TAKEN_AT, renderStandard, runStandardAdapter } from "./standard.js";
@@ -0,0 +1,60 @@
1
+ import type { StoredEntry } from "../copies.js";
2
+ import type { Runner } from "./driver.js";
3
+ /**
4
+ * The adapter of a seed-standard repository (PROJECT-STACKS-13): a standard whose content is awkward to author as JSON
5
+ * records keeps what people write in `raw/` and an `adapter/` whose `convert(rawDir)` turns it into entries; this file
6
+ * writes those entries through the one git driver into the standard layout (`manifest.json`, `data/`, `blobs/`,
7
+ * `index/`), so an adapter never writes a copy file itself.
8
+ *
9
+ * Determinism is the contract the commit hook proves (R434, HFS_DATA_ADAPTER_DRIFT): the same `raw/` yields the same
10
+ * bytes. The driver already sorts sets and keys and writes canonical JSON; this file fixes the rest: `takenAt` is
11
+ * STANDARD_TAKEN_AT (a standard is identified by its commit, never by a time) and `source.environment` is
12
+ * STANDARD_ENVIRONMENT. What is left to the adapter is its content: no clock, no random id, no order the file system
13
+ * chose.
14
+ */
15
+ /** The `takenAt` of every standard an adapter renders: fixed, so the manifest is a function of `raw/` alone. */
16
+ export declare const STANDARD_TAKEN_AT = "1970-01-01T00:00:00.000Z";
17
+ /** The `source.environment` of a rendered standard: it belongs to every environment that loads it, so it names none. */
18
+ export declare const STANDARD_ENVIRONMENT = "standard";
19
+ /** What an adapter run writes at the root of the standard, and all it may write: the rest of the repository is people's. */
20
+ export declare const STANDARD_OUTPUTS: readonly string[];
21
+ /** The folder people author in, kept plain; never written by an adapter. */
22
+ export declare const STANDARD_RAW = "raw";
23
+ /** What `convert` answers: the standard's format (`<name>-seed-standard@<major>`), its project and every entry with its store. */
24
+ export interface StandardContent {
25
+ /** `<repository name>@<major>`; the name ends `-seed-standard`. */
26
+ readonly format: string;
27
+ /** The project the standard seeds. */
28
+ readonly project: string;
29
+ /** Every entry, each with the store id it is loaded into (a store `data.stores` of the app declares). Any order. */
30
+ readonly entries: readonly StoredEntry[];
31
+ }
32
+ /** An adapter's one export: reads `rawDir` (the standard's `raw/`) and answers its content; it writes nothing. */
33
+ export type StandardConvert = (rawDir: string) => StandardContent | Promise<StandardContent>;
34
+ /** Seams of a render: the program runner of the git driver, and where its scratch working copy is made. */
35
+ export interface RenderOptions {
36
+ /** Runs git; spawnSync by default. */
37
+ readonly run?: Runner;
38
+ /** The parent of the scratch working copy; the system temp directory by default. */
39
+ readonly scratch?: string;
40
+ }
41
+ /**
42
+ * Writes `content` into `outDir` as a seed-standard: the entries go through `createGitDriver` into a scratch working copy
43
+ * (plaintext, the standard's purpose), and its `manifest.json`, `data/`, `blobs/` and `index/` replace those of `outDir`.
44
+ * A format that is no standard's is refused `DATA_FORM_REFUSED`; a repeated key or a set mixing forms is the manifest
45
+ * builder's TypeError.
46
+ */
47
+ export declare function renderStandard(content: StandardContent, outDir: string, options?: RenderOptions): Promise<void>;
48
+ /** Where an adapter run prints. */
49
+ export interface AdapterIo {
50
+ /** The directory `--raw` and `--out` resolve against; the process's by default. */
51
+ readonly cwd?: string;
52
+ /** Prints one line of the run; stderr by default. */
53
+ readonly print?: (line: string) => void;
54
+ }
55
+ /**
56
+ * The body of an adapter's `main.ts`: `process.exitCode = await runStandardAdapter(convert, process.argv.slice(2))`.
57
+ * Converts `--raw` (default `raw`) and renders the standard into `--out` (default `.`, the repository root): exit 0;
58
+ * a usage problem is 2, a failed convert or render 1, with the refusal's code or the error on one line.
59
+ */
60
+ export declare function runStandardAdapter(convert: StandardConvert, argv: readonly string[], io?: AdapterIo): Promise<number>;
@@ -0,0 +1,105 @@
1
+ import { cpSync, existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
2
+ import { tmpdir } from "node:os";
3
+ import { join, resolve } from "node:path";
4
+ import { memorySpool, spooledSource } from "../copies.js";
5
+ import { DataRefusal } from "../driver.js";
6
+ import { ManifestBuilder } from "../manifest.js";
7
+ import { createGitDriver } from "./driver.js";
8
+ /**
9
+ * The adapter of a seed-standard repository (PROJECT-STACKS-13): a standard whose content is awkward to author as JSON
10
+ * records keeps what people write in `raw/` and an `adapter/` whose `convert(rawDir)` turns it into entries; this file
11
+ * writes those entries through the one git driver into the standard layout (`manifest.json`, `data/`, `blobs/`,
12
+ * `index/`), so an adapter never writes a copy file itself.
13
+ *
14
+ * Determinism is the contract the commit hook proves (R434, HFS_DATA_ADAPTER_DRIFT): the same `raw/` yields the same
15
+ * bytes. The driver already sorts sets and keys and writes canonical JSON; this file fixes the rest: `takenAt` is
16
+ * STANDARD_TAKEN_AT (a standard is identified by its commit, never by a time) and `source.environment` is
17
+ * STANDARD_ENVIRONMENT. What is left to the adapter is its content: no clock, no random id, no order the file system
18
+ * chose.
19
+ */
20
+ /** The `takenAt` of every standard an adapter renders: fixed, so the manifest is a function of `raw/` alone. */
21
+ export const STANDARD_TAKEN_AT = "1970-01-01T00:00:00.000Z";
22
+ /** The `source.environment` of a rendered standard: it belongs to every environment that loads it, so it names none. */
23
+ export const STANDARD_ENVIRONMENT = "standard";
24
+ /** What an adapter run writes at the root of the standard, and all it may write: the rest of the repository is people's. */
25
+ export const STANDARD_OUTPUTS = Object.freeze(["manifest.json", "data", "blobs", "index"]);
26
+ /** The folder people author in, kept plain; never written by an adapter. */
27
+ export const STANDARD_RAW = "raw";
28
+ const STANDARD_FORMAT = /^[a-z0-9][a-z0-9-]*-seed-standard@[1-9]\d*$/u;
29
+ /** The git identity and configuration of the scratch working copy: nothing of the machine's git configuration reaches it. */
30
+ const SCRATCH_GIT = Object.freeze({
31
+ GIT_AUTHOR_NAME: "starci-seed",
32
+ GIT_AUTHOR_EMAIL: "seed@starci.invalid",
33
+ GIT_COMMITTER_NAME: "starci-seed",
34
+ GIT_COMMITTER_EMAIL: "seed@starci.invalid",
35
+ GIT_CONFIG_NOSYSTEM: "1",
36
+ });
37
+ /**
38
+ * Writes `content` into `outDir` as a seed-standard: the entries go through `createGitDriver` into a scratch working copy
39
+ * (plaintext, the standard's purpose), and its `manifest.json`, `data/`, `blobs/` and `index/` replace those of `outDir`.
40
+ * A format that is no standard's is refused `DATA_FORM_REFUSED`; a repeated key or a set mixing forms is the manifest
41
+ * builder's TypeError.
42
+ */
43
+ export async function renderStandard(content, outDir, options = {}) {
44
+ if (!STANDARD_FORMAT.test(content.format))
45
+ throw new DataRefusal("DATA_FORM_REFUSED", "configuration", { format: content.format, reason: "a standard's format is <name>-seed-standard@<major>" });
46
+ const builder = new ManifestBuilder();
47
+ const spool = memorySpool();
48
+ for (const item of content.entries) {
49
+ builder.add(item.store, item.entry);
50
+ await spool.append(item);
51
+ }
52
+ const manifest = builder.build({ format: content.format, purpose: "seed-standard", takenAt: STANDARD_TAKEN_AT, source: { project: content.project, environment: STANDARD_ENVIRONMENT } });
53
+ const scratch = mkdtempSync(join(options.scratch ?? tmpdir(), "seed-standard-"));
54
+ try {
55
+ const config = join(scratch, "gitconfig");
56
+ writeFileSync(config, "");
57
+ const checkout = join(scratch, "copy");
58
+ const driver = createGitDriver({ id: content.format, checkout, env: { ...SCRATCH_GIT, GIT_CONFIG_GLOBAL: config }, ...(options.run === undefined ? {} : { run: options.run }) });
59
+ await driver.write(spooledSource(manifest, spool), { recipients: [], dryRun: false });
60
+ mkdirSync(outDir, { recursive: true });
61
+ for (const name of STANDARD_OUTPUTS) {
62
+ rmSync(join(outDir, name), { recursive: true, force: true });
63
+ if (existsSync(join(checkout, name)))
64
+ cpSync(join(checkout, name), join(outDir, name), { recursive: true });
65
+ }
66
+ }
67
+ finally {
68
+ rmSync(scratch, { recursive: true, force: true });
69
+ }
70
+ }
71
+ const ADAPTER_USAGE = "usage: node adapter/main.ts [--raw <dir>] [--out <dir>] (defaults: raw and the repository root, from the repository root)";
72
+ /** The `--raw` and `--out` of an adapter run, or a usage problem. */
73
+ function adapterArgs(argv) {
74
+ const values = { "--raw": STANDARD_RAW, "--out": "." };
75
+ for (let at = 0; at < argv.length; at += 2) {
76
+ const flag = String(argv[at]);
77
+ const value = argv[at + 1];
78
+ if (!(flag in values) || value === undefined || value.startsWith("--"))
79
+ return `${flag} is no flag of an adapter run, or has no value`;
80
+ values[flag] = value;
81
+ }
82
+ return { raw: String(values["--raw"]), out: String(values["--out"]) };
83
+ }
84
+ /**
85
+ * The body of an adapter's `main.ts`: `process.exitCode = await runStandardAdapter(convert, process.argv.slice(2))`.
86
+ * Converts `--raw` (default `raw`) and renders the standard into `--out` (default `.`, the repository root): exit 0;
87
+ * a usage problem is 2, a failed convert or render 1, with the refusal's code or the error on one line.
88
+ */
89
+ export async function runStandardAdapter(convert, argv, io = {}) {
90
+ const print = io.print ?? ((line) => void process.stderr.write(`${line}\n`));
91
+ const args = adapterArgs(argv);
92
+ if (typeof args === "string") {
93
+ print(`${args}; ${ADAPTER_USAGE}`);
94
+ return 2;
95
+ }
96
+ const cwd = io.cwd ?? process.cwd();
97
+ try {
98
+ await renderStandard(await convert(resolve(cwd, args.raw)), resolve(cwd, args.out));
99
+ return 0;
100
+ }
101
+ catch (cause) {
102
+ print(`the adapter failed: ${cause instanceof DataRefusal ? cause.code : String(cause)}`);
103
+ return 1;
104
+ }
105
+ }
@@ -0,0 +1,12 @@
1
+ export { DATA_PURPOSES, DRIVER_KINDS, GIT_CAP_BYTES, GIT_SHARD_BYTES, GIT_WARN_BYTES, STORE_KINDS } from "./words.js";
2
+ export type { DataPurpose, DriverKind, EntryForm, StoreKind, StoreLevel } from "./words.js";
3
+ export { ManifestBuilder, blobEntry, byteOrder, canonicalJson, entryIntact, exportDigestOf, manifestProblems, recordEntry, recordValue, setDigest, setSchemaOf, sha256Hex, } from "./manifest.js";
4
+ export type { DataEntry, DataManifest, DataSetSummary, DataSource, JsonValue, ManifestHeader, Selection } from "./manifest.js";
5
+ export { importOutcome } from "./store.js";
6
+ export type { ImportOptions, ImportReport, Inability, RefusedKey, SetImport, StoreAdapter, StoreDescription, StoreSet } from "./store.js";
7
+ export { DataRefusal, holdingFor } from "./driver.js";
8
+ export type { CopyReceipt, CopyRef, CopyVerdict, DataRefusalCode, DriverAdapter, DriverLimits, ExportSource, Holding, OmittedSet, OpenContext, VerifyCheck, WriteContext, } from "./driver.js";
9
+ export { causeLabel, chooseCopy, compareCopies, exportOnce, memorySpool, restoreFrom, spooledSource, unchangedSince, writeEverywhere } from "./copies.js";
10
+ export type { CopyDifference, CopyLeg, CopyRun, Spool, StoredEntry } from "./copies.js";
11
+ export { DATA_PROTOCOL, commandStore, decodeEntry, encodeEntry, serveStore } from "./protocol.js";
12
+ export type { CommandRunner, EntryLine, ServeIo } from "./protocol.js";
package/dist/index.js ADDED
@@ -0,0 +1,6 @@
1
+ export { DATA_PURPOSES, DRIVER_KINDS, GIT_CAP_BYTES, GIT_SHARD_BYTES, GIT_WARN_BYTES, STORE_KINDS } from "./words.js";
2
+ export { ManifestBuilder, blobEntry, byteOrder, canonicalJson, entryIntact, exportDigestOf, manifestProblems, recordEntry, recordValue, setDigest, setSchemaOf, sha256Hex, } from "./manifest.js";
3
+ export { importOutcome } from "./store.js";
4
+ export { DataRefusal, holdingFor } from "./driver.js";
5
+ export { causeLabel, chooseCopy, compareCopies, exportOnce, memorySpool, restoreFrom, spooledSource, unchangedSince, writeEverywhere } from "./copies.js";
6
+ export { DATA_PROTOCOL, commandStore, decodeEntry, encodeEntry, serveStore } from "./protocol.js";