@starci/seed 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/LICENSE +21 -0
- package/README.md +51 -2
- package/dist/copies.d.ts +83 -0
- package/dist/copies.js +146 -0
- package/dist/driver.d.ts +145 -0
- package/dist/driver.js +21 -0
- package/dist/git/driver.d.ts +53 -0
- package/dist/git/driver.js +309 -0
- package/dist/git/files.d.ts +48 -0
- package/dist/git/files.js +78 -0
- package/dist/git/index.d.ts +4 -0
- package/dist/git/index.js +2 -0
- package/dist/index.d.ts +12 -0
- package/dist/index.js +6 -0
- package/dist/manifest.d.ts +108 -0
- package/dist/manifest.js +131 -0
- package/dist/postgres/index.d.ts +2 -0
- package/dist/postgres/index.js +1 -0
- package/dist/postgres/pipe.d.ts +11 -0
- package/dist/postgres/pipe.js +60 -0
- package/dist/postgres/sql.d.ts +42 -0
- package/dist/postgres/sql.js +54 -0
- package/dist/postgres/store.d.ts +33 -0
- package/dist/postgres/store.js +231 -0
- package/dist/protocol.d.ts +52 -0
- package/dist/protocol.js +101 -0
- package/dist/store.d.ts +97 -0
- package/dist/store.js +6 -0
- package/dist/testing/contract.d.ts +26 -0
- package/dist/testing/contract.js +219 -0
- package/dist/testing/index.d.ts +5 -0
- package/dist/testing/index.js +3 -0
- package/dist/testing/memory.d.ts +53 -0
- package/dist/testing/memory.js +0 -0
- package/dist/words.d.ts +33 -0
- package/dist/words.js +16 -0
- package/package.json +85 -4
package/dist/protocol.js
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { sha256Hex } from "./manifest.js";
|
|
2
|
+
/**
|
|
3
|
+
* The line protocol `starci/data-entries@1` between the harness and an app's store command: newline-delimited JSON.
|
|
4
|
+
*
|
|
5
|
+
* <argv> describe -> one line: StoreDescription
|
|
6
|
+
* <argv> export <Selection JSON> -> one line per entry, then {"end": true, "items": n}
|
|
7
|
+
* <argv> import <ImportOptions> <- entry lines on stdin; -> last line: ImportReport
|
|
8
|
+
* <argv> identity <set> <key> -> one line: {"identity": string | null}
|
|
9
|
+
*
|
|
10
|
+
* A record's body travels as its canonical JSON text, a blob's as base64. A value never reaches argv or a log.
|
|
11
|
+
*/
|
|
12
|
+
export const DATA_PROTOCOL = "starci/data-entries@1";
|
|
13
|
+
const encoder = new TextEncoder();
|
|
14
|
+
const decoder = new TextDecoder("utf-8", { fatal: true });
|
|
15
|
+
/** The protocol line of an entry. */
|
|
16
|
+
export function encodeEntry(entry) {
|
|
17
|
+
const body = entry.form === "record" ? decoder.decode(entry.body) : Buffer.from(entry.body).toString("base64");
|
|
18
|
+
const line = { set: entry.set, key: entry.key, form: entry.form, media: entry.media, sha256: entry.sha256, body };
|
|
19
|
+
return JSON.stringify(line);
|
|
20
|
+
}
|
|
21
|
+
/** The entry of a protocol line; a body that does not match its sha256 is refused. */
|
|
22
|
+
export function decodeEntry(text) {
|
|
23
|
+
const line = JSON.parse(text);
|
|
24
|
+
const body = line.form === "record" ? encoder.encode(line.body) : new Uint8Array(Buffer.from(line.body, "base64"));
|
|
25
|
+
if (sha256Hex(body) !== line.sha256)
|
|
26
|
+
throw new TypeError(`${line.set}/${line.key}: the body does not match its sha256`);
|
|
27
|
+
return { set: line.set, key: line.key, form: line.form, media: line.media, body, sha256: line.sha256 };
|
|
28
|
+
}
|
|
29
|
+
async function* entriesFromLines(lines) {
|
|
30
|
+
for await (const line of lines)
|
|
31
|
+
if (line.trim() !== "")
|
|
32
|
+
yield decodeEntry(line);
|
|
33
|
+
}
|
|
34
|
+
/** Serves one store over the protocol from an app's command; answers the exit code. */
|
|
35
|
+
export async function serveStore(store, io) {
|
|
36
|
+
const [verb, argument] = io.args;
|
|
37
|
+
if (verb === "describe") {
|
|
38
|
+
io.write(JSON.stringify(await store.describe()));
|
|
39
|
+
return 0;
|
|
40
|
+
}
|
|
41
|
+
if (verb === "export") {
|
|
42
|
+
const selection = JSON.parse(argument ?? "{}");
|
|
43
|
+
let items = 0;
|
|
44
|
+
for await (const entry of store.export(selection)) {
|
|
45
|
+
io.write(encodeEntry(entry));
|
|
46
|
+
items += 1;
|
|
47
|
+
}
|
|
48
|
+
io.write(JSON.stringify({ end: true, items }));
|
|
49
|
+
return 0;
|
|
50
|
+
}
|
|
51
|
+
if (verb === "import") {
|
|
52
|
+
const options = JSON.parse(argument ?? '{"dryRun":true,"mode":"merge"}');
|
|
53
|
+
io.write(JSON.stringify(await store.import(entriesFromLines(io.lines), options)));
|
|
54
|
+
return 0;
|
|
55
|
+
}
|
|
56
|
+
if (verb === "identity") {
|
|
57
|
+
io.write(JSON.stringify({ identity: await store.identityOf(argument ?? "", io.args[2] ?? "") }));
|
|
58
|
+
return 0;
|
|
59
|
+
}
|
|
60
|
+
io.write(JSON.stringify({ error: "DATA_PROTOCOL_USAGE", usage: "describe | export <selection> | import <options> | identity <set> <key>" }));
|
|
61
|
+
return 2;
|
|
62
|
+
}
|
|
63
|
+
async function lastLine(lines) {
|
|
64
|
+
let last = "";
|
|
65
|
+
for await (const line of lines)
|
|
66
|
+
if (line.trim() !== "")
|
|
67
|
+
last = line;
|
|
68
|
+
return last;
|
|
69
|
+
}
|
|
70
|
+
async function* linesOf(entries) {
|
|
71
|
+
for await (const entry of entries)
|
|
72
|
+
yield encodeEntry(entry);
|
|
73
|
+
}
|
|
74
|
+
/** A store reached through an app's command: the harness side of the protocol. */
|
|
75
|
+
export function commandStore(declared, run) {
|
|
76
|
+
return {
|
|
77
|
+
...declared,
|
|
78
|
+
describe: async () => JSON.parse(await lastLine(run(["describe"]))),
|
|
79
|
+
async *export(selection) {
|
|
80
|
+
let items = 0;
|
|
81
|
+
let ended = false;
|
|
82
|
+
for await (const line of run(["export", JSON.stringify(selection)])) {
|
|
83
|
+
if (line.trim() === "")
|
|
84
|
+
continue;
|
|
85
|
+
const parsed = JSON.parse(line);
|
|
86
|
+
if (parsed.end === true) {
|
|
87
|
+
if (parsed.items !== items)
|
|
88
|
+
throw new TypeError(`${declared.id}: the command counted ${String(parsed.items)} entries, ${String(items)} arrived`);
|
|
89
|
+
ended = true;
|
|
90
|
+
continue;
|
|
91
|
+
}
|
|
92
|
+
yield decodeEntry(line);
|
|
93
|
+
items += 1;
|
|
94
|
+
}
|
|
95
|
+
if (!ended)
|
|
96
|
+
throw new TypeError(`${declared.id}: the export ended without its end line`);
|
|
97
|
+
},
|
|
98
|
+
identityOf: async (set, key) => JSON.parse(await lastLine(run(["identity", set, key]))).identity,
|
|
99
|
+
import: async (entries, options) => JSON.parse(await lastLine(run(["import", JSON.stringify(options)], linesOf(entries)))),
|
|
100
|
+
};
|
|
101
|
+
}
|
package/dist/store.d.ts
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import type { DataEntry, Selection } from "./manifest.js";
|
|
2
|
+
import type { DataPurpose, EntryForm, StoreKind, StoreLevel } from "./words.js";
|
|
3
|
+
/** One thing a store cannot do, said with its reason, never left silent. */
|
|
4
|
+
export interface Inability {
|
|
5
|
+
/** What it cannot do, e.g. `import of set audit-log`. */
|
|
6
|
+
readonly what: string;
|
|
7
|
+
/** Why. */
|
|
8
|
+
readonly reason: string;
|
|
9
|
+
}
|
|
10
|
+
/** One set a store holds. */
|
|
11
|
+
export interface StoreSet {
|
|
12
|
+
/** The set name (kebab-case). */
|
|
13
|
+
readonly set: string;
|
|
14
|
+
/** `<format>/<set>@<major>` of a record set, `blob@1` of a blob set. */
|
|
15
|
+
readonly schema: string;
|
|
16
|
+
/** The form of its entries. */
|
|
17
|
+
readonly form: EntryForm;
|
|
18
|
+
}
|
|
19
|
+
/** What a store holds and can do. */
|
|
20
|
+
export interface StoreDescription {
|
|
21
|
+
/** Every set, sorted by name. */
|
|
22
|
+
readonly sets: readonly StoreSet[];
|
|
23
|
+
/** The purposes it serves. */
|
|
24
|
+
readonly purposes: readonly DataPurpose[];
|
|
25
|
+
/** Whether `export` reads anything. */
|
|
26
|
+
readonly canExport: boolean;
|
|
27
|
+
/** Whether `import` writes anything. */
|
|
28
|
+
readonly canImport: boolean;
|
|
29
|
+
/** What it cannot do, each with its reason. */
|
|
30
|
+
readonly cannot: readonly Inability[];
|
|
31
|
+
}
|
|
32
|
+
/** How an import treats what the entries do not name. */
|
|
33
|
+
export interface ImportOptions {
|
|
34
|
+
/** Count what would change and write nothing. */
|
|
35
|
+
readonly dryRun: boolean;
|
|
36
|
+
/**
|
|
37
|
+
* `merge`: create and update by natural key, remove nothing. `mirror`: also remove the keys of the selected sets the
|
|
38
|
+
* entries lack. A mirror is destructive: its caller refuses it without a verified copy of the target younger than the
|
|
39
|
+
* declared maximum age (the reset rule of PROJECT-STACKS-6).
|
|
40
|
+
*/
|
|
41
|
+
readonly mode: "merge" | "mirror";
|
|
42
|
+
/** The narrowing the import writes into, e.g. the workspace a seed is applied to. */
|
|
43
|
+
readonly scope?: Readonly<Record<string, string>>;
|
|
44
|
+
}
|
|
45
|
+
/** A key a door refused, with its code; never a value. */
|
|
46
|
+
export interface RefusedKey {
|
|
47
|
+
/** The natural key. */
|
|
48
|
+
readonly key: string;
|
|
49
|
+
/** The refusal's code. */
|
|
50
|
+
readonly code: string;
|
|
51
|
+
}
|
|
52
|
+
/** What an import did to one set. */
|
|
53
|
+
export interface SetImport {
|
|
54
|
+
/** The set. */
|
|
55
|
+
readonly set: string;
|
|
56
|
+
/** Keys created. */
|
|
57
|
+
readonly created: number;
|
|
58
|
+
/** Keys whose content changed. */
|
|
59
|
+
readonly updated: number;
|
|
60
|
+
/** Keys already equal. */
|
|
61
|
+
readonly unchanged: number;
|
|
62
|
+
/** Keys removed by a mirror. */
|
|
63
|
+
readonly removed: number;
|
|
64
|
+
/** Keys a door refused. */
|
|
65
|
+
readonly refused: readonly RefusedKey[];
|
|
66
|
+
}
|
|
67
|
+
/** What an import did. `unchanged` when nothing moved: the second run of the same import answers it. */
|
|
68
|
+
export interface ImportReport {
|
|
69
|
+
/** Per set, sorted by name. */
|
|
70
|
+
readonly sets: readonly SetImport[];
|
|
71
|
+
/** `unchanged`, `applied`, or `partial` when a door refused a key. */
|
|
72
|
+
readonly outcome: "unchanged" | "applied" | "partial";
|
|
73
|
+
}
|
|
74
|
+
/** A thing that must be kept: one per declared store of `application-stacks.yaml` `data.stores`. */
|
|
75
|
+
export interface StoreAdapter {
|
|
76
|
+
/** The store id the stack manifest declares. */
|
|
77
|
+
readonly id: string;
|
|
78
|
+
/** Its kind, from the closed list. */
|
|
79
|
+
readonly kind: StoreKind;
|
|
80
|
+
/** logical or physical. */
|
|
81
|
+
readonly level: StoreLevel;
|
|
82
|
+
/** What it holds and can do. */
|
|
83
|
+
describe(): Promise<StoreDescription>;
|
|
84
|
+
/** Every entry of the selection, in (set, key) byte order; the same content yields the same stream. */
|
|
85
|
+
export(selection: Selection): AsyncIterable<DataEntry>;
|
|
86
|
+
/**
|
|
87
|
+
* An upsert by natural key through the app's own doors that KEEPS RELATIONS: a key already held is updated in place and
|
|
88
|
+
* keeps its identity (never deleted and re-created), so every reference to it stays valid; the same entries imported
|
|
89
|
+
* twice change nothing the second time; a key the entries do not name is left alone (reported by a mirror, removed
|
|
90
|
+
* only by it); operational data the entries do not carry is never overwritten.
|
|
91
|
+
*/
|
|
92
|
+
import(entries: AsyncIterable<DataEntry>, options: ImportOptions): Promise<ImportReport>;
|
|
93
|
+
/** The store's own identity of the row or object a natural key names (a database id, an object version), or null. The contract suite proves imports keep it. */
|
|
94
|
+
identityOf(set: string, key: string): Promise<string | null>;
|
|
95
|
+
}
|
|
96
|
+
/** The outcome of an import from its per-set counts. */
|
|
97
|
+
export declare function importOutcome(sets: readonly SetImport[]): ImportReport["outcome"];
|
package/dist/store.js
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/** The outcome of an import from its per-set counts. */
|
|
2
|
+
export function importOutcome(sets) {
|
|
3
|
+
if (sets.some((set) => set.refused.length > 0))
|
|
4
|
+
return "partial";
|
|
5
|
+
return sets.some((set) => set.created + set.updated + set.removed > 0) ? "applied" : "unchanged";
|
|
6
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { DriverAdapter, OpenContext, WriteContext } from "../driver.js";
|
|
2
|
+
import type { DataEntry } from "../manifest.js";
|
|
3
|
+
import type { StoreAdapter } from "../store.js";
|
|
4
|
+
/** One case of a contract suite: a name and a body that throws on a broken promise. Run it in any test runner. */
|
|
5
|
+
export interface ContractCase {
|
|
6
|
+
/** What the case proves. */
|
|
7
|
+
readonly name: string;
|
|
8
|
+
/** Throws when the adapter breaks the contract. */
|
|
9
|
+
readonly run: () => Promise<void>;
|
|
10
|
+
}
|
|
11
|
+
/** What a store may state about itself for the suite. */
|
|
12
|
+
export interface StoreContractOptions {
|
|
13
|
+
/** Answers a changed copy of an entry that is still valid for the store; by default a record gains a field and a blob a byte. */
|
|
14
|
+
readonly change?: (entry: DataEntry) => DataEntry;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* The promises every store adapter keeps. `make` builds a fresh store holding at least one entry; `empty` a fresh store of
|
|
18
|
+
* the same kind holding nothing, the target of an import. A real adapter runs this suite against its fakes. A store that
|
|
19
|
+
* holds only typed rows (a database) says in `change` how one of its entries becomes another valid one.
|
|
20
|
+
*/
|
|
21
|
+
export declare function storeContract(make: () => Promise<StoreAdapter>, empty: () => Promise<StoreAdapter>, options?: StoreContractOptions): readonly ContractCase[];
|
|
22
|
+
/**
|
|
23
|
+
* The promises every driver adapter keeps. `make` builds a fresh driver; `store` a store with content to export; `open`
|
|
24
|
+
* the identity a sealed copy is read with (a throwaway test key in a driver's own specs, never a real one).
|
|
25
|
+
*/
|
|
26
|
+
export declare function driverContract(make: () => Promise<DriverAdapter>, store: () => Promise<StoreAdapter>, context: WriteContext, open?: OpenContext): readonly ContractCase[];
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
import { exportOnce, memorySpool, restoreFrom, writeEverywhere } from "../copies.js";
|
|
2
|
+
import { DataRefusal } from "../driver.js";
|
|
3
|
+
import { blobEntry, byteOrder, entryIntact, recordEntry, recordValue } from "../manifest.js";
|
|
4
|
+
function check(condition, message) {
|
|
5
|
+
if (!condition)
|
|
6
|
+
throw new Error(`contract broken: ${message}`);
|
|
7
|
+
}
|
|
8
|
+
async function collect(items) {
|
|
9
|
+
const out = [];
|
|
10
|
+
for await (const item of items)
|
|
11
|
+
out.push(item);
|
|
12
|
+
return out;
|
|
13
|
+
}
|
|
14
|
+
async function* from(items) {
|
|
15
|
+
await Promise.resolve();
|
|
16
|
+
yield* items;
|
|
17
|
+
}
|
|
18
|
+
const header = { format: "contract-seed-backup@1", purpose: "seed-backup", takenAt: "2026-01-01T00:00:00.000Z", source: { project: "contract", environment: "test" } };
|
|
19
|
+
const sameBytes = (left, right) => left.sha256 === right.sha256 && left.key === right.key && left.set === right.set;
|
|
20
|
+
/**
|
|
21
|
+
* The promises every store adapter keeps. `make` builds a fresh store holding at least one entry; `empty` a fresh store of
|
|
22
|
+
* the same kind holding nothing, the target of an import. A real adapter runs this suite against its fakes. A store that
|
|
23
|
+
* holds only typed rows (a database) says in `change` how one of its entries becomes another valid one.
|
|
24
|
+
*/
|
|
25
|
+
export function storeContract(make, empty, options = {}) {
|
|
26
|
+
return [
|
|
27
|
+
{
|
|
28
|
+
name: "describes its sets, each with a schema and a form, and says what it cannot do",
|
|
29
|
+
run: async () => {
|
|
30
|
+
const description = await (await make()).describe();
|
|
31
|
+
check(description.sets.length > 0, "a store with content describes at least one set");
|
|
32
|
+
check(description.sets.every((set) => set.schema !== ""), "every set has a schema");
|
|
33
|
+
check(Array.isArray(description.cannot), "cannot is a list, even when empty");
|
|
34
|
+
},
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
name: "exports in (set, key) byte order, every entry intact, each key once",
|
|
38
|
+
run: async () => {
|
|
39
|
+
const entries = await collect((await make()).export({}));
|
|
40
|
+
check(entries.length > 0, "the export holds entries");
|
|
41
|
+
check(entries.every(entryIntact), "every sha256 matches its body");
|
|
42
|
+
entries.reduce((before, after) => {
|
|
43
|
+
check((byteOrder(before.set, after.set) || byteOrder(before.key, after.key)) < 0, `${after.set}/${after.key} is out of order or repeated`);
|
|
44
|
+
return after;
|
|
45
|
+
});
|
|
46
|
+
},
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
name: "exports the same content twice as the same bytes",
|
|
50
|
+
run: async () => {
|
|
51
|
+
const store = await make();
|
|
52
|
+
const first = await collect(store.export({}));
|
|
53
|
+
const second = await collect(store.export({}));
|
|
54
|
+
check(first.length === second.length && first.every((entry, at) => second[at] !== undefined && sameBytes(entry, second[at])), "two exports differ");
|
|
55
|
+
},
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
name: "exports only the selected sets",
|
|
59
|
+
run: async () => {
|
|
60
|
+
const store = await make();
|
|
61
|
+
const [first] = (await store.describe()).sets;
|
|
62
|
+
const entries = await collect(store.export({ sets: [first?.set ?? ""] }));
|
|
63
|
+
check(entries.every((entry) => entry.set === first?.set), "an entry of an unselected set was exported");
|
|
64
|
+
},
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
name: "imports idempotently by natural key: the second import changes nothing",
|
|
68
|
+
run: async () => {
|
|
69
|
+
const entries = await collect((await make()).export({}));
|
|
70
|
+
const target = await empty();
|
|
71
|
+
const first = await target.import(from(entries), { dryRun: false, mode: "merge" });
|
|
72
|
+
check(first.outcome === "applied", `the first import answered ${first.outcome}`);
|
|
73
|
+
const second = await target.import(from(entries), { dryRun: false, mode: "merge" });
|
|
74
|
+
check(second.outcome === "unchanged", `the second import answered ${second.outcome}`);
|
|
75
|
+
},
|
|
76
|
+
},
|
|
77
|
+
{
|
|
78
|
+
name: "a re-import keeps every identity: the same entries keep the same ids (relations stay valid)",
|
|
79
|
+
run: async () => {
|
|
80
|
+
const entries = await collect((await make()).export({}));
|
|
81
|
+
const target = await empty();
|
|
82
|
+
await target.import(from(entries), { dryRun: false, mode: "merge" });
|
|
83
|
+
const before = await Promise.all(entries.map((entry) => target.identityOf(entry.set, entry.key)));
|
|
84
|
+
check(before.every((id) => id !== null), "an imported key has no identity");
|
|
85
|
+
await target.import(from(entries), { dryRun: false, mode: "merge" });
|
|
86
|
+
const after = await Promise.all(entries.map((entry) => target.identityOf(entry.set, entry.key)));
|
|
87
|
+
check(after.every((id, at) => id === before[at]), "a re-import changed an identity");
|
|
88
|
+
},
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
name: "a changed entry is updated in place: same identity, new content",
|
|
92
|
+
run: async () => {
|
|
93
|
+
const entries = await collect((await make()).export({}));
|
|
94
|
+
const target = await empty();
|
|
95
|
+
await target.import(from(entries), { dryRun: false, mode: "merge" });
|
|
96
|
+
const first = entries.find((entry) => entry.form === "record") ?? entries[0];
|
|
97
|
+
if (first === undefined)
|
|
98
|
+
throw new Error("contract broken: no entry to change");
|
|
99
|
+
const id = await target.identityOf(first.set, first.key);
|
|
100
|
+
const changed = options.change !== undefined
|
|
101
|
+
? options.change(first)
|
|
102
|
+
: first.form === "record"
|
|
103
|
+
? recordEntry(first.set, first.key, { ...recordValue(first), contractChanged: true })
|
|
104
|
+
: blobEntry(first.set, first.key, new Uint8Array([...first.body, 1]), first.media);
|
|
105
|
+
const report = await target.import(from([changed]), { dryRun: false, mode: "merge" });
|
|
106
|
+
check(report.sets.some((set) => set.updated === 1), "the changed entry was not counted as an update");
|
|
107
|
+
check((await target.identityOf(first.set, first.key)) === id, "the update re-created the row");
|
|
108
|
+
const back = await collect(target.export({ sets: [first.set] }));
|
|
109
|
+
check(back.some((entry) => entry.key === first.key && entry.sha256 === changed.sha256), "the new content was not written");
|
|
110
|
+
},
|
|
111
|
+
},
|
|
112
|
+
{
|
|
113
|
+
name: "a merge leaves alone every key the entries do not name",
|
|
114
|
+
run: async () => {
|
|
115
|
+
const entries = await collect((await make()).export({}));
|
|
116
|
+
const target = await empty();
|
|
117
|
+
await target.import(from(entries), { dryRun: false, mode: "merge" });
|
|
118
|
+
const sets = new Map();
|
|
119
|
+
for (const entry of entries)
|
|
120
|
+
sets.set(entry.set, [...(sets.get(entry.set) ?? []), entry]);
|
|
121
|
+
const several = [...sets.values()].find((items) => items.length > 1) ?? entries;
|
|
122
|
+
const report = await target.import(from(several.slice(0, 1)), { dryRun: false, mode: "merge" });
|
|
123
|
+
check(report.sets.every((set) => set.removed === 0), "a merge removed a key");
|
|
124
|
+
const back = await collect(target.export({}));
|
|
125
|
+
check(back.length === entries.length, "a key absent from the import disappeared");
|
|
126
|
+
},
|
|
127
|
+
},
|
|
128
|
+
{
|
|
129
|
+
name: "a dry-run import writes nothing",
|
|
130
|
+
run: async () => {
|
|
131
|
+
const entries = await collect((await make()).export({}));
|
|
132
|
+
const target = await empty();
|
|
133
|
+
await target.import(from(entries), { dryRun: true, mode: "merge" });
|
|
134
|
+
const after = await collect(target.export({}));
|
|
135
|
+
check(after.length === 0, "a dry run wrote entries");
|
|
136
|
+
},
|
|
137
|
+
},
|
|
138
|
+
{
|
|
139
|
+
name: "export then import into an empty store round-trips every entry",
|
|
140
|
+
run: async () => {
|
|
141
|
+
const entries = await collect((await make()).export({}));
|
|
142
|
+
const target = await empty();
|
|
143
|
+
await target.import(from(entries), { dryRun: false, mode: "merge" });
|
|
144
|
+
const back = await collect(target.export({}));
|
|
145
|
+
check(back.length === entries.length && back.every((entry, at) => entries[at] !== undefined && sameBytes(entry, entries[at])), "the round trip changed the content");
|
|
146
|
+
},
|
|
147
|
+
},
|
|
148
|
+
];
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* The promises every driver adapter keeps. `make` builds a fresh driver; `store` a store with content to export; `open`
|
|
152
|
+
* the identity a sealed copy is read with (a throwaway test key in a driver's own specs, never a real one).
|
|
153
|
+
*/
|
|
154
|
+
export function driverContract(make, store, context, open = { identity: { kind: "age-identity", env: {} } }) {
|
|
155
|
+
const exported = async () => exportOnce([await store()], {}, memorySpool(), header);
|
|
156
|
+
return [
|
|
157
|
+
{
|
|
158
|
+
name: "plans a holding of the export, naming every omitted set",
|
|
159
|
+
run: async () => {
|
|
160
|
+
const source = await exported();
|
|
161
|
+
const holding = (await make()).plan(source.manifest);
|
|
162
|
+
check(holding.exportDigest === source.manifest.exportDigest, "the holding names another export");
|
|
163
|
+
check((holding.holds === "all") === (holding.omitted.length === 0), "holds and omitted disagree");
|
|
164
|
+
},
|
|
165
|
+
},
|
|
166
|
+
{
|
|
167
|
+
name: "writes, lists, reads back the same entries and verifies healthy",
|
|
168
|
+
run: async () => {
|
|
169
|
+
const driver = await make();
|
|
170
|
+
const source = await exported();
|
|
171
|
+
const receipt = await driver.write(source, context);
|
|
172
|
+
const [ref] = await driver.list();
|
|
173
|
+
if (ref === undefined)
|
|
174
|
+
throw new Error("contract broken: no copy listed after a write");
|
|
175
|
+
check(ref.exportDigest === source.manifest.exportDigest, "the newest copy is not the export written");
|
|
176
|
+
check(receipt.ref.locator === ref.locator, "the receipt names another copy");
|
|
177
|
+
const back = await collect((await driver.read(ref, open)).entries());
|
|
178
|
+
const sent = await collect(source.entries());
|
|
179
|
+
check(back.length === sent.length && back.every((item, at) => sameBytes(item.entry, sent[at].entry)), "the copy reads back other entries");
|
|
180
|
+
const verdict = await driver.verify(ref, open);
|
|
181
|
+
check(verdict.healthy, `verify: ${verdict.checks.filter((item) => !item.ok).map((item) => item.detail).join("; ")}`);
|
|
182
|
+
},
|
|
183
|
+
},
|
|
184
|
+
{
|
|
185
|
+
name: "writing an export it already holds changes nothing",
|
|
186
|
+
run: async () => {
|
|
187
|
+
const driver = await make();
|
|
188
|
+
const source = await exported();
|
|
189
|
+
await driver.write(source, context);
|
|
190
|
+
const again = await driver.write(source, context);
|
|
191
|
+
check(again.changed === 0, `the second write changed ${String(again.changed)} files`);
|
|
192
|
+
},
|
|
193
|
+
},
|
|
194
|
+
{
|
|
195
|
+
name: "a failing driver is one failed leg; the others still hold the export",
|
|
196
|
+
run: async () => {
|
|
197
|
+
const driver = await make();
|
|
198
|
+
const broken = { ...driver, id: "broken", write: () => Promise.reject(new DataRefusal("DATA_DRIVER_UNAVAILABLE", "transient")) };
|
|
199
|
+
const run = await writeEverywhere(await exported(), [driver, broken], context);
|
|
200
|
+
check(run.outcome === "partial", `the run answered ${run.outcome}`);
|
|
201
|
+
check(run.legs[1]?.failure?.code === "DATA_DRIVER_UNAVAILABLE", "the failed leg lost its code");
|
|
202
|
+
},
|
|
203
|
+
},
|
|
204
|
+
{
|
|
205
|
+
name: "a copy restores into an empty store",
|
|
206
|
+
run: async () => {
|
|
207
|
+
const driver = await make();
|
|
208
|
+
const source = await exported();
|
|
209
|
+
await driver.write(source, context);
|
|
210
|
+
const [ref] = await driver.list();
|
|
211
|
+
if (ref === undefined)
|
|
212
|
+
throw new Error("contract broken: no copy listed");
|
|
213
|
+
const target = await store();
|
|
214
|
+
const reports = await restoreFrom(await driver.read(ref, open), [target], { dryRun: false, mode: "merge" });
|
|
215
|
+
check(reports.every((report) => report.outcome !== "partial"), "the restore was refused in part");
|
|
216
|
+
},
|
|
217
|
+
},
|
|
218
|
+
];
|
|
219
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export { memoryDriver, memoryStore } from "./memory.js";
|
|
2
|
+
export type { MemoryDriver, MemoryDriverInit, MemorySets, MemoryStore, MemoryStoreInit } from "./memory.js";
|
|
3
|
+
export { driverContract, storeContract } from "./contract.js";
|
|
4
|
+
export type { ContractCase, StoreContractOptions } from "./contract.js";
|
|
5
|
+
export { memorySpool } from "../copies.js";
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import type { DriverAdapter, DriverLimits } from "../driver.js";
|
|
2
|
+
import type { JsonValue } from "../manifest.js";
|
|
3
|
+
import type { Inability, StoreAdapter } from "../store.js";
|
|
4
|
+
import type { DataRefusalCode } from "../driver.js";
|
|
5
|
+
import type { DriverKind, StoreKind, StoreLevel } from "../words.js";
|
|
6
|
+
/** The content of a memory store: set -> key -> a record value or blob bytes. */
|
|
7
|
+
export type MemorySets = Record<string, Record<string, JsonValue | Uint8Array>>;
|
|
8
|
+
/** What a memory store starts with. */
|
|
9
|
+
export interface MemoryStoreInit {
|
|
10
|
+
/** The store id. */
|
|
11
|
+
readonly id: string;
|
|
12
|
+
/** The kind; `database` by default. */
|
|
13
|
+
readonly kind?: StoreKind;
|
|
14
|
+
/** The level; `logical` by default. */
|
|
15
|
+
readonly level?: StoreLevel;
|
|
16
|
+
/** The content. */
|
|
17
|
+
readonly sets?: MemorySets;
|
|
18
|
+
/** The schema format of its record sets, e.g. `shop-seed-standard@1`. */
|
|
19
|
+
readonly format?: string;
|
|
20
|
+
/** What it says it cannot do. */
|
|
21
|
+
readonly cannot?: readonly Inability[];
|
|
22
|
+
/** Keys whose import the door refuses, with the refusal code. */
|
|
23
|
+
readonly refuse?: Readonly<Record<string, string>>;
|
|
24
|
+
}
|
|
25
|
+
/** A memory store, with a view of what it holds now. */
|
|
26
|
+
export interface MemoryStore extends StoreAdapter {
|
|
27
|
+
/** A copy of the content, for assertions. */
|
|
28
|
+
snapshot(): MemorySets;
|
|
29
|
+
}
|
|
30
|
+
/** A store over plain maps: deterministic export in (set, key) byte order; merge and mirror imports with exact counts. */
|
|
31
|
+
export declare function memoryStore(init: MemoryStoreInit): MemoryStore;
|
|
32
|
+
/** What a memory driver starts with. */
|
|
33
|
+
export interface MemoryDriverInit {
|
|
34
|
+
/** The driver id. */
|
|
35
|
+
readonly id: string;
|
|
36
|
+
/** The kind; `git` by default. */
|
|
37
|
+
readonly kind?: DriverKind;
|
|
38
|
+
/** Limits; a git driver's by default (records and blobs, the 500 MiB whole-copy cap measured on entry bytes). */
|
|
39
|
+
readonly limits?: Partial<DriverLimits>;
|
|
40
|
+
/** Fails one operation with this code, to test partial runs and chooseCopy. */
|
|
41
|
+
readonly fail?: {
|
|
42
|
+
readonly on: "write" | "read" | "verify";
|
|
43
|
+
readonly code: DataRefusalCode;
|
|
44
|
+
readonly retry?: "configuration" | "transient";
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
/** A memory driver, with a hook to corrupt a kept copy. */
|
|
48
|
+
export interface MemoryDriver extends DriverAdapter {
|
|
49
|
+
/** Changes one entry of the newest copy without changing its manifest, as a corrupted shard would. */
|
|
50
|
+
corrupt(set: string, key: string): void;
|
|
51
|
+
}
|
|
52
|
+
/** A driver over a list of kept copies, newest first, with real limits applied. */
|
|
53
|
+
export declare function memoryDriver(init: MemoryDriverInit): MemoryDriver;
|
|
Binary file
|
package/dist/words.d.ts
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The closed vocabulary of the data standard (PROJECT-STACKS-14). A new store kind or driver is a harness change: this
|
|
3
|
+
* file, the rule that lists it (R426, R427) and the stack manifest's parser move together, never an app's choice.
|
|
4
|
+
*/
|
|
5
|
+
/** What must be kept: the relational database, or the object store (its buckets and their binaries). */
|
|
6
|
+
export type StoreKind = "database" | "storage";
|
|
7
|
+
/** Where a copy is kept: a sealed git repository now, an S3 bucket when the project outgrows git. */
|
|
8
|
+
export type DriverKind = "git" | "s3";
|
|
9
|
+
/**
|
|
10
|
+
* The two kinds of seed, which differ only in who writes them and when they are read:
|
|
11
|
+
* `seed-standard`: curated by people, pinned by commit, loaded by default when an environment or workspace is created;
|
|
12
|
+
* `seed-backup`: written only by the scheduled backup job, read back only by an explicit `sync`, never loaded by default.
|
|
13
|
+
*/
|
|
14
|
+
export type DataPurpose = "seed-standard" | "seed-backup";
|
|
15
|
+
/**
|
|
16
|
+
* `logical`: records by natural key, read and written through the app's own doors (portable, idempotent, partial).
|
|
17
|
+
* `physical`: the engine's own dump; restored whole only, and only from a `seed-backup`.
|
|
18
|
+
*/
|
|
19
|
+
export type StoreLevel = "logical" | "physical";
|
|
20
|
+
/** `record`: one canonical JSON object. `blob`: opaque bytes (a stored file, a part of a dump). */
|
|
21
|
+
export type EntryForm = "record" | "blob";
|
|
22
|
+
/** The store kinds, in the order the manifest lists them. */
|
|
23
|
+
export declare const STORE_KINDS: readonly StoreKind[];
|
|
24
|
+
/** The drivers, in the order the manifest lists them. */
|
|
25
|
+
export declare const DRIVER_KINDS: readonly DriverKind[];
|
|
26
|
+
/** The purposes. */
|
|
27
|
+
export declare const DATA_PURPOSES: readonly DataPurpose[];
|
|
28
|
+
/** The whole snapshot one git copy may hold (its data files in the working tree, history not counted): 500 MiB. Past it the copy needs s3. */
|
|
29
|
+
export declare const GIT_CAP_BYTES = 524288000;
|
|
30
|
+
/** The size at which a git copy warns that it nears its cap: 80% of it, 400 MiB. */
|
|
31
|
+
export declare const GIT_WARN_BYTES = 419430400;
|
|
32
|
+
/** The largest shard or blob chunk file the git driver writes: 4 MiB, far under GitHub's 50 MiB push warning and 100 MiB per-file block. */
|
|
33
|
+
export declare const GIT_SHARD_BYTES = 4194304;
|
package/dist/words.js
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The closed vocabulary of the data standard (PROJECT-STACKS-14). A new store kind or driver is a harness change: this
|
|
3
|
+
* file, the rule that lists it (R426, R427) and the stack manifest's parser move together, never an app's choice.
|
|
4
|
+
*/
|
|
5
|
+
/** The store kinds, in the order the manifest lists them. */
|
|
6
|
+
export const STORE_KINDS = Object.freeze(["database", "storage"]);
|
|
7
|
+
/** The drivers, in the order the manifest lists them. */
|
|
8
|
+
export const DRIVER_KINDS = Object.freeze(["git", "s3"]);
|
|
9
|
+
/** The purposes. */
|
|
10
|
+
export const DATA_PURPOSES = Object.freeze(["seed-standard", "seed-backup"]);
|
|
11
|
+
/** The whole snapshot one git copy may hold (its data files in the working tree, history not counted): 500 MiB. Past it the copy needs s3. */
|
|
12
|
+
export const GIT_CAP_BYTES = 524_288_000;
|
|
13
|
+
/** The size at which a git copy warns that it nears its cap: 80% of it, 400 MiB. */
|
|
14
|
+
export const GIT_WARN_BYTES = 419_430_400;
|
|
15
|
+
/** The largest shard or blob chunk file the git driver writes: 4 MiB, far under GitHub's 50 MiB push warning and 100 MiB per-file block. */
|
|
16
|
+
export const GIT_SHARD_BYTES = 4_194_304;
|