@starci/seed 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/LICENSE +21 -0
- package/README.md +51 -2
- package/dist/copies.d.ts +83 -0
- package/dist/copies.js +146 -0
- package/dist/driver.d.ts +145 -0
- package/dist/driver.js +21 -0
- package/dist/git/driver.d.ts +53 -0
- package/dist/git/driver.js +309 -0
- package/dist/git/files.d.ts +48 -0
- package/dist/git/files.js +78 -0
- package/dist/git/index.d.ts +4 -0
- package/dist/git/index.js +2 -0
- package/dist/index.d.ts +12 -0
- package/dist/index.js +6 -0
- package/dist/manifest.d.ts +108 -0
- package/dist/manifest.js +131 -0
- package/dist/postgres/index.d.ts +2 -0
- package/dist/postgres/index.js +1 -0
- package/dist/postgres/pipe.d.ts +11 -0
- package/dist/postgres/pipe.js +60 -0
- package/dist/postgres/sql.d.ts +42 -0
- package/dist/postgres/sql.js +54 -0
- package/dist/postgres/store.d.ts +33 -0
- package/dist/postgres/store.js +231 -0
- package/dist/protocol.d.ts +52 -0
- package/dist/protocol.js +101 -0
- package/dist/store.d.ts +97 -0
- package/dist/store.js +6 -0
- package/dist/testing/contract.d.ts +26 -0
- package/dist/testing/contract.js +219 -0
- package/dist/testing/index.d.ts +5 -0
- package/dist/testing/index.js +3 -0
- package/dist/testing/memory.d.ts +53 -0
- package/dist/testing/memory.js +0 -0
- package/dist/words.d.ts +33 -0
- package/dist/words.js +16 -0
- package/package.json +85 -4
package/dist/manifest.js
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
const encoder = new TextEncoder();
|
|
3
|
+
const decoder = new TextDecoder("utf-8", { fatal: true });
|
|
4
|
+
/** Lower-case hex sha256 of bytes or of a UTF-8 string. */
|
|
5
|
+
export function sha256Hex(content) {
|
|
6
|
+
return createHash("sha256").update(typeof content === "string" ? encoder.encode(content) : content).digest("hex");
|
|
7
|
+
}
|
|
8
|
+
/** Compares two strings by their UTF-8 bytes, the one order every set and key list uses. */
|
|
9
|
+
export function byteOrder(left, right) {
|
|
10
|
+
return Buffer.compare(Buffer.from(left, "utf8"), Buffer.from(right, "utf8"));
|
|
11
|
+
}
|
|
12
|
+
function canonical(value) {
|
|
13
|
+
if (value === null || typeof value === "boolean" || typeof value === "string")
|
|
14
|
+
return JSON.stringify(value);
|
|
15
|
+
if (typeof value === "number") {
|
|
16
|
+
if (!Number.isFinite(value))
|
|
17
|
+
throw new TypeError("a record holds no NaN or Infinity");
|
|
18
|
+
return JSON.stringify(value);
|
|
19
|
+
}
|
|
20
|
+
if (Array.isArray(value))
|
|
21
|
+
return `[${value.map(canonical).join(",")}]`;
|
|
22
|
+
if (typeof value === "object") {
|
|
23
|
+
const keys = Object.keys(value).sort(byteOrder);
|
|
24
|
+
const fields = keys.flatMap((key) => {
|
|
25
|
+
const item = value[key];
|
|
26
|
+
return item === undefined ? [] : [`${JSON.stringify(key)}:${canonical(item)}`];
|
|
27
|
+
});
|
|
28
|
+
return `{${fields.join(",")}}`;
|
|
29
|
+
}
|
|
30
|
+
throw new TypeError(`a record holds no ${typeof value}`);
|
|
31
|
+
}
|
|
32
|
+
/** The canonical JSON text of a value: keys sorted by byte order, no insignificant whitespace, undefined fields dropped. */
|
|
33
|
+
export function canonicalJson(value) {
|
|
34
|
+
return canonical(value);
|
|
35
|
+
}
|
|
36
|
+
/** A record entry of `value`, its body the canonical JSON. */
|
|
37
|
+
export function recordEntry(set, key, value) {
|
|
38
|
+
const body = encoder.encode(canonicalJson(value));
|
|
39
|
+
return { set, key, form: "record", media: "application/json", body, sha256: sha256Hex(body) };
|
|
40
|
+
}
|
|
41
|
+
/** A blob entry of `body`. */
|
|
42
|
+
export function blobEntry(set, key, body, media) {
|
|
43
|
+
return { set, key, form: "blob", media, body, sha256: sha256Hex(body) };
|
|
44
|
+
}
|
|
45
|
+
/** The value of a record entry. */
|
|
46
|
+
export function recordValue(entry) {
|
|
47
|
+
if (entry.form !== "record")
|
|
48
|
+
throw new TypeError(`${entry.set}/${entry.key} is a blob, not a record`);
|
|
49
|
+
return JSON.parse(decoder.decode(entry.body));
|
|
50
|
+
}
|
|
51
|
+
/** True when the entry's sha256 is the hash of its body. */
|
|
52
|
+
export function entryIntact(entry) {
|
|
53
|
+
return sha256Hex(entry.body) === entry.sha256;
|
|
54
|
+
}
|
|
55
|
+
/** The digest of a set: sha256 over the sorted lines `<key>\t<sha256>\n`. */
|
|
56
|
+
export function setDigest(items) {
|
|
57
|
+
const lines = [...items].sort((left, right) => byteOrder(left.key, right.key)).map((item) => `${item.key}\t${item.sha256}\n`);
|
|
58
|
+
return sha256Hex(lines.join(""));
|
|
59
|
+
}
|
|
60
|
+
/** The identity of an export: sha256 of the canonical JSON of its sets. */
|
|
61
|
+
export function exportDigestOf(sets) {
|
|
62
|
+
return sha256Hex(canonicalJson(sets));
|
|
63
|
+
}
|
|
64
|
+
/** The schema id of a set: `<format without @major>/<set>@<major>` for records, `blob@1` for blobs. */
|
|
65
|
+
export function setSchemaOf(format, set, form) {
|
|
66
|
+
if (form === "blob")
|
|
67
|
+
return "blob@1";
|
|
68
|
+
const at = format.lastIndexOf("@");
|
|
69
|
+
return at < 0 ? `${format}/${set}@1` : `${format.slice(0, at)}/${set}${format.slice(at)}`;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Builds summaries from entries as they pass, refusing an entry whose hash does not match its body, a key repeated in
|
|
73
|
+
* its set, or a set whose entries mix forms.
|
|
74
|
+
*/
|
|
75
|
+
export class ManifestBuilder {
|
|
76
|
+
tallies = new Map();
|
|
77
|
+
/** Counts one entry of `store`. */
|
|
78
|
+
add(store, entry) {
|
|
79
|
+
if (!entryIntact(entry))
|
|
80
|
+
throw new TypeError(`${store}/${entry.set}/${entry.key}: sha256 does not match the body`);
|
|
81
|
+
const id = `${store}\u0000${entry.set}`;
|
|
82
|
+
const tally = this.tallies.get(id) ?? { store, set: entry.set, form: entry.form, items: [], bytes: 0 };
|
|
83
|
+
if (tally.form !== entry.form)
|
|
84
|
+
throw new TypeError(`${store}/${entry.set} mixes records and blobs`);
|
|
85
|
+
if (tally.items.some((item) => item.key === entry.key))
|
|
86
|
+
throw new TypeError(`${store}/${entry.set}/${entry.key} is repeated`);
|
|
87
|
+
tally.items.push({ key: entry.key, sha256: entry.sha256 });
|
|
88
|
+
tally.bytes += entry.body.byteLength;
|
|
89
|
+
this.tallies.set(id, tally);
|
|
90
|
+
}
|
|
91
|
+
/** The finished manifest. */
|
|
92
|
+
build(header) {
|
|
93
|
+
const sets = [...this.tallies.values()]
|
|
94
|
+
.map((tally) => ({
|
|
95
|
+
store: tally.store,
|
|
96
|
+
set: tally.set,
|
|
97
|
+
schema: setSchemaOf(header.format, tally.set, tally.form),
|
|
98
|
+
form: tally.form,
|
|
99
|
+
items: tally.items.length,
|
|
100
|
+
bytes: tally.bytes,
|
|
101
|
+
digest: setDigest(tally.items),
|
|
102
|
+
}))
|
|
103
|
+
.sort((left, right) => byteOrder(left.store, right.store) || byteOrder(left.set, right.set));
|
|
104
|
+
return { schema: "starci/data-manifest@1", ...header, sets, exportDigest: exportDigestOf(sets) };
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
/** The reasons a manifest document is not a valid `starci/data-manifest@1`, empty when it is. */
|
|
108
|
+
export function manifestProblems(document) {
|
|
109
|
+
const problems = [];
|
|
110
|
+
if (typeof document !== "object" || document === null || Array.isArray(document))
|
|
111
|
+
return ["the manifest is not an object"];
|
|
112
|
+
const value = document;
|
|
113
|
+
if (value.schema !== "starci/data-manifest@1")
|
|
114
|
+
problems.push("schema is not starci/data-manifest@1");
|
|
115
|
+
if (typeof value.format !== "string" || !/^[a-z0-9][a-z0-9-]*@\d+$/u.test(value.format))
|
|
116
|
+
problems.push("format is not <name>@<major>");
|
|
117
|
+
if (!["seed-standard", "seed-backup"].includes(String(value.purpose)))
|
|
118
|
+
problems.push("purpose is not seed-standard or seed-backup");
|
|
119
|
+
if (typeof value.takenAt !== "string")
|
|
120
|
+
problems.push("takenAt is missing");
|
|
121
|
+
if (!Array.isArray(value.sets))
|
|
122
|
+
return [...problems, "sets is not a list"];
|
|
123
|
+
const sets = value.sets;
|
|
124
|
+
for (const set of sets) {
|
|
125
|
+
if (typeof set?.store !== "string" || typeof set.set !== "string" || typeof set.digest !== "string" || !Number.isInteger(set.items))
|
|
126
|
+
problems.push(`a set entry is incomplete: ${JSON.stringify(set)}`);
|
|
127
|
+
}
|
|
128
|
+
if (problems.length === 0 && value.exportDigest !== exportDigestOf(sets))
|
|
129
|
+
problems.push("exportDigest does not match the sets");
|
|
130
|
+
return problems;
|
|
131
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { postgresStore } from "./store.js";
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/** Thrown inside a producer when its consumer stopped reading: the producer's transaction rolls back and the stream ends quietly. */
|
|
2
|
+
export declare class Cancelled extends Error {
|
|
3
|
+
constructor();
|
|
4
|
+
}
|
|
5
|
+
/**
|
|
6
|
+
* Turns a producer that must run inside a callback (a transaction) into an async iterable with back pressure: `send` hands
|
|
7
|
+
* over one item and resumes when the reader has taken it, so memory holds at most the item being read and the next one.
|
|
8
|
+
* A reader that stops early cancels the producer (its next `send` throws `Cancelled`, which a transaction rolls back);
|
|
9
|
+
* a producer that fails ends the stream with its error.
|
|
10
|
+
*/
|
|
11
|
+
export declare function streamed<T>(produce: (send: (item: T) => Promise<void>) => Promise<void>): AsyncGenerator<T, void, undefined>;
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/** Thrown inside a producer when its consumer stopped reading: the producer's transaction rolls back and the stream ends quietly. */
|
|
2
|
+
export class Cancelled extends Error {
|
|
3
|
+
constructor() {
|
|
4
|
+
super("the reader stopped before the end of the stream");
|
|
5
|
+
this.name = "Cancelled";
|
|
6
|
+
}
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Turns a producer that must run inside a callback (a transaction) into an async iterable with back pressure: `send` hands
|
|
10
|
+
* over one item and resumes when the reader has taken it, so memory holds at most the item being read and the next one.
|
|
11
|
+
* A reader that stops early cancels the producer (its next `send` throws `Cancelled`, which a transaction rolls back);
|
|
12
|
+
* a producer that fails ends the stream with its error.
|
|
13
|
+
*/
|
|
14
|
+
export async function* streamed(produce) {
|
|
15
|
+
const queue = [];
|
|
16
|
+
const state = { finished: false, cancelled: false, failure: null };
|
|
17
|
+
const wake = { reader: null, producer: null };
|
|
18
|
+
const send = async (item) => {
|
|
19
|
+
queue.push(item);
|
|
20
|
+
wake.reader?.();
|
|
21
|
+
await new Promise((resolve) => {
|
|
22
|
+
wake.producer = resolve;
|
|
23
|
+
if (state.cancelled)
|
|
24
|
+
resolve();
|
|
25
|
+
});
|
|
26
|
+
if (state.cancelled)
|
|
27
|
+
throw new Cancelled();
|
|
28
|
+
};
|
|
29
|
+
const running = produce(send).then(() => {
|
|
30
|
+
state.finished = true;
|
|
31
|
+
wake.reader?.();
|
|
32
|
+
}, (cause) => {
|
|
33
|
+
if (!(cause instanceof Cancelled))
|
|
34
|
+
state.failure = { cause };
|
|
35
|
+
state.finished = true;
|
|
36
|
+
wake.reader?.();
|
|
37
|
+
});
|
|
38
|
+
try {
|
|
39
|
+
for (;;) {
|
|
40
|
+
const next = queue.shift();
|
|
41
|
+
if (next !== undefined) {
|
|
42
|
+
wake.producer?.();
|
|
43
|
+
yield next;
|
|
44
|
+
continue;
|
|
45
|
+
}
|
|
46
|
+
if (state.finished)
|
|
47
|
+
break;
|
|
48
|
+
await new Promise((resolve) => {
|
|
49
|
+
wake.reader = resolve;
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
if (state.failure !== null)
|
|
53
|
+
throw state.failure.cause;
|
|
54
|
+
}
|
|
55
|
+
finally {
|
|
56
|
+
state.cancelled = true;
|
|
57
|
+
wake.producer?.();
|
|
58
|
+
await running;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/** One column the copy holds: not generated, not dropped. */
|
|
2
|
+
export interface Column {
|
|
3
|
+
readonly name: string;
|
|
4
|
+
/** `format_type` of the column, the type its text is cast back to on import. */
|
|
5
|
+
readonly type: string;
|
|
6
|
+
}
|
|
7
|
+
/** One table the copy holds. */
|
|
8
|
+
export interface Table {
|
|
9
|
+
readonly schema: string;
|
|
10
|
+
readonly table: string;
|
|
11
|
+
/** `<schema>.<table>`, the set name. */
|
|
12
|
+
readonly set: string;
|
|
13
|
+
readonly columns: readonly Column[];
|
|
14
|
+
/** The primary key columns in key order (each is also one of `columns`). */
|
|
15
|
+
readonly pk: readonly Column[];
|
|
16
|
+
}
|
|
17
|
+
/** Quotes one identifier. */
|
|
18
|
+
export declare const quote: (name: string) => string;
|
|
19
|
+
/** The quoted, schema-qualified name of a table, also the text of a `regclass` parameter. */
|
|
20
|
+
export declare const qualified: (table: Table) => string;
|
|
21
|
+
/** Session settings that make text output the same in every session: one zone, one date style, hex bytes, shortest floats. */
|
|
22
|
+
export declare const SESSION = "/* seed:session */ SELECT set_config('TimeZone', 'UTC', true), set_config('DateStyle', 'ISO, YMD', true), set_config('IntervalStyle', 'iso_8601', true), set_config('bytea_output', 'hex', true), set_config('extra_float_digits', '1', true), set_config('lc_monetary', 'C', true)";
|
|
23
|
+
/** Foreign keys and triggers stay quiet while a copy is restored; needs the owner role. */
|
|
24
|
+
export declare const REPLICA = "/* seed:replica */ SET LOCAL session_replication_role = replica";
|
|
25
|
+
/** The ordinary and partitioned-parent tables of the schemas ($1 text[]), each with its held columns and primary key as JSON text. */
|
|
26
|
+
export declare const CATALOG = "/* seed:catalog */ SELECT n.nspname AS schema, c.relname AS \"table\",\n (SELECT coalesce(json_agg(json_build_object('name', a.attname, 'type', format_type(a.atttypid, a.atttypmod)) ORDER BY a.attnum), '[]'::json)::text\n FROM pg_attribute a WHERE a.attrelid = c.oid AND a.attnum > 0 AND NOT a.attisdropped AND a.attgenerated = '') AS columns,\n (SELECT json_agg(a.attname ORDER BY k.ord)::text\n FROM pg_index i CROSS JOIN LATERAL unnest(i.indkey::int2[]) WITH ORDINALITY AS k(attnum, ord)\n JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum = k.attnum\n WHERE i.indrelid = c.oid AND i.indisprimary) AS pk\nFROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace\nWHERE c.relkind IN ('r', 'p') AND NOT c.relispartition AND n.nspname = ANY($1::text[])";
|
|
27
|
+
/** One page of keys and rows in UTF-8 byte order of the key, after the key in $1 when given. */
|
|
28
|
+
export declare const page: (table: Table, rows: number, after: boolean) => string;
|
|
29
|
+
/** The stored row of the primary key in $1... as the same JSON object text an export writes. */
|
|
30
|
+
export declare const get: (table: Table) => string;
|
|
31
|
+
/** Whether a row of the primary key in $1... exists. */
|
|
32
|
+
export declare const exists: (table: Table) => string;
|
|
33
|
+
/** Deletes the row of the primary key in $1.... */
|
|
34
|
+
export declare const remove: (table: Table) => string;
|
|
35
|
+
/** Every key of the table with each primary key column's text (p0, p1, ...). */
|
|
36
|
+
export declare const keys: (table: Table) => string;
|
|
37
|
+
/** Upserts the row object in $1 (a JSON text of every held column as text or null) by primary key. */
|
|
38
|
+
export declare const upsert: (table: Table) => string;
|
|
39
|
+
/** The sequences owned by a column of the table named in $1 (quoted, schema-qualified), each with its quoted name and column. */
|
|
40
|
+
export declare const SEQUENCES = "/* seed:sequences */ SELECT quote_ident(sn.nspname) || '.' || quote_ident(s.relname) AS sequence, a.attname AS \"column\"\nFROM pg_depend d JOIN pg_class s ON s.oid = d.objid AND s.relkind = 'S' JOIN pg_namespace sn ON sn.oid = s.relnamespace\nJOIN pg_attribute a ON a.attrelid = d.refobjid AND a.attnum = d.refobjsubid\nWHERE d.refclassid = 'pg_class'::regclass AND d.classid = 'pg_class'::regclass AND d.deptype IN ('a', 'i') AND d.refobjid = ($1::text)::regclass\nORDER BY 1, 2";
|
|
41
|
+
/** Sets the sequence named in $1 to the column's maximum; does nothing when the table is empty. */
|
|
42
|
+
export declare const setval: (table: Table, column: string) => string;
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// Every statement the postgres store sends, built from the catalog with quoted identifiers only; a value never enters
|
|
2
|
+
// a statement's text, it travels as a parameter. Each statement starts with a seed marker comment so a log line or a
|
|
3
|
+
// test double can tell them apart without parsing SQL.
|
|
4
|
+
/** Quotes one identifier. */
|
|
5
|
+
export const quote = (name) => `"${name.replaceAll('"', '""')}"`;
|
|
6
|
+
/** The quoted, schema-qualified name of a table, also the text of a `regclass` parameter. */
|
|
7
|
+
export const qualified = (table) => `${quote(table.schema)}.${quote(table.table)}`;
|
|
8
|
+
/** The SQL text of the primary key as the entry key: the one column's text, or a JSON array text of several. */
|
|
9
|
+
const keyOf = (table) => {
|
|
10
|
+
const parts = table.pk.map((column) => `t.${quote(column.name)}::text`);
|
|
11
|
+
return table.pk.length === 1 ? parts.join("") : `array_to_json(ARRAY[${parts.join(", ")}])::text`;
|
|
12
|
+
};
|
|
13
|
+
/** The row as one JSON object text of every held column cast to text: numbers, bytes, timestamps and json keep their exact digits. */
|
|
14
|
+
const rowOf = (table) => `(SELECT row_to_json(x)::text FROM (SELECT ${table.columns.map((column) => `t.${quote(column.name)}::text AS ${quote(column.name)}`).join(", ")}) x)`;
|
|
15
|
+
const byKey = (table) => table.pk.map((column, at) => `t.${quote(column.name)} = ($${String(at + 1)}::text)::${column.type}`).join(" AND ");
|
|
16
|
+
/** Session settings that make text output the same in every session: one zone, one date style, hex bytes, shortest floats. */
|
|
17
|
+
export const SESSION = `/* seed:session */ SELECT set_config('TimeZone', 'UTC', true), set_config('DateStyle', 'ISO, YMD', true), set_config('IntervalStyle', 'iso_8601', true), set_config('bytea_output', 'hex', true), set_config('extra_float_digits', '1', true), set_config('lc_monetary', 'C', true)`;
|
|
18
|
+
/** Foreign keys and triggers stay quiet while a copy is restored; needs the owner role. */
|
|
19
|
+
export const REPLICA = `/* seed:replica */ SET LOCAL session_replication_role = replica`;
|
|
20
|
+
/** The ordinary and partitioned-parent tables of the schemas ($1 text[]), each with its held columns and primary key as JSON text. */
|
|
21
|
+
export const CATALOG = `/* seed:catalog */ SELECT n.nspname AS schema, c.relname AS "table",
|
|
22
|
+
(SELECT coalesce(json_agg(json_build_object('name', a.attname, 'type', format_type(a.atttypid, a.atttypmod)) ORDER BY a.attnum), '[]'::json)::text
|
|
23
|
+
FROM pg_attribute a WHERE a.attrelid = c.oid AND a.attnum > 0 AND NOT a.attisdropped AND a.attgenerated = '') AS columns,
|
|
24
|
+
(SELECT json_agg(a.attname ORDER BY k.ord)::text
|
|
25
|
+
FROM pg_index i CROSS JOIN LATERAL unnest(i.indkey::int2[]) WITH ORDINALITY AS k(attnum, ord)
|
|
26
|
+
JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum = k.attnum
|
|
27
|
+
WHERE i.indrelid = c.oid AND i.indisprimary) AS pk
|
|
28
|
+
FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
29
|
+
WHERE c.relkind IN ('r', 'p') AND NOT c.relispartition AND n.nspname = ANY($1::text[])`;
|
|
30
|
+
/** One page of keys and rows in UTF-8 byte order of the key, after the key in $1 when given. */
|
|
31
|
+
export const page = (table, rows, after) => `/* seed:page */ SELECT ${keyOf(table)} AS k, ${rowOf(table)} AS r FROM ${qualified(table)} t${after ? ` WHERE convert_to(${keyOf(table)}, 'UTF8') > convert_to($1::text, 'UTF8')` : ""} ORDER BY convert_to(${keyOf(table)}, 'UTF8') LIMIT ${String(rows)}`;
|
|
32
|
+
/** The stored row of the primary key in $1... as the same JSON object text an export writes. */
|
|
33
|
+
export const get = (table) => `/* seed:get */ SELECT ${rowOf(table)} AS r FROM ${qualified(table)} t WHERE ${byKey(table)}`;
|
|
34
|
+
/** Whether a row of the primary key in $1... exists. */
|
|
35
|
+
export const exists = (table) => `/* seed:exists */ SELECT 1 AS present FROM ${qualified(table)} t WHERE ${byKey(table)}`;
|
|
36
|
+
/** Deletes the row of the primary key in $1.... */
|
|
37
|
+
export const remove = (table) => `/* seed:delete */ DELETE FROM ${qualified(table)} t WHERE ${byKey(table)}`;
|
|
38
|
+
/** Every key of the table with each primary key column's text (p0, p1, ...). */
|
|
39
|
+
export const keys = (table) => `/* seed:keys */ SELECT ${keyOf(table)} AS k${table.pk.map((column, at) => `, t.${quote(column.name)}::text AS p${String(at)}`).join("")} FROM ${qualified(table)} t`;
|
|
40
|
+
/** Upserts the row object in $1 (a JSON text of every held column as text or null) by primary key. */
|
|
41
|
+
export const upsert = (table) => {
|
|
42
|
+
const names = table.columns.map((column) => quote(column.name));
|
|
43
|
+
const changed = table.columns.filter((column) => !table.pk.some((key) => key.name === column.name)).map((column) => `${quote(column.name)} = EXCLUDED.${quote(column.name)}`);
|
|
44
|
+
const casts = table.columns.map((column) => `(x.${quote(column.name)})::${column.type}`);
|
|
45
|
+
return `/* seed:upsert */ INSERT INTO ${qualified(table)} (${names.join(", ")}) OVERRIDING SYSTEM VALUE SELECT ${casts.join(", ")} FROM json_to_record($1::json) AS x(${names.map((name) => `${name} text`).join(", ")}) ON CONFLICT (${table.pk.map((column) => quote(column.name)).join(", ")}) ${changed.length === 0 ? "DO NOTHING" : `DO UPDATE SET ${changed.join(", ")}`}`;
|
|
46
|
+
};
|
|
47
|
+
/** The sequences owned by a column of the table named in $1 (quoted, schema-qualified), each with its quoted name and column. */
|
|
48
|
+
export const SEQUENCES = `/* seed:sequences */ SELECT quote_ident(sn.nspname) || '.' || quote_ident(s.relname) AS sequence, a.attname AS "column"
|
|
49
|
+
FROM pg_depend d JOIN pg_class s ON s.oid = d.objid AND s.relkind = 'S' JOIN pg_namespace sn ON sn.oid = s.relnamespace
|
|
50
|
+
JOIN pg_attribute a ON a.attrelid = d.refobjid AND a.attnum = d.refobjsubid
|
|
51
|
+
WHERE d.refclassid = 'pg_class'::regclass AND d.classid = 'pg_class'::regclass AND d.deptype IN ('a', 'i') AND d.refobjid = ($1::text)::regclass
|
|
52
|
+
ORDER BY 1, 2`;
|
|
53
|
+
/** Sets the sequence named in $1 to the column's maximum; does nothing when the table is empty. */
|
|
54
|
+
export const setval = (table, column) => `/* seed:setval */ SELECT setval(($1::text)::regclass, m) AS value FROM (SELECT max(t.${quote(column)}) AS m FROM ${qualified(table)} t) x WHERE m IS NOT NULL`;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { StoreAdapter } from "../store.js";
|
|
2
|
+
/** Runs statements against one PostgreSQL connection or pool: the only thing the app hands this package. */
|
|
3
|
+
export interface PostgresRunner {
|
|
4
|
+
/** Runs one statement with positional parameters and answers its rows as plain objects. */
|
|
5
|
+
query(sql: string, params?: readonly unknown[]): Promise<readonly Record<string, unknown>[]>;
|
|
6
|
+
/**
|
|
7
|
+
* Runs `work` in ONE transaction (REPEATABLE READ for an export: one consistent snapshot; READ COMMITTED for an import)
|
|
8
|
+
* and commits, or rolls back and rethrows the same error when `work` throws.
|
|
9
|
+
*/
|
|
10
|
+
transaction<T>(isolation: "repeatable read" | "read committed", work: (tx: PostgresRunner) => Promise<T>): Promise<T>;
|
|
11
|
+
}
|
|
12
|
+
/** A table the copy leaves out, with the reason shown in `describe().cannot`. */
|
|
13
|
+
export interface ExcludedTable {
|
|
14
|
+
/** `<schema>.<table>`. */
|
|
15
|
+
readonly table: string;
|
|
16
|
+
/** Why it is rebuildable or not worth keeping: a queue, a cache, vectors, a finished checkpoint, raw data past retention. */
|
|
17
|
+
readonly reason: string;
|
|
18
|
+
}
|
|
19
|
+
/** What `postgresStore` needs. */
|
|
20
|
+
export interface PostgresStoreOptions {
|
|
21
|
+
/** The store id the stack declares, e.g. `database`. */
|
|
22
|
+
readonly id: string;
|
|
23
|
+
/** The connection. An export needs only read access (a role that bypasses row level security to see every row); an import needs the owner role. */
|
|
24
|
+
readonly run: PostgresRunner;
|
|
25
|
+
/** The schemas to copy; `["public"]` by default. */
|
|
26
|
+
readonly schemas?: readonly string[];
|
|
27
|
+
/** Rebuildable tables, each with its reason. */
|
|
28
|
+
readonly exclude?: readonly ExcludedTable[];
|
|
29
|
+
/** Rows per keyset page; 5000 by default. */
|
|
30
|
+
readonly pageRows?: number;
|
|
31
|
+
}
|
|
32
|
+
/** The store that copies the whole relational database behind the `StoreAdapter` port (kind `database`, level `physical`). */
|
|
33
|
+
export declare function postgresStore(options: PostgresStoreOptions): StoreAdapter;
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
import { causeLabel } from "../copies.js";
|
|
2
|
+
import { DataRefusal } from "../driver.js";
|
|
3
|
+
import { byteOrder, canonicalJson, recordEntry, recordValue } from "../manifest.js";
|
|
4
|
+
import { importOutcome } from "../store.js";
|
|
5
|
+
import { Cancelled, streamed } from "./pipe.js";
|
|
6
|
+
import * as sql from "./sql.js";
|
|
7
|
+
const SET_NAME = /^[A-Za-z0-9_$.-]+$/u;
|
|
8
|
+
/** A dry run's result, thrown out of the transaction so it rolls back. */
|
|
9
|
+
class DryRun extends Error {
|
|
10
|
+
report;
|
|
11
|
+
constructor(report) {
|
|
12
|
+
super("dry run");
|
|
13
|
+
this.report = report;
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
const text = (value) => value;
|
|
17
|
+
/** The store that copies the whole relational database behind the `StoreAdapter` port (kind `database`, level `physical`). */
|
|
18
|
+
export function postgresStore(options) {
|
|
19
|
+
const { id, run } = options;
|
|
20
|
+
const schemas = options.schemas ?? ["public"];
|
|
21
|
+
const exclude = options.exclude ?? [];
|
|
22
|
+
const pageRows = options.pageRows ?? 5000;
|
|
23
|
+
if (!Number.isInteger(pageRows) || pageRows < 1)
|
|
24
|
+
throw new TypeError("pageRows is a whole number of at least 1");
|
|
25
|
+
const refuse = (cause, table) => {
|
|
26
|
+
if (cause instanceof DataRefusal || cause instanceof Cancelled)
|
|
27
|
+
return cause;
|
|
28
|
+
const params = table === undefined ? { store: id, cause: causeLabel(cause) } : { store: id, table, cause: causeLabel(cause) };
|
|
29
|
+
return new DataRefusal("DATA_STORE_FAILED", "configuration", params, cause);
|
|
30
|
+
};
|
|
31
|
+
async function loadCatalog(from) {
|
|
32
|
+
const excluded = new Set(exclude.map((item) => item.table));
|
|
33
|
+
const cannot = exclude.map((item) => ({ what: `export and import of table ${item.table}`, reason: `excluded by the app: ${item.reason}` }));
|
|
34
|
+
const held = [];
|
|
35
|
+
const rows = [...(await from.query(sql.CATALOG, [schemas]))].map((row) => ({ schema: text(row.schema), table: text(row.table), row }));
|
|
36
|
+
for (const { schema, table, row } of rows.sort((left, right) => byteOrder(`${left.schema}.${left.table}`, `${right.schema}.${right.table}`))) {
|
|
37
|
+
const set = `${schema}.${table}`;
|
|
38
|
+
const skip = (reason) => {
|
|
39
|
+
cannot.push({ what: `export and import of table ${set}`, reason });
|
|
40
|
+
};
|
|
41
|
+
if (excluded.has(set))
|
|
42
|
+
continue;
|
|
43
|
+
if (!SET_NAME.test(set)) {
|
|
44
|
+
skip("its name has characters a set name cannot hold (only letters, digits, _ $ . and -)");
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
if (row.pk === null) {
|
|
48
|
+
skip("it has no primary key, so a row cannot keep its identity");
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
const columns = JSON.parse(text(row.columns));
|
|
52
|
+
const names = JSON.parse(text(row.pk));
|
|
53
|
+
const pk = names.flatMap((name) => columns.filter((column) => column.name === name));
|
|
54
|
+
if (pk.length !== names.length) {
|
|
55
|
+
skip("its primary key holds a generated column");
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
held.push({ schema, table, set, columns, pk });
|
|
59
|
+
}
|
|
60
|
+
return { held, cannot: cannot.sort((left, right) => byteOrder(left.what, right.what)) };
|
|
61
|
+
}
|
|
62
|
+
/** The key's primary key column texts, or null when the key is not one of this table's. */
|
|
63
|
+
function partsOf(table, key) {
|
|
64
|
+
if (table.pk.length === 1)
|
|
65
|
+
return [key];
|
|
66
|
+
try {
|
|
67
|
+
const parts = JSON.parse(key);
|
|
68
|
+
return Array.isArray(parts) && parts.length === table.pk.length && parts.every((part) => typeof part === "string") ? parts : null;
|
|
69
|
+
}
|
|
70
|
+
catch {
|
|
71
|
+
return null;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
/** The row object of an entry when it is a whole row of this table under this key, else null. */
|
|
75
|
+
function rowOf(table, entry) {
|
|
76
|
+
let value;
|
|
77
|
+
try {
|
|
78
|
+
value = recordValue(entry);
|
|
79
|
+
}
|
|
80
|
+
catch {
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
83
|
+
if (typeof value !== "object" || value === null || Array.isArray(value))
|
|
84
|
+
return null;
|
|
85
|
+
const row = value;
|
|
86
|
+
const names = Object.keys(row).sort(byteOrder);
|
|
87
|
+
const wanted = table.columns.map((column) => column.name).sort(byteOrder);
|
|
88
|
+
if (names.length !== wanted.length || names.some((name, at) => name !== wanted[at]))
|
|
89
|
+
return null;
|
|
90
|
+
if (!Object.values(row).every((item) => item === null || typeof item === "string"))
|
|
91
|
+
return null;
|
|
92
|
+
const parts = partsOf(table, entry.key);
|
|
93
|
+
if (parts === null || table.pk.some((column, at) => row[column.name] !== parts[at]))
|
|
94
|
+
return null;
|
|
95
|
+
return row;
|
|
96
|
+
}
|
|
97
|
+
async function* exportEntries(selection) {
|
|
98
|
+
const batches = streamed(async (send) => {
|
|
99
|
+
let current;
|
|
100
|
+
try {
|
|
101
|
+
await run.transaction("repeatable read", async (tx) => {
|
|
102
|
+
await tx.query(sql.SESSION);
|
|
103
|
+
for (const table of (await loadCatalog(tx)).held) {
|
|
104
|
+
if (selection.sets !== undefined && !selection.sets.includes(table.set))
|
|
105
|
+
continue;
|
|
106
|
+
current = table.set;
|
|
107
|
+
let after = null;
|
|
108
|
+
for (;;) {
|
|
109
|
+
const rows = await tx.query(sql.page(table, pageRows, after !== null), after === null ? [] : [after]);
|
|
110
|
+
const entries = rows.map((row) => recordEntry(table.set, text(row.k), JSON.parse(text(row.r)))).sort((left, right) => byteOrder(left.key, right.key));
|
|
111
|
+
if (entries.length > 0)
|
|
112
|
+
await send(entries);
|
|
113
|
+
if (rows.length < pageRows)
|
|
114
|
+
break;
|
|
115
|
+
after = entries.reduce((_, entry) => entry.key, "");
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
catch (cause) {
|
|
121
|
+
throw refuse(cause, current);
|
|
122
|
+
}
|
|
123
|
+
});
|
|
124
|
+
for await (const batch of batches)
|
|
125
|
+
yield* batch;
|
|
126
|
+
}
|
|
127
|
+
async function importEntries(entries, options) {
|
|
128
|
+
const tallies = new Map();
|
|
129
|
+
let current;
|
|
130
|
+
try {
|
|
131
|
+
return await run.transaction("read committed", async (tx) => {
|
|
132
|
+
await tx.query(sql.SESSION);
|
|
133
|
+
await tx.query(sql.REPLICA);
|
|
134
|
+
const byName = new Map((await loadCatalog(tx)).held.map((table) => [table.set, table]));
|
|
135
|
+
for await (const entry of entries) {
|
|
136
|
+
const tally = tallies.get(entry.set) ?? { created: 0, updated: 0, unchanged: 0, removed: 0, refused: [], seen: new Set() };
|
|
137
|
+
tallies.set(entry.set, tally);
|
|
138
|
+
tally.seen.add(entry.key);
|
|
139
|
+
const table = byName.get(entry.set);
|
|
140
|
+
if (table === undefined) {
|
|
141
|
+
tally.refused.push({ key: entry.key, code: "DATA_TABLE_NOT_HELD" });
|
|
142
|
+
continue;
|
|
143
|
+
}
|
|
144
|
+
current = table.set;
|
|
145
|
+
const row = rowOf(table, entry);
|
|
146
|
+
if (row === null) {
|
|
147
|
+
tally.refused.push({ key: entry.key, code: "DATA_ROW_INVALID" });
|
|
148
|
+
continue;
|
|
149
|
+
}
|
|
150
|
+
const [stored] = await tx.query(sql.get(table), table.pk.map((column) => row[column.name]));
|
|
151
|
+
if (stored === undefined)
|
|
152
|
+
tally.created += 1;
|
|
153
|
+
else if (canonicalJson(JSON.parse(text(stored.r))) === canonicalJson(row)) {
|
|
154
|
+
tally.unchanged += 1;
|
|
155
|
+
continue;
|
|
156
|
+
}
|
|
157
|
+
else
|
|
158
|
+
tally.updated += 1;
|
|
159
|
+
await tx.query(sql.upsert(table), [JSON.stringify(row)]);
|
|
160
|
+
}
|
|
161
|
+
const held = [...tallies].flatMap(([set, tally]) => {
|
|
162
|
+
const table = byName.get(set);
|
|
163
|
+
return table === undefined ? [] : [{ table, tally }];
|
|
164
|
+
});
|
|
165
|
+
for (const { table, tally } of held) {
|
|
166
|
+
current = table.set;
|
|
167
|
+
if (options.mode === "mirror") {
|
|
168
|
+
for (const row of await tx.query(sql.keys(table))) {
|
|
169
|
+
if (tally.seen.has(text(row.k)))
|
|
170
|
+
continue;
|
|
171
|
+
await tx.query(sql.remove(table), table.pk.map((_, at) => row[`p${String(at)}`]));
|
|
172
|
+
tally.removed += 1;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
// setval is not transactional: a dry run must not move a sequence.
|
|
176
|
+
if (options.dryRun)
|
|
177
|
+
continue;
|
|
178
|
+
for (const owned of await tx.query(sql.SEQUENCES, [sql.qualified(table)]))
|
|
179
|
+
await tx.query(sql.setval(table, text(owned.column)), [text(owned.sequence)]);
|
|
180
|
+
}
|
|
181
|
+
const sets = [...tallies]
|
|
182
|
+
.map(([set, tally]) => ({ set, created: tally.created, updated: tally.updated, unchanged: tally.unchanged, removed: tally.removed, refused: [...tally.refused].sort((left, right) => byteOrder(left.key, right.key)) }))
|
|
183
|
+
.sort((left, right) => byteOrder(left.set, right.set));
|
|
184
|
+
const report = { sets, outcome: importOutcome(sets) };
|
|
185
|
+
// A dry run did every write above, counted it, and now rolls the transaction back.
|
|
186
|
+
if (options.dryRun)
|
|
187
|
+
throw new DryRun(report);
|
|
188
|
+
return report;
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
catch (cause) {
|
|
192
|
+
if (cause instanceof DryRun)
|
|
193
|
+
return cause.report;
|
|
194
|
+
throw refuse(cause, current);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
return {
|
|
198
|
+
id,
|
|
199
|
+
kind: "database",
|
|
200
|
+
level: "physical",
|
|
201
|
+
async describe() {
|
|
202
|
+
try {
|
|
203
|
+
const catalog = await loadCatalog(run);
|
|
204
|
+
return {
|
|
205
|
+
sets: catalog.held.map((table) => ({ set: table.set, schema: `postgres/${table.set}@1`, form: "record" })),
|
|
206
|
+
purposes: ["seed-backup"],
|
|
207
|
+
canExport: true,
|
|
208
|
+
canImport: true,
|
|
209
|
+
cannot: catalog.cannot,
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
catch (cause) {
|
|
213
|
+
throw refuse(cause);
|
|
214
|
+
}
|
|
215
|
+
},
|
|
216
|
+
export: (selection) => exportEntries(selection),
|
|
217
|
+
import: importEntries,
|
|
218
|
+
async identityOf(set, key) {
|
|
219
|
+
try {
|
|
220
|
+
const table = (await loadCatalog(run)).held.find((item) => item.set === set);
|
|
221
|
+
const parts = table === undefined ? null : partsOf(table, key);
|
|
222
|
+
if (table === undefined || parts === null)
|
|
223
|
+
return null;
|
|
224
|
+
return (await run.query(sql.exists(table), parts)).length > 0 ? key : null;
|
|
225
|
+
}
|
|
226
|
+
catch (cause) {
|
|
227
|
+
throw refuse(cause, set);
|
|
228
|
+
}
|
|
229
|
+
},
|
|
230
|
+
};
|
|
231
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import type { DataEntry } from "./manifest.js";
|
|
2
|
+
import type { StoreAdapter } from "./store.js";
|
|
3
|
+
import type { StoreKind, StoreLevel } from "./words.js";
|
|
4
|
+
/**
|
|
5
|
+
* The line protocol `starci/data-entries@1` between the harness and an app's store command: newline-delimited JSON.
|
|
6
|
+
*
|
|
7
|
+
* <argv> describe -> one line: StoreDescription
|
|
8
|
+
* <argv> export <Selection JSON> -> one line per entry, then {"end": true, "items": n}
|
|
9
|
+
* <argv> import <ImportOptions> <- entry lines on stdin; -> last line: ImportReport
|
|
10
|
+
* <argv> identity <set> <key> -> one line: {"identity": string | null}
|
|
11
|
+
*
|
|
12
|
+
* A record's body travels as its canonical JSON text, a blob's as base64. A value never reaches argv or a log.
|
|
13
|
+
*/
|
|
14
|
+
export declare const DATA_PROTOCOL = "starci/data-entries@1";
|
|
15
|
+
/** One entry as a protocol line. */
|
|
16
|
+
export interface EntryLine {
|
|
17
|
+
/** The set. */
|
|
18
|
+
readonly set: string;
|
|
19
|
+
/** The natural key. */
|
|
20
|
+
readonly key: string;
|
|
21
|
+
/** record or blob. */
|
|
22
|
+
readonly form: DataEntry["form"];
|
|
23
|
+
/** The media type. */
|
|
24
|
+
readonly media: string;
|
|
25
|
+
/** sha256 of the body. */
|
|
26
|
+
readonly sha256: string;
|
|
27
|
+
/** The canonical JSON text of a record, the base64 of a blob. */
|
|
28
|
+
readonly body: string;
|
|
29
|
+
}
|
|
30
|
+
/** The protocol line of an entry. */
|
|
31
|
+
export declare function encodeEntry(entry: DataEntry): string;
|
|
32
|
+
/** The entry of a protocol line; a body that does not match its sha256 is refused. */
|
|
33
|
+
export declare function decodeEntry(text: string): DataEntry;
|
|
34
|
+
/** What the app's command reads and writes. */
|
|
35
|
+
export interface ServeIo {
|
|
36
|
+
/** The arguments after the command's own: `describe`, `export <json>` or `import <json>`. */
|
|
37
|
+
readonly args: readonly string[];
|
|
38
|
+
/** stdin, line by line (only `import` reads it). */
|
|
39
|
+
readonly lines: AsyncIterable<string>;
|
|
40
|
+
/** Writes one line to stdout. */
|
|
41
|
+
readonly write: (line: string) => void;
|
|
42
|
+
}
|
|
43
|
+
/** Serves one store over the protocol from an app's command; answers the exit code. */
|
|
44
|
+
export declare function serveStore(store: StoreAdapter, io: ServeIo): Promise<number>;
|
|
45
|
+
/** Runs the app's command with extra arguments and optional stdin lines; yields its stdout lines. */
|
|
46
|
+
export type CommandRunner = (args: readonly string[], input?: AsyncIterable<string>) => AsyncIterable<string>;
|
|
47
|
+
/** A store reached through an app's command: the harness side of the protocol. */
|
|
48
|
+
export declare function commandStore(declared: {
|
|
49
|
+
readonly id: string;
|
|
50
|
+
readonly kind: StoreKind;
|
|
51
|
+
readonly level: StoreLevel;
|
|
52
|
+
}, run: CommandRunner): StoreAdapter;
|