@substrio/cli 0.0.0-stage → 0.1.1

package/src/collections.mjs

@@ -0,0 +1,701 @@
+/**
+ * Substrio CLI: the collections an app declares, one file each:
+ *
+ *   collections/events.json       the directory is substrio.toml's collections_dir
+ *   { "schema":     { "rules": [...], "strict": true },   as schema.update takes it
+ *     "rules":      { "read": "allow", "write": "auth" }, this collection's entry of the security rules
+ *     "indexes":    [...],                               every key is optional
+ *     "expire":     { "field": "...", "after_ms": 0, "existing": "refuse" },
+ *     "id":         { "derive": "..." },
+ *     "aggregates": [...] }
+ *
+ * The file's name is the collection's. Applied idempotently: the collection is
+ * made when it is not there, its schema replaced by the declared one, and its
+ * rules join the app's security rules (rules_file, then these, one
+ * security.rules.set body). A collection whose rules are in rules_file and in
+ * its own file is refused, naming both. `substrio deploy` and `substrio dev` apply them
+ * with the server's operations (collectionSteps); `substrio push` carries them in
+ * the version, with the other members, when the server's versions take them.
+ * Nothing here talks to the server except applyCollections and collectionDrift
+ * / pruneDrift, through the `invoke` they are given.
+ *
+ * Every declaration this engine takes for a collection (index.create's kind, see
+ * src/substrio/api/substrio_sql_http.cpp SubstrioAPI::handle_index_create) is one entry of `indexes` or `expire`,
+ * passed through under the server's own names, nothing translated:
+ *
+ *   server declaration (index.create)                    file entry (one of `indexes`)
+ *   ---------------------------------------------------   ------------------------------------------------
+ *   kind collection, field                                "field.path"
+ *   kind collection, field, unique                        { field, unique: true }
+ *   kind collection, field, keyword, keyword_term_bytes?   { field, keyword: true, keyword_term_bytes? }
+ *   kind collection, fields, unique: true, nulls?, within? { fields: [...], unique: true, nulls?, within? }
+ *   kind collection, field, elements                      { field, elements: [...] }
+ *   kind collection, compound                              { compound: [...] }
+ *   kind aggregate                                        one entry of `aggregates` (AggregateKey), not `indexes`
+ *   kind expiry                                           the file's `expire` (one per collection, not an array)
+ *   kind id_derive                                        the file's `id.derive` (one per collection, UM-3d)
+ *   kind space (spans collections)                        not here: it is not one collection's; it belongs in a
+ *                                                          project-level file, which does not exist yet
+ *
+ * Each declared index is sent after the collection and its schema with if_not_exists, so applying twice changes
+ * nothing; the server builds the field's keys before it answers, and a unique declaration over documents that
+ * already share a value is refused (UNIQUE or INDEX_DUPLICATES) and stored nowhere. The declared `id.derive` is
+ * one `index.create` of kind id_derive, sent right after the collection is made (UM-3d): idempotent for its own
+ * value on its own (no if_not_exists), refused when it would CHANGE the prefix of a collection that already
+ * holds an object. The declared `expire` is one `index.create` of kind expiry, sent after the indexes, with
+ * if_not_exists (a second declaration of the same collection is a no-op, not a conflict). Each declared
+ * aggregate ({name, group?, metrics, where?}) is one `index.create` of kind aggregate, sent after `expire`, in
+ * file order; it builds its groups before it answers and is idempotent for the same name and shape on its own
+ * (no if_not_exists: a different shape under the same name is refused).
+ *
+ * collectionDrift reads what the server has declared for `collections` (index.list kind=collection, aggregate,
+ * expiry) and reports what the file no longer names (a field's undeclared adaptive key is never in that answer,
+ * so it is never reported); `substrio deploy --prune` and `substrio dev --prune` call pruneDrift to drop it with
+ * index.drop. Without --prune a drift is only reported, left on the server.
+ */
+
+import { existsSync, readdirSync } from 'node:fs';
+import { basename, extname, join, resolve } from 'node:path';
+
+import { sha256 } from './blobs.mjs';
+import { CliError, ErrorCode } from './cli.mjs';
+import { readJsonFile } from './jsonfile.mjs';
+import { PAGE_CURSOR_INPUT, PAGE_NEXT_CURSOR_FIELD, buildRequest, findOperation, pickRoute } from './ops.mjs';
+
+/** The directory of collection files unless substrio.toml's collections_dir names another. */
+export const DEFAULT_COLLECTIONS_DIR = 'collections';
+/** The version member that carries the collections. */
+export const COLLECTIONS_MEMBER = 'collections';
+
+/** The keys of a collection file. */
+export const CollectionKey = Object.freeze({
+    SCHEMA: 'schema',
+    RULES: 'rules',
+    INDEXES: 'indexes',
+    EXPIRE: 'expire',
+    AGGREGATES: 'aggregates',
+    ID: 'id',
+});
+const KEYS = new Set(Object.values(CollectionKey));
+
+/** The keys of one entry of a collection file's `indexes`: a single field, a compound unique, an element tuple or a compound index. */
+export const IndexKey = Object.freeze({
+    FIELD: 'field',
+    FIELDS: 'fields',
+    UNIQUE: 'unique',
+    NULLS: 'nulls',
+    WITHIN: 'within',
+    KEYWORD: 'keyword',
+    KEYWORD_TERM_BYTES: 'keyword_term_bytes',
+    ELEMENTS: 'elements',
+    COMPOUND: 'compound',
+});
+const INDEX_KEYS = new Set([IndexKey.FIELD, IndexKey.UNIQUE, IndexKey.KEYWORD, IndexKey.KEYWORD_TERM_BYTES]);
+const FIELDS_KEYS = new Set([IndexKey.FIELDS, IndexKey.UNIQUE, IndexKey.NULLS, IndexKey.WITHIN]);
+const ELEMENTS_KEYS = new Set([IndexKey.FIELD, IndexKey.ELEMENTS]);
+const COMPOUND_KEYS = new Set([IndexKey.COMPOUND]);
+
+/** The shape an entry of `indexes` normalizes to (indexEntry's `type`). */
+export const IndexEntryKind = Object.freeze({
+    FIELD: 'field',
+    FIELDS: 'fields',
+    ELEMENTS: 'elements',
+    COMPOUND: 'compound',
+});
+
+/** The keys of the collection file's `expire`. */
+export const ExpireKey = Object.freeze({
+    FIELD: 'field',
+    AFTER_MS: 'after_ms',
+    EXISTING: 'existing',
+});
+const EXPIRE_KEYS = new Set(Object.values(ExpireKey));
+
+/** The keys of the collection file's `id`. */
+export const IdKey = Object.freeze({
+    DERIVE: 'derive',
+});
+const ID_KEYS = new Set(Object.values(IdKey));
+const ID_DERIVE_MAX_BYTES = 64;
+
+/** The keys of one entry of a collection file's `aggregates`. */
+export const AggregateKey = Object.freeze({
+    NAME: 'name',
+    GROUP: 'group',
+    METRICS: 'metrics',
+    WHERE: 'where',
+});
+const AGGREGATE_KEYS = new Set(Object.values(AggregateKey));
+
+/** The HTTP status of "that collection exists already" (collection.create). */
+const EXISTS = 409;
+
+const isObject = (value) => value !== null && typeof value === 'object' && !Array.isArray(value);
+
+/** The directory name substrio.toml gives (collections_dir), or the default. */
+export const collectionsDirOf = (config) => config.collections_dir || DEFAULT_COLLECTIONS_DIR;
+
+/**
+ * One entry of a collection file's `indexes`, normalized by shape (pure): a field name or {field, unique?,
+ * keyword?, keyword_term_bytes?} is IndexEntryKind.FIELD ({field, unique, keyword, termBytes}); {fields, unique:
+ * true, nulls?, within?} is FIELDS, a compound unique declaration ({fields, nulls, within}); {field, elements}
+ * is ELEMENTS, an element tuple ({field, elements}); {compound} is COMPOUND, an ordered compound index
+ * ({compound}). A field name alone, or {field, unique: false}, is an index that is not unique.
+ */
+export const indexEntry = (entry) => {
+    if (typeof entry === 'string') return { type: IndexEntryKind.FIELD, field: entry, unique: false, keyword: false, termBytes: undefined };
+    if (Object.hasOwn(entry, IndexKey.COMPOUND)) return { type: IndexEntryKind.COMPOUND, compound: entry[IndexKey.COMPOUND] };
+    if (Object.hasOwn(entry, IndexKey.ELEMENTS)) return { type: IndexEntryKind.ELEMENTS, field: entry.field, elements: entry[IndexKey.ELEMENTS] };
+    if (Object.hasOwn(entry, IndexKey.FIELDS)) {
+        return { type: IndexEntryKind.FIELDS, fields: entry[IndexKey.FIELDS], nulls: entry[IndexKey.NULLS], within: entry[IndexKey.WITHIN] };
+    }
+    return { type: IndexEntryKind.FIELD, field: entry.field, unique: entry.unique === true, keyword: entry.keyword === true, termBytes: entry[IndexKey.KEYWORD_TERM_BYTES] };
+};
+
+/** The `index.create` input of one declared index entry (pure), under the server's own kind and names. */
+export const indexCreateInput = (collection, entry) => {
+    if (entry.type === IndexEntryKind.COMPOUND) return { kind: 'collection', collection, compound: entry.compound, if_not_exists: true };
+    if (entry.type === IndexEntryKind.ELEMENTS) return { kind: 'collection', collection, field: entry.field, elements: entry.elements, if_not_exists: true };
+    if (entry.type === IndexEntryKind.FIELDS) {
+        return {
+            kind: 'collection', collection, fields: entry.fields, unique: true,
+            ...(entry.nulls === undefined ? {} : { nulls: entry.nulls }),
+            ...(entry.within === undefined ? {} : { within: entry.within }),
+            if_not_exists: true,
+        };
+    }
+    return entry.keyword
+        ? { kind: 'collection', collection, field: entry.field, keyword: true, ...(entry.termBytes === undefined ? {} : { keyword_term_bytes: entry.termBytes }), if_not_exists: true }
+        : { kind: 'collection', collection, field: entry.field, unique: entry.unique, if_not_exists: true };
+};
+
+/**
+ * The identity of one declared index entry, normalized by indexEntry (pure): matches serverIndexIdentity's of
+ * the same declaration read back from index.list, so collectionDrift never reports a declaration the file
+ * still names in a different but equivalent form (a plain and a unique index on the same field are still two
+ * different declarations, by design: they are never the same identity).
+ */
+function fileIndexIdentity(entry) {
+    if (entry.type === IndexEntryKind.COMPOUND) return `compound:${entry.compound.join(',')}`;
+    if (entry.type === IndexEntryKind.ELEMENTS) return `elements:${entry.field}`;
+    if (entry.type === IndexEntryKind.FIELDS) return `unique:${[...entry.fields].sort().join(',')}`;
+    return entry.keyword ? `keyword:${entry.field}` : `field:${entry.field}`;
+}
+
+/** The identity of one index.list kind=collection entry (the server's own answer), matched against fileIndexIdentity. */
+function serverIndexIdentity(entry) {
+    if (Object.hasOwn(entry, IndexKey.COMPOUND)) return `compound:${entry.compound.join(',')}`;
+    if (Object.hasOwn(entry, IndexKey.ELEMENTS)) return `elements:${entry.field}`;
+    if (Object.hasOwn(entry, IndexKey.KEYWORD)) return `keyword:${entry.field}`;
+    if (Object.hasOwn(entry, IndexKey.FIELDS)) return `unique:${[...entry.fields].sort().join(',')}`;
+    return `field:${entry.field}`;
+}
+
+// One index.list kind=collection entry, as a drift report's label (pure).
+function serverIndexLabel(entry) {
+    if (Object.hasOwn(entry, IndexKey.COMPOUND)) return `compound (${entry.compound.join(', ')})`;
+    if (Object.hasOwn(entry, IndexKey.ELEMENTS)) return `${entry.field} elements (${entry.elements.join(', ')})`;
+    if (Object.hasOwn(entry, IndexKey.KEYWORD)) return `${entry.field} (keyword)`;
+    if (Object.hasOwn(entry, IndexKey.FIELDS)) return `unique (${entry.fields.join(', ')})`;
+    return entry.unique ? `${entry.field} (unique)` : entry.field;
+}
+
+// The `index.drop` input that removes one index.list kind=collection entry (pure); `name` fills the route's
+// {name}, ignored by the server when `fields` or `compound` names the declaration instead (index_declaration_options,
+// handle_index_drop), so any of the declaration's own fields does for it.
+function serverIndexDropInput(collection, entry) {
+    if (Object.hasOwn(entry, IndexKey.COMPOUND)) return { kind: 'collection', collection, name: entry.compound[0], compound: entry.compound };
+    if (Object.hasOwn(entry, IndexKey.ELEMENTS)) return { kind: 'collection', collection, name: entry.field, elements: entry.elements };
+    if (Object.hasOwn(entry, IndexKey.KEYWORD)) return { kind: 'collection', collection, name: entry.field, keyword: true };
+    if (Object.hasOwn(entry, IndexKey.FIELDS)) return { kind: 'collection', collection, name: entry.fields[0], fields: entry.fields };
+    return { kind: 'collection', collection, name: entry.field };
+}
+
+/**
+ * Whether `field` is a field path the server takes (pure): a top-level name or member names joined by single
+ * dots, no segment empty ("a..b", ".a" and "a." are not).
+ */
+export const isFieldPath = (field) => typeof field === 'string' && field !== '' && field.split('.').every((segment) => segment !== '');
+
+// `compound`, checked: an array of two or more field paths, the lead fields then the order field. Returns it as read.
+function checkedCompoundEntry(where, entry) {
+    const unknown = Object.keys(entry).find((key) => !COMPOUND_KEYS.has(key));
+    if (unknown !== undefined) throw new CliError(`${where} has unknown key "${unknown}"; a compound entry takes compound`, { code: ErrorCode.INVALID_ARGUMENT });
+    const compound = entry[IndexKey.COMPOUND];
+    if (!Array.isArray(compound) || compound.length < 2 || !compound.every(isFieldPath)) {
+        throw new CliError(`${where}: compound must be an array of two or more field names, the lead fields then the order field`, { code: ErrorCode.INVALID_ARGUMENT });
+    }
+    return compound;
+}
+
+// {field, elements}, checked: field a path, elements one or more member paths of its array. Returns elements as read.
+function checkedElementsEntry(where, entry) {
+    const unknown = Object.keys(entry).find((key) => !ELEMENTS_KEYS.has(key));
+    if (unknown !== undefined) throw new CliError(`${where} has unknown key "${unknown}"; an elements entry takes field, elements`, { code: ErrorCode.INVALID_ARGUMENT });
+    if (!isFieldPath(entry.field)) {
+        throw new CliError(`${where}: field must be the name of a field of the collection or a dotted path into one (no empty segment)`, { code: ErrorCode.INVALID_ARGUMENT });
+    }
+    const elements = entry[IndexKey.ELEMENTS];
+    if (!Array.isArray(elements) || elements.length === 0 || !elements.every(isFieldPath)) {
+        throw new CliError(`${where}: elements must be an array of one or more field names, the array's members`, { code: ErrorCode.INVALID_ARGUMENT });
+    }
+    return elements;
+}
+
+// {fields, unique: true, nulls?, within?}, checked: a compound unique declaration. Returns fields as read.
+function checkedFieldsEntry(where, entry) {
+    const unknown = Object.keys(entry).find((key) => !FIELDS_KEYS.has(key));
+    if (unknown !== undefined) throw new CliError(`${where} has unknown key "${unknown}"; a compound unique entry takes fields, unique, nulls, within`, { code: ErrorCode.INVALID_ARGUMENT });
+    if (entry[IndexKey.UNIQUE] !== true) {
+        throw new CliError(`${where}: fields is a compound unique declaration; unique must be true`, { code: ErrorCode.INVALID_ARGUMENT });
+    }
+    const fields = entry[IndexKey.FIELDS];
+    if (!Array.isArray(fields) || fields.length < 2 || !fields.every(isFieldPath)) {
+        throw new CliError(`${where}: fields must be an array of two or more field names`, { code: ErrorCode.INVALID_ARGUMENT });
+    }
+    const nulls = entry[IndexKey.NULLS];
+    if (nulls !== undefined && nulls !== 'skip' && nulls !== 'count') {
+        throw new CliError(`${where}: nulls must be skip or count`, { code: ErrorCode.INVALID_ARGUMENT });
+    }
+    const within = entry[IndexKey.WITHIN];
+    if (within !== undefined && within !== 'collection' && within !== 'document' && within !== 'both') {
+        throw new CliError(`${where}: within must be collection, document or both`, { code: ErrorCode.INVALID_ARGUMENT });
+    }
+    return fields;
+}
+
+// The `indexes` of a collection file, checked entry by entry: a field name or path, {field, unique?}, {fields,
+// unique: true, nulls?, within?}, {field, elements} or {compound}; each declaration named once. Returns them as written.
+function checkedIndexes(shown, indexes) {
+    const seen = new Set();
+    indexes.forEach((entry, at) => {
+        const where = `${shown}: indexes[${at}]`;
+        if (typeof entry === 'string') {
+            if (!isFieldPath(entry)) throw new CliError(`${where}: field must be the name of a field of the collection or a dotted path into one (no empty segment)`, { code: ErrorCode.INVALID_ARGUMENT });
+            if (seen.has(`index:${entry}`)) throw new CliError(`${shown}: the index on "${entry}" is declared twice`, { code: ErrorCode.INVALID_ARGUMENT });
+            seen.add(`index:${entry}`);
+            return;
+        }
+        if (!isObject(entry)) throw new CliError(`${where} must be a field name or an object {field, unique}`, { code: ErrorCode.INVALID_ARGUMENT });
+        if (Object.hasOwn(entry, IndexKey.COMPOUND)) {
+            const compound = checkedCompoundEntry(where, entry);
+            const key = `compound:${compound.join(',')}`;
+            if (seen.has(key)) throw new CliError(`${shown}: the compound index on "${compound.join(', ')}" is declared twice`, { code: ErrorCode.INVALID_ARGUMENT });
+            seen.add(key);
+            return;
+        }
+        if (Object.hasOwn(entry, IndexKey.ELEMENTS)) {
+            checkedElementsEntry(where, entry);
+            const key = `elements:${entry.field}`;
+            if (seen.has(key)) throw new CliError(`${shown}: the elements index on "${entry.field}" is declared twice`, { code: ErrorCode.INVALID_ARGUMENT });
+            seen.add(key);
+            return;
+        }
+        if (Object.hasOwn(entry, IndexKey.FIELDS)) {
+            const fields = checkedFieldsEntry(where, entry);
+            const key = `unique:${[...fields].sort().join(',')}`;
+            if (seen.has(key)) throw new CliError(`${shown}: the unique index on "${[...fields].sort().join(', ')}" is declared twice`, { code: ErrorCode.INVALID_ARGUMENT });
+            seen.add(key);
+            return;
+        }
+        const unknown = Object.keys(entry).find((key) => !INDEX_KEYS.has(key));
+        if (unknown !== undefined) {
+            throw new CliError(`${where} has unknown key "${unknown}"; an index takes ${[...INDEX_KEYS].join(', ')}`, { code: ErrorCode.INVALID_ARGUMENT });
+        }
+        const field = entry[IndexKey.FIELD];
+        if (!isFieldPath(field)) {
+            throw new CliError(`${where}: field must be the name of a field of the collection or a dotted path into one (no empty segment)`, { code: ErrorCode.INVALID_ARGUMENT });
+        }
+        const unique = entry[IndexKey.UNIQUE];
+        if (unique !== undefined && typeof unique !== 'boolean') {
+            throw new CliError(`${where}: unique must be true or false`, { code: ErrorCode.INVALID_ARGUMENT });
+        }
+        const keyword = entry[IndexKey.KEYWORD];
+        if (keyword !== undefined && typeof keyword !== 'boolean') {
+            throw new CliError(`${where}: keyword must be true or false`, { code: ErrorCode.INVALID_ARGUMENT });
+        }
+        const termBytes = entry[IndexKey.KEYWORD_TERM_BYTES];
+        if (termBytes !== undefined && (keyword !== true || !Number.isInteger(termBytes) || termBytes < 1)) {
+            throw new CliError(`${where}: keyword_term_bytes is a whole number of bytes from 1, and only with keyword: true`, { code: ErrorCode.INVALID_ARGUMENT });
+        }
+        if (keyword === true && unique === true) {
+            throw new CliError(`${where}: keyword and unique are separate declarations of a field; list the field twice`, { code: ErrorCode.INVALID_ARGUMENT });
+        }
+        // A field may be declared once as an index (plain or unique) and once as a keyword field.
+        const declared = keyword === true ? `keyword:${field}` : `index:${field}`;
+        if (seen.has(declared)) throw new CliError(`${shown}: the ${keyword === true ? 'keyword ' : ''}index on "${field}" is declared twice`, { code: ErrorCode.INVALID_ARGUMENT });
+        seen.add(declared);
+    });
+    return indexes;
+}
+
+/** The `index.create` input of the declared `expire` (pure), under the server's own kind and names. */
+export const expireCreateInput = (collection, expire) => ({
+    kind: 'expiry', collection, field: expire.field, after_ms: expire.after_ms,
+    ...(expire.existing ? { existing: expire.existing } : {}),
+    if_not_exists: true,
+});
+
+// A collection file's `expire`, checked: {field, after_ms, existing?}, one per collection. Returns it as
+// {field, after_ms, existing} (existing: null when absent).
+function checkedExpire(shown, expire) {
+    const where = `${shown}: expire`;
+    const unknown = Object.keys(expire).find((key) => !EXPIRE_KEYS.has(key));
+    if (unknown !== undefined) throw new CliError(`${where} has unknown key "${unknown}"; expire takes field, after_ms, existing`, { code: ErrorCode.INVALID_ARGUMENT });
+    const field = expire[ExpireKey.FIELD];
+    if (!isFieldPath(field)) {
+        throw new CliError(`${where}: field must be the name of a field of the collection or a dotted path into one (no empty segment)`, { code: ErrorCode.INVALID_ARGUMENT });
+    }
+    const afterMs = expire[ExpireKey.AFTER_MS];
+    if (!Number.isInteger(afterMs) || afterMs < 0) {
+        throw new CliError(`${where}: after_ms must be a whole number of milliseconds from 0`, { code: ErrorCode.INVALID_ARGUMENT });
+    }
+    const existing = expire[ExpireKey.EXISTING];
+    if (existing !== undefined && existing !== 'refuse' && existing !== 'allow') {
+        throw new CliError(`${where}: existing must be refuse or allow`, { code: ErrorCode.INVALID_ARGUMENT });
+    }
+    return { field, after_ms: afterMs, existing: existing ?? null };
+}
+
+// A collection file's `id`, checked: {derive}, derive a non-empty string of at most 64 bytes.
+// Returns it as {derive}.
+function checkedId(shown, id) {
+    const where = `${shown}: id`;
+    const unknown = Object.keys(id).find((key) => !ID_KEYS.has(key));
+    if (unknown !== undefined) throw new CliError(`${where} has unknown key "${unknown}"; id takes derive`, { code: ErrorCode.INVALID_ARGUMENT });
+    const derive = id[IdKey.DERIVE];
+    if (typeof derive !== 'string' || derive === '' || Buffer.byteLength(derive, 'utf8') > ID_DERIVE_MAX_BYTES) {
+        throw new CliError(`${where}: derive must be a non-empty string of at most ${ID_DERIVE_MAX_BYTES} bytes`, { code: ErrorCode.INVALID_ARGUMENT });
+    }
+    return { derive };
+}
+
+/** The `index.create` input of the declared `id.derive` (pure), under the server's own kind and names. */
+export const idCreateInput = (collection, id) => ({ kind: 'id_derive', collection, derive: id.derive });
+
+/** The `index.create` input of one declared aggregate entry (pure). */
+export const aggregateCreateInput = (collection, entry) => ({
+    kind: 'aggregate', collection, name: entry.name,
+    ...(entry.group.length === 0 ? {} : { group: entry.group }),
+    metrics: entry.metrics,
+    ...(entry.where === null ? {} : { where: entry.where }),
+});
+
+// One entry of `group`, checked: a field path, or {field, step?}. Returns it as read.AggregateGroupPartText takes.
+function checkedGroupPart(where, entry) {
+    if (typeof entry === 'string') {
+        if (!isFieldPath(entry)) throw new CliError(`${where}: group entries must be a field path or {field, step}`, { code: ErrorCode.INVALID_ARGUMENT });
+        return entry;
+    }
+    if (!isObject(entry) || !isFieldPath(entry.field)) {
+        throw new CliError(`${where}: group entries must be a field path or {field, step}`, { code: ErrorCode.INVALID_ARGUMENT });
+    }
+    const unknown = Object.keys(entry).find((key) => key !== 'field' && key !== 'step');
+    if (unknown !== undefined) throw new CliError(`${where} has unknown key "${unknown}"; a group entry takes field, step`, { code: ErrorCode.INVALID_ARGUMENT });
+    if (entry.step !== undefined && typeof entry.step !== 'number') {
+        throw new CliError(`${where}: step must be a number`, { code: ErrorCode.INVALID_ARGUMENT });
+    }
+    return entry.step === undefined ? { field: entry.field } : { field: entry.field, step: entry.step };
+}
+
+// `metrics`, checked: {name: "count" | {sum: field}, ...}, at least one. Returns it as read.
+function checkedMetrics(where, metrics) {
+    if (!isObject(metrics)) throw new CliError(`${where}: metrics must be an object {name: "count" or {sum: field}, ...}`, { code: ErrorCode.INVALID_ARGUMENT });
+    const names = Object.keys(metrics);
+    if (names.length === 0) throw new CliError(`${where}: metrics names at least one metric`, { code: ErrorCode.INVALID_ARGUMENT });
+    for (const name of names) {
+        const value = metrics[name];
+        if (value === 'count') continue;
+        if (isObject(value) && Object.keys(value).length === 1 && isFieldPath(value.sum)) continue;
+        throw new CliError(`${where}: metrics.${name} must be "count" or {sum: field}`, { code: ErrorCode.INVALID_ARGUMENT });
+    }
+    return metrics;
+}
+
+// The `aggregates` of a collection file, checked entry by entry: {name, group?, metrics, where?}; each name once.
+// Returns them as {name, group, metrics, where} (group: [], where: null when absent).
+function checkedAggregates(shown, aggregates) {
+    const seen = new Set();
+    return aggregates.map((entry, at) => {
+        const where = `${shown}: aggregates[${at}]`;
+        if (!isObject(entry)) throw new CliError(`${where} must be an object {name, group, metrics, where}`, { code: ErrorCode.INVALID_ARGUMENT });
+        const unknown = Object.keys(entry).find((key) => !AGGREGATE_KEYS.has(key));
+        if (unknown !== undefined) {
+            throw new CliError(`${where} has unknown key "${unknown}"; an aggregate takes ${[...AGGREGATE_KEYS].join(', ')}`, { code: ErrorCode.INVALID_ARGUMENT });
+        }
+        const name = entry[AggregateKey.NAME];
+        if (typeof name !== 'string' || name === '') throw new CliError(`${where}: name must be a non-empty string`, { code: ErrorCode.INVALID_ARGUMENT });
+        if (seen.has(name)) throw new CliError(`${shown}: the aggregate "${name}" is declared twice`, { code: ErrorCode.INVALID_ARGUMENT });
+        seen.add(name);
+        const group = entry[AggregateKey.GROUP];
+        if (group !== undefined && !Array.isArray(group)) throw new CliError(`${where}: group must be an array`, { code: ErrorCode.INVALID_ARGUMENT });
+        const metrics = checkedMetrics(where, entry[AggregateKey.METRICS]);
+        const whereClause = entry[AggregateKey.WHERE];
+        if (whereClause !== undefined && (typeof whereClause !== 'string' || whereClause === '')) {
+            throw new CliError(`${where}: where must be a non-empty string`, { code: ErrorCode.INVALID_ARGUMENT });
+        }
+        return {
+            name,
+            group: (group ?? []).map((part) => checkedGroupPart(where, part)),
+            metrics,
+            where: whereClause ?? null,
+        };
+    });
+}
+
+// One collection file, checked: {name, file, schema, rules, indexes, aggregates, id} (an absent key is null).
+function parseCollection(config, cwd, fileName) {
+    const shown = `${collectionsDirOf(config)}/${fileName}`;
+    const name = basename(fileName, '.json');
+    const body = readJsonFile(join(resolve(cwd, collectionsDirOf(config)), fileName), 'collection file', shown);
+    if (!isObject(body)) throw new CliError(`${shown} must be a JSON object with schema, rules and indexes`, { code: ErrorCode.INVALID_ARGUMENT });
+    const unknown = Object.keys(body).filter((key) => !KEYS.has(key));
+    if (unknown.length) {
+        throw new CliError(`${shown} has unknown key "${unknown[0]}"; a collection file takes ${[...KEYS].join(', ')}`, { code: ErrorCode.INVALID_ARGUMENT });
+    }
+    const checked = (key, ok, what) => {
+        const value = body[key];
+        if (value === undefined || value === null) return null;
+        if (!ok(value)) throw new CliError(`${shown}: ${key} must be ${what}`, { code: ErrorCode.INVALID_ARGUMENT });
+        return value;
+    };
+    const rules = checked(CollectionKey.RULES, isObject, 'an object (read, write, create, delete)');
+    if (rules !== null && Object.hasOwn(rules, 'collection')) {
+        throw new CliError(`${shown}: rules must not name a collection; the file's name is the collection`, { code: ErrorCode.INVALID_ARGUMENT });
+    }
+    return {
+        name, file: shown,
+        schema: checked(CollectionKey.SCHEMA, isObject, 'an object (rules, strict)'),
+        rules,
+        indexes: ((indexes) => (indexes === null ? null : checkedIndexes(shown, indexes)))(checked(CollectionKey.INDEXES, Array.isArray, 'an array')),
+        expire: ((expire) => (expire === null ? null : checkedExpire(shown, expire)))(checked(CollectionKey.EXPIRE, isObject, 'an object (field, after_ms, existing)')),
+        aggregates: ((aggregates) => (aggregates === null ? null : checkedAggregates(shown, aggregates)))(checked(CollectionKey.AGGREGATES, Array.isArray, 'an array')),
+        id: ((id) => (id === null ? null : checkedId(shown, id)))(checked(CollectionKey.ID, isObject, 'an object (derive)')),
+    };
+}
+
+/**
+ * The collections the project declares, by name: [{name, file, schema, rules, indexes, aggregates, id}].
+ * None when the directory is not there. Throws a CliError for a file that is not
+ * valid, naming it.
+ */
+export function declaredCollections(config, cwd) {
+    const dir = resolve(cwd, collectionsDirOf(config));
+    if (!existsSync(dir)) return [];
+    let names;
+    try {
+        names = readdirSync(dir).filter((f) => extname(f) === '.json').sort();
+    } catch (err) {
+        throw new CliError(`${collectionsDirOf(config)}: ${err.message}`, { code: ErrorCode.INVALID_ARGUMENT });
+    }
+    return names.map((fileName) => parseCollection(config, cwd, fileName));
+}
+
+/**
+ * The app's security rules, from rules_file and the collections' own rules:
+ * null when there are none, else {body (the security.rules.set input), sources
+ * (the files they came from), merged (more than rules_file is in the body)}.
+ * Throws a CliError naming both files when a collection has rules in both.
+ */
+export function declaredRules(config, cwd, collections) {
+    const base = config.rules_file ? readJsonFile(resolve(cwd, config.rules_file), 'rules_file', config.rules_file) : null;
+    const own = collections.filter((c) => c.rules !== null);
+    if (own.length === 0) return base === null ? null : { body: base, sources: [config.rules_file], merged: false };
+    if (base !== null && !isObject(base)) throw new CliError(`rules_file ${config.rules_file} must be a JSON object with a rules array`, { code: ErrorCode.INVALID_ARGUMENT });
+    const entries = base?.rules ?? [];
+    if (!Array.isArray(entries)) throw new CliError(`rules_file ${config.rules_file}: rules must be an array`, { code: ErrorCode.INVALID_ARGUMENT });
+    for (const collection of own) {
+        if (entries.some((entry) => entry?.collection === collection.name)) {
+            throw new CliError(`rules for collection "${collection.name}" are defined in both ${config.rules_file} and ${collection.file}; keep them in one`,
+                { code: ErrorCode.INVALID_ARGUMENT });
+        }
+    }
+    return {
+        body: { ...(base ?? {}), rules: [...entries, ...own.map((c) => ({ collection: c.name, ...c.rules }))] },
+        sources: [...(config.rules_file ? [config.rules_file] : []), ...own.map((c) => c.file)],
+        merged: true,
+    };
+}
+
+/** The rules as a blob of a version, made in memory (like a precompressed sidecar): {path, data, size, hash}. */
+export function rulesBlob(rules) {
+    const data = Buffer.from(JSON.stringify(rules.body));
+    return { path: rules.sources.join(' + '), data, size: data.length, hash: sha256(data) };
+}
+
+/** " (2 indexes)" for a collection's line in deploy's output; nothing for none. */
+export const indexCountLabel = (count) => (count > 0 ? ` (${count} index${count === 1 ? '' : 'es'})` : '');
+
+/** " (2 aggregates)" for a collection's line in deploy's output; nothing for none. */
+export const aggregateCountLabel = (count) => (count > 0 ? ` (${count} aggregate${count === 1 ? '' : 's'})` : '');
+
+/** " (expiry)" for a collection's line in deploy's output when it declares one; nothing otherwise. */
+export const expireLabel = (declared) => (declared ? ' (expiry)' : '');
+
+/**
+ * The calls that bring the server to the declared collections (pure): per collection
+ * `collection.create` (a collection that exists is not an error: see applyCollections),
+ * when it declares `id.derive` one `index.create` of kind id_derive (UM-3d; idempotent for
+ * its own value), when it declares a schema `schema.update`, per declared index one
+ * `index.create` of kind collection (idempotent: if_not_exists), then when it declares
+ * `expire` one `index.create` of kind expiry (idempotent: if_not_exists), then per declared
+ * aggregate one `index.create` of kind aggregate (idempotent on its own: a redeclaration of
+ * the same name and shape changes nothing), in file order.
+ */
+export function collectionSteps(collections) {
+    return collections.flatMap((c) => [
+        { collection: c.name, operation: 'collection.create', input: { name: c.name }, existsIsFine: true },
+        ...(c.id ? [{ collection: c.name, operation: 'index.create', existsIsFine: false, input: idCreateInput(c.name, c.id) }] : []),
+        ...(c.schema === null ? [] : [{ collection: c.name, operation: 'schema.update', input: { collection: c.name, ...c.schema }, existsIsFine: false }]),
+        ...(c.indexes ?? []).map(indexEntry).map((entry) => ({
+            collection: c.name, operation: 'index.create', existsIsFine: false,
+            input: indexCreateInput(c.name, entry),
+        })),
+        ...(c.expire ? [{ collection: c.name, operation: 'index.create', existsIsFine: false, input: expireCreateInput(c.name, c.expire) }] : []),
+        ...(c.aggregates ?? []).map((entry) => ({
+            collection: c.name, operation: 'index.create', existsIsFine: false,
+            input: aggregateCreateInput(c.name, entry),
+        })),
+    ]);
+}
+
+
+/**
+ * Applies `collections` with `invoke(name, input)` (throws a CliError on a failed
+ * call): the collection made when missing, its id.derive, schema, indexes, expire and
+ * aggregates declared. Idempotent. `done` receives what was applied as it goes, so a
+ * failure keeps its progress. Returns {applied: [{collection, created, id, schema, indexes,
+ * expire, aggregates}], warnings} (indexes, aggregates: how many of each were declared;
+ * id, expire: whether each was declared; warnings: none, every declaration is applied).
+ */
+export async function applyCollections(invoke, collections, done = []) {
+    for (const c of collections) {
+        const entry = { collection: c.name, created: true, id: false, schema: false, indexes: 0, aggregates: 0, expire: false };
+        for (const step of collectionSteps([c])) {
+            try {
+                await invoke(step.operation, step.input);
+            } catch (err) {
+                if (!(step.existsIsFine && err instanceof CliError && err.status === EXISTS)) throw err;
+                entry.created = false;
+            }
+            if (step.operation === 'schema.update') entry.schema = true;
+            if (step.operation === 'index.create' && step.input.kind === 'collection') entry.indexes += 1;
+            if (step.operation === 'index.create' && step.input.kind === 'id_derive') entry.id = true;
+            if (step.operation === 'index.create' && step.input.kind === 'expiry') entry.expire = true;
+            if (step.operation === 'index.create' && step.input.kind === 'aggregate') entry.aggregates += 1;
+        }
+        done.push(entry);
+    }
+    return { applied: done, warnings: [] };
+}
+
+/**
+ * The names of the collections the server has, every page of collection.list read with `invoke(name, input)`
+ * (throws a CliError on a failed call): a Set.
+ */
+export async function serverCollectionNames(invoke) {
+    const names = new Set();
+    let cursor;
+    for (;;) {
+        const page = await invoke('collection.list', cursor === undefined ? {} : { [PAGE_CURSOR_INPUT]: cursor });
+        for (const entry of page?.collections ?? []) names.add(entry.name);
+        const next = page?.[PAGE_NEXT_CURSOR_FIELD];
+        if (typeof next !== 'string' || next === '') return names;
+        if (next === cursor) throw new CliError(`collection.list answered the cursor it was given (${next}) as its next one`, { code: ErrorCode.UNKNOWN });
+        cursor = next;
+    }
+}
+
+/** The declared `collections` the server does not have, by name (a collection deleted on the server): [name]. */
+export async function missingCollections(invoke, collections) {
+    if (collections.length === 0) return [];
+    const present = await serverCollectionNames(invoke);
+    return collections.map((c) => c.name).filter((name) => !present.has(name));
+}
+
+/**
+ * The version member for `collections` (pure): [{name, id?, schema?, indexes?, expire?, aggregates?}]. The
+ * rules go with the version's rules, so they are not here.
+ */
+export function collectionsMember(collections) {
+    return collections.map((c) => ({
+        name: c.name,
+        ...(c.id ? { id: c.id } : {}),
+        ...(c.schema === null ? {} : { schema: c.schema }),
+        ...(c.indexes === null ? {} : { indexes: c.indexes }),
+        ...(c.expire ? { expire: c.expire } : {}),
+        ...(c.aggregates === null ? {} : { aggregates: c.aggregates }),
+    }));
+}
+
+/**
+ * The version member for `collections` (VERSION-COLLECTIONS): the calls substrio deploy makes
+ * (collectionSteps), which the server runs in order on the target when the version goes live, as
+ * [{operation, collection?, body}]. Each input is split by the route the call would take, as
+ * callOperation splits it: schema.update's collection is its route's, named apart; the rest is the body.
+ */
+export function versionCollectionSteps(ops, collections) {
+    return collectionSteps(collections).map(({ operation, input }) => {
+        const route = pickRoute(findOperation(ops, operation), input);
+        const { body } = buildRequest(route, input);
+        return route.params.includes('collection') ? { operation, collection: input.collection, body } : { operation, body };
+    });
+}
+
+/**
+ * Checks that the server's version.create takes the collections member (its input
+ * schema in the indexed operations `ops` declares it). Throws a CliError saying
+ * so when it does not: a version that silently left them out would put an app live
+ * without its collections.
+ */
+export function checkVersionCarriesCollections(ops) {
+    const schema = findOperation(ops, 'version.create').find((r) => r.bodySchema)?.bodySchema;
+    if (Object.hasOwn(schema?.properties ?? {}, COLLECTIONS_MEMBER)) return;
+    throw new CliError(`this server's versions cannot carry the declared collections yet (version.create has no "${COLLECTIONS_MEMBER}" member)`, {
+        code: ErrorCode.INVALID_ARGUMENT,
+        hint: 'substrio deploy applies them to the app directly; substrio dev applies them to the local server',
+    });
+}
+
+/**
+ * The server's declarations for `collections` that the file no longer names: an index of kind collection
+ * (plain, unique, keyword, elements or compound), an aggregate, or an expiry. Read with `invoke(name, input)`
+ * (index.list, kind collection then aggregate then expiry, once per collection; throws a CliError on a failed
+ * call). Returns [{collection, kind ('index', 'aggregate' or 'expiry'), label, drop}] (drop: the index.drop
+ * input pruneDrift sends). A declaration the file still names, in the same shape, is matched and never
+ * reported (fileIndexIdentity, serverIndexIdentity); a field's undeclared adaptive key is not an answer of
+ * index.list, so it is never reported or pruned.
+ */
+export async function collectionDrift(invoke, collections) {
+    const drift = [];
+    for (const c of collections) {
+        const fileIndexIds = new Set((c.indexes ?? []).map(indexEntry).map(fileIndexIdentity));
+        const indexPage = await invoke('index.list', { kind: 'collection', collection: c.name });
+        for (const entry of indexPage?.indexes ?? []) {
+            if (fileIndexIds.has(serverIndexIdentity(entry))) continue;
+            drift.push({ collection: c.name, kind: 'index', label: serverIndexLabel(entry), drop: serverIndexDropInput(c.name, entry) });
+        }
+        const fileAggNames = new Set((c.aggregates ?? []).map((a) => a.name));
+        const aggPage = await invoke('index.list', { kind: 'aggregate', collection: c.name });
+        for (const entry of aggPage?.indexes ?? []) {
+            if (fileAggNames.has(entry.name)) continue;
+            drift.push({ collection: c.name, kind: 'aggregate', label: entry.name, drop: { kind: 'aggregate', collection: c.name, name: entry.name } });
+        }
+        if (!c.expire) {
+            const expiry = await invoke('index.list', { kind: 'expiry', collection: c.name });
+            if (expiry?.declared) {
+                drift.push({ collection: c.name, kind: 'expiry', label: expiry.field, drop: { kind: 'expiry', collection: c.name, name: 'expiry' } });
+            }
+        }
+    }
+    return drift;
+}
+
+/**
+ * Drops every entry of `drift` with `invoke(name, input)` (index.drop; throws a CliError on a failed call, the
+ * run stops there, as applyCollections does). Returns `drift`, for the caller to report what was pruned.
+ */
+export async function pruneDrift(invoke, drift) {
+    for (const d of drift) await invoke('index.drop', d.drop);
+    return drift;
+}