ghost 6.55.0 → 6.57.0

package/core/server/services/members-custom-fields/values-service.js

@@ -14,41 +14,34 @@ const queries_1 = require("./queries");
 const storage_1 = require("./storage");
 const FIELDS_TABLE = 'members_custom_fields';
 const VALUES_TABLE = 'members_custom_field_values';
-// Matches the `members_custom_fields.key` column (schema.js), so no key a site
-// could actually have minted is ever refused by it.
+/** Matches the `members_custom_fields.key` column, so no key a site could hold is refused. */
 const MAX_KEY_LENGTH = 191;
-// Values arrive keyed by field key. Each value stays `unknown` here: it is
-// validated by its own field type's schema, which isn't known until the key is
-// resolved to a definition.
-const ValuesInput = zod_1.z.record(zod_1.z.string().max(MAX_KEY_LENGTH), zod_1.z.unknown());
 /**
- * The value half of member custom fields: reading and writing what a member holds
- * for each defined field. It belongs to the member aggregate — a value is data
- * about a member — while field definitions belong to the site's settings; that
- * aggregate boundary, not a technical layer, is why this is a separate service.
+ * Rows per insert statement, bounded by knex rather than by either database. SQLite takes
+ * a multi-row `VALUES` perfectly well; knex's SQLite dialect emits that form only for a
+ * single row and compiles anything longer into `INSERT ... SELECT ? UNION ALL SELECT ?`,
+ * which SQLite refuses past 500 terms.
  *
- * It owns the `members_custom_field_values` table and reads `members_custom_fields`
- * directly for the reference data it needs (which fields are active, and how they
- * validate), rather than calling the definitions service — the same way the other
- * knex services here talk to the database.
+ * Open as knex#721 since 2015, with an approved but unmerged fix in knex#5780. This can
+ * go when that lands; until then the alternative is hand-written upsert SQL per engine.
+ */
+const UPSERT_CHUNK = 400;
+// Values stay `unknown`: each is validated by its own field type, which is not known
+// until the key is resolved to a definition.
+const ValuesInput = zod_1.z.record(zod_1.z.string().max(MAX_KEY_LENGTH), zod_1.z.unknown());
+/**
+ * What a member holds for each defined field. Separate from the definitions service
+ * because a value belongs to the member and a definition belongs to the site's settings,
+ * which are different aggregates rather than different layers.
  */
 class CustomFieldValuesService {
     knex;
-    /**
-     * @private
-     * A getter rather than a number: the ceiling is an operator setting that can
-     * change between requests.
-     */
+    /** A getter, not a number: the ceiling is an operator setting that changes between requests. */
     getMaxDefinitions;
     constructor({ knex, getMaxDefinitions }) {
         this.knex = knex;
         this.getMaxDefinitions = getMaxDefinitions;
     }
-    /**
-     * The active fields for a set of keys, keyed by key, for resolving a write.
-     * Scoped to the keys the caller is actually writing rather than every active
-     * field, since a publisher may have defined many and an edit touches few.
-     */
     async activeFieldsByKey(keys) {
         if (keys.length === 0) {
             return new Map();
@@ -59,60 +52,36 @@ class CustomFieldValuesService {
         return new Map(fields.map(field => [field.key, field]));
     }
     /**
-     * Members' values for every active field, keyed by member id, then by field
-     * key. A member with no values is absent from the outer map; a field a member
-     * has no value for is absent from the inner one — "not set" is the absence of
-     * a key, told apart from a value by not being there. Archived fields are
-     * excluded to match the definitions `browse`: their values stay in the
-     * database, they just stop being addressable.
+     * Members' values, keyed by member id then field key; anything absent is unset.
      *
-     * One query for the whole set, so a member list can't turn into an N+1.
-     *
-     * Reading is best-effort per value: a stored value whose type has since left
-     * the catalog, or that won't decode, is omitted and logged rather than failing
-     * the whole read. Stored data outlives the catalog, so one stale row must not
-     * take down a member's payload.
+     * Archived fields are excluded to match the definitions browse: their values stay in
+     * the database and stop being addressable. A row that will not parse is dropped and
+     * logged rather than failing the read, so one stale row cannot take down a member.
      */
     async getValuesForMembers(memberIds) {
-        const byMember = new Map();
         if (memberIds.length === 0) {
-            return byMember;
+            return new Map();
         }
         const rows = await this.knex(VALUES_TABLE)
             .join(FIELDS_TABLE, `${VALUES_TABLE}.custom_field_id`, `${FIELDS_TABLE}.id`)
             .whereIn(`${VALUES_TABLE}.member_id`, memberIds)
-            // Read join, so the status filter is qualified and lives inline — it
-            // shares the invariant's vocabulary with queries.ts, not its builder.
             .where(`${FIELDS_TABLE}.status`, schema_1.FIELD_STATUS.active)
             .orderBy(`${FIELDS_TABLE}.created_at`, 'asc')
             .orderBy(`${FIELDS_TABLE}.id`, 'asc')
-            .select(`${VALUES_TABLE}.member_id`, `${FIELDS_TABLE}.key`, `${FIELDS_TABLE}.type`, `${VALUES_TABLE}.value_text`, `${VALUES_TABLE}.value_json`);
+            .orderBy(`${VALUES_TABLE}.path`, 'asc')
+            .select(`${VALUES_TABLE}.member_id`, `${FIELDS_TABLE}.key`, `${FIELDS_TABLE}.type`, `${VALUES_TABLE}.path`, `${VALUES_TABLE}.value_text`);
+        const leaves = [];
         for (const row of rows) {
             try {
-                const { member_id: memberId, key, type, ...stored } = schema_1.DbCustomFieldValueWithField.parse(row);
-                const codec = (0, storage_1.storageCodecFor)(type);
-                const raw = stored[codec.column];
-                // The column the type routes to is the only one that can hold this
-                // field's value; a null there means the row is not carrying one.
-                if (raw === null) {
-                    continue;
-                }
-                const values = byMember.get(memberId) ?? {};
-                values[key] = codec.decode(raw);
-                byMember.set(memberId, values);
+                leaves.push(schema_1.DbCustomFieldLeaf.parse(row));
             }
             catch (err) {
-                logging_1.default.warn(`Skipping undecodable custom field value (field '${row.key}', type '${row.type}'): ${err instanceof Error ? err.message : String(err)}`);
+                logging_1.default.warn(`Skipping unreadable custom field value (field '${row.key}', path '${row.path}'): ${err instanceof Error ? err.message : String(err)}`);
             }
         }
-        return byMember;
+        return (0, storage_1.valuesFromLeaves)(leaves);
     }
-    /**
-     * @private
-     * Input as the values object it claims to be, rejecting anything that isn't
-     * one. Shared by every caller so they cannot disagree on what a values object
-     * is, or on the error when it isn't one.
-     */
+    /** Shared by every caller so they cannot disagree on what a values object is. */
     parseValues(input) {
         const parsed = ValuesInput.safeParse(input);
         if (!parsed.success) {
@@ -121,12 +90,8 @@ class CustomFieldValuesService {
         return parsed.data;
     }
     /**
-     * Whether input names any values. An absent key names none; anything present
-     * that isn't a values object throws, with the same error resolving it would
-     * have raised.
-     *
-     * Answers the shape question alone, with no catalog lookup, so it can be asked
-     * before a write is known to be permitted.
+     * Whether input names any values. Asks the shape question alone, with no catalog
+     * lookup, so it can be asked before a write is known to be permitted.
      */
     namesValues(input) {
         if (input === undefined) {
@@ -135,19 +100,15 @@ class CustomFieldValuesService {
         return Object.keys(this.parseValues(input)).length > 0;
     }
     /**
-     * Resolve input into the writes it implies, rejecting anything invalid, and
-     * writing nothing. Returned so a caller can validate before it commits to a
-     * change it would have to unwind (the member edit and the importer both validate
-     * up front, before opening a transaction), then apply the same plan without
-     * re-resolving or re-validating.
+     * Resolve input into the writes it implies, writing nothing. Returned so a caller can
+     * validate before opening a transaction it would otherwise have to unwind, then apply
+     * the same plan without re-resolving it.
      */
     async planWrite(input) {
         const values = this.parseValues(input);
         const keys = Object.keys(values);
-        // A write cannot name more fields than the site may define, so the
-        // definitions ceiling bounds it. This also holds the lookup below within the
-        // database driver's bound-parameter limit, which one key per parameter would
-        // otherwise exceed.
+        // Bounded by the definitions ceiling, which also holds the lookup below inside
+        // the driver's bound-parameter limit.
         const maxKeys = this.getMaxDefinitions();
         if (keys.length > maxKeys) {
             throw new errors_1.default.ValidationError({
@@ -160,27 +121,27 @@ class CustomFieldValuesService {
         for (const [key, raw] of Object.entries(values)) {
             const field = byKey.get(key);
             if (!field) {
-                // Unknown (or archived) key. Rejected rather than ignored: a typo
-                // that silently drops a value is worse than a failed save.
+                // Unknown or archived. Refused rather than ignored: a typo that silently
+                // drops what somebody typed is worse than a save that fails. The catalog
+                // applies the same rule to the parts of a composite value.
                 throw new errors_1.default.ValidationError({ message: `Unknown custom field: ${key}`, property: `custom_fields.${key}` });
             }
-            // `null` clears any field. An empty string clears a text-backed field —
-            // an emptied input — but for any other storage type '' is a value to
-            // validate, not a clear, so nothing is coerced: a string sent to a
-            // (future) non-text field fails validation rather than silently deleting.
-            if (raw === null || (raw === '' && custom_field_types_1.FIELD_TYPES[field.type].storageType === 'text')) {
+            // `null` clears any field, and `''` clears one with no parts. For a value
+            // with parts `''` names nothing, so it is left to fail validation rather than
+            // being read as a silent delete.
+            if (raw === null || (raw === '' && (0, custom_field_types_1.subFieldsOf)(field.type) === null)) {
                 writes.push({ field });
                 continue;
             }
-            // The field type owns what a valid value is — core just runs it. For a
-            // composite the issue path points at the offending sub-field, so the
-            // property reads `custom_fields.home_address.postal_code`.
+            // Message only, no `context`: the API error handler moves a message into
+            // `context` when `context` is empty and prepends it when it is not, so
+            // anything added here reaches the client glued to the front of the reason.
+            // Which field failed rides in `property`.
             const value = custom_field_types_1.FIELD_TYPES[field.type].value.safeParse(raw);
             if (!value.success) {
                 const issue = value.error.issues[0];
                 throw new errors_1.default.ValidationError({
-                    message: `Invalid value for custom field '${field.name}'.`,
-                    context: issue.message,
+                    message: issue.message,
                     property: [`custom_fields.${key}`, ...issue.path].join('.')
                 });
             }
@@ -191,35 +152,69 @@ class CustomFieldValuesService {
     /**
      * Apply a plan from `planWrite`.
      *
-     * Merge, not replace: only the fields in the plan are touched, so a caller
-     * that doesn't know about a field can't erase it.
+     * A write touches the paths it names and nothing else, at every level: naming a path
+     * with an empty value clears that part, naming the field with `null` clears all of
+     * them, and saying nothing about a path leaves it alone. There is no whole-value
+     * replace, so a caller that does not know about a field cannot erase it.
      *
-     * Always transactional, so a mid-batch failure rolls the batch back. Passed an
-     * executor it joins that transaction -- the importer passes its per-member one, so a
-     * failed value write takes the member with it; passed nothing it opens its own.
+     * Always transactional. Given an executor it joins that transaction, so the importer's
+     * failed value write takes its member with it; given none it opens its own.
      */
-    async applyWrite(memberId, writes, executor = this.knex) {
+    async applyWrite(memberId, writes, { executor = this.knex } = {}) {
         if (writes.length === 0) {
             return;
         }
         const apply = async (trx) => {
+            // Built first, then sent as whole statements: a handful per member rather
+            // than one per part, under one timestamp, because a write happened once
+            // however many rows record it.
+            const now = new Date();
+            const clearedFields = [];
+            const clearedPaths = [];
+            const rows = [];
             for (const { field, value } of writes) {
-                const target = { member_id: memberId, custom_field_id: field.id };
                 if (value === undefined) {
-                    await trx(VALUES_TABLE).where(target).del();
+                    clearedFields.push(field.id);
                     continue;
                 }
-                const valueColumns = (0, storage_1.storageColumnsFor)(field.type, value);
+                const { set, cleared } = (0, storage_1.leavesToWrite)(value);
+                if (cleared.length > 0) {
+                    clearedPaths.push({ fieldId: field.id, paths: cleared });
+                }
+                rows.push(...set.map(leaf => ({
+                    id: new bson_objectid_1.default().toHexString(),
+                    member_id: memberId,
+                    custom_field_id: field.id,
+                    path: leaf.path,
+                    value_text: leaf.value_text,
+                    created_at: now,
+                    updated_at: now
+                })));
+            }
+            if (clearedFields.length > 0) {
+                await trx(VALUES_TABLE).where('member_id', memberId).whereIn('custom_field_id', clearedFields).del();
+            }
+            if (clearedPaths.length > 0) {
+                // One statement with a group per field, rather than a statement per field.
+                await trx(VALUES_TABLE).where('member_id', memberId).where((builder) => {
+                    for (const { fieldId, paths } of clearedPaths) {
+                        builder.orWhere(pair => pair.where('custom_field_id', fieldId).whereIn('path', paths));
+                    }
+                }).del();
+            }
+            for (let from = 0; from < rows.length; from += UPSERT_CHUNK) {
+                // Typed as the plain row because `merge` takes its columns as `keyof` the
+                // builder's record, which for a composite table registration is the scope
+                // names rather than the columns.
                 await trx(VALUES_TABLE)
-                    .insert({ id: new bson_objectid_1.default().toHexString(), ...target, ...valueColumns, created_at: new Date() })
-                    // The unique index on (member_id, custom_field_id) is
-                    // what makes this an upsert rather than a read-then-write race.
-                    .onConflict(['member_id', 'custom_field_id'])
-                    .merge({ ...valueColumns, updated_at: new Date() });
+                    .insert(rows.slice(from, from + UPSERT_CHUNK))
+                    // Naming the columns rather than giving values takes each from the row
+                    // that lost the conflict, so every part updates to its own value.
+                    .onConflict(['member_id', 'custom_field_id', 'path'])
+                    .merge(['value_text', 'updated_at']);
             }
         };
-        // isTransaction is knex's marker for a transactor: join an existing transaction
-        // rather than nesting a savepoint under it, otherwise open one.
+        // knex's marker for a transactor: join it rather than nesting a savepoint under it.
         if (executor.isTransaction) {
             await apply(executor);
         }