@byline/core 6.7.2 → 6.9.0

package/dist/storage/storage-restore.js

@@ -8,11 +8,16 @@
 import { RESERVED_FIELD_NAMES } from '../config/validate-collections.js';
 import { ERR_DATABASE } from '../lib/errors.js';
 import { getLogger } from '../lib/logger.js';
+// Shared across duplicate server module graphs, just like the logger registry.
+const loggedVersionsKey = Symbol.for('byline.unavailable-block-versions');
+const processState = globalThis;
+processState[loggedVersionsKey] ??= new Set();
+const loggedVersions = processState[loggedVersionsKey];
 /**
  * Main entrypoint for restoring a document's field data from flattened form
  * back into the original nested structure.
  *
- * Always returns `{ data, warnings }`. Warnings carry per-row reasons (e.g.
+ * Always returns `{ data, warnings, unavailableBlocks }`. Warnings carry per-row reasons (e.g.
  * "Invalid block index 'undefined'") that are typically the result of a
  * collection schema change leaving orphan rows behind. Callers decide what
  * to do with them — strict reads reject any non-empty `warnings`, lenient
@@ -21,9 +26,10 @@ import { getLogger } from '../lib/logger.js';
  * @param fields - The field definitions for the collection.
  * @param flattenedData - The flattened field data to restore.
  */
-export const restoreFieldSetData = (fields, flattenedData, resolveLocale) => {
+export const restoreFieldSetData = (fields, flattenedData, resolveLocale, identity) => {
     const result = {};
     const warnings = [];
+    const unavailableByArray = new Map();
     for (const item of flattenedData) {
         // biome-ignore lint/style/noNonNullAssertion: TODO: put in a proper check here?
         const fieldName = item.field_path[0];
@@ -38,7 +44,7 @@ export const restoreFieldSetData = (fields, flattenedData, resolveLocale) => {
         if (!field) {
         }
         else {
-            result[fieldName] = restoreFieldData(field, result[fieldName], item, 1, warnings, resolveLocale);
+            result[fieldName] = restoreFieldData(field, result[fieldName], item, 1, warnings, unavailableByArray, resolveLocale);
         }
     }
     // Array/blocks/group/hasMany-relation fixup — a required list-shaped field
@@ -54,17 +60,52 @@ export const restoreFieldSetData = (fields, flattenedData, resolveLocale) => {
             result[field.name] = result[field.name] || {};
         }
     }
-    return { data: result, warnings };
+    const unavailableBlocks = [];
+    // Finish only after all rows (including identity metadata) have been read.
+    // Walk parents before children so nested report paths use compacted indices.
+    const finish = (fields, value, prefix = '') => {
+        for (const field of fields) {
+            const path = prefix ? `${prefix}.${field.name}` : field.name;
+            const child = value?.[field.name];
+            if (field.type === 'group' && child)
+                finish(field.fields, child, path);
+            if ((field.type === 'blocks' || field.type === 'array') && Array.isArray(child)) {
+                for (const issue of [...(unavailableByArray.get(child)?.values() ?? [])].sort((a, b) => a.storedIndex - b.storedIndex))
+                    unavailableBlocks.push({ fieldPath: path, ...issue });
+                const compacted = child.filter((item) => item != null);
+                value[field.name] = compacted;
+                compacted.forEach((item, index) => {
+                    const itemFields = field.type === 'array'
+                        ? field.fields
+                        : field.blocks.find((block) => block.blockType === item._type)?.fields;
+                    if (itemFields)
+                        finish(itemFields, item, `${path}[${index}]`);
+                });
+            }
+        }
+    };
+    finish(fields, result);
+    if (identity && unavailableBlocks.length > 0) {
+        const key = JSON.stringify([identity.collectionPath, identity.documentId, identity.versionId]);
+        if (!loggedVersions.has(key)) {
+            loggedVersions.add(key);
+            getLogger().warn({
+                ...identity,
+                blockTypes: [...new Set(unavailableBlocks.map((issue) => issue.blockType))],
+            }, 'Document contains unavailable blocks; saving content will remove them');
+        }
+    }
+    return { data: result, warnings, unavailableBlocks };
 };
-const restoreFieldData = (field, target, data, pathIndex, warnings, resolveLocale) => {
+const restoreFieldData = (field, target, data, pathIndex, warnings, unavailableByArray, resolveLocale) => {
     if (field.type === 'group') {
-        return restoreGroupFieldData(field, target, data, pathIndex, warnings, resolveLocale);
+        return restoreGroupFieldData(field, target, data, pathIndex, warnings, unavailableByArray, resolveLocale);
     }
     else if (field.type === 'array') {
-        return restoreArrayFieldData(field, target, data, pathIndex, warnings, resolveLocale);
+        return restoreArrayFieldData(field, target, data, pathIndex, warnings, unavailableByArray, resolveLocale);
     }
     else if (field.type === 'blocks') {
-        return restoreBlocksFieldData(field, target, data, pathIndex, warnings, resolveLocale);
+        return restoreBlocksFieldData(field, target, data, pathIndex, warnings, unavailableByArray, resolveLocale);
     }
     else if (field.type === 'relation' && field.hasMany) {
         return restoreMultiRelationFieldData(target, data, pathIndex, warnings);
@@ -95,7 +136,7 @@ const restoreFieldData = (field, target, data, pathIndex, warnings, resolveLocal
     }
     return target;
 };
-const restoreGroupFieldData = (field, target, data, pathIndex, warnings, resolveLocale) => {
+const restoreGroupFieldData = (field, target, data, pathIndex, warnings, unavailableByArray, resolveLocale) => {
     if (pathIndex >= data.field_path.length) {
         warnings.push(`Path ended unexpectedly while restoring group: ${data.field_path.join('.')}`);
         return target;
@@ -108,10 +149,10 @@ const restoreGroupFieldData = (field, target, data, pathIndex, warnings, resolve
         return target;
     }
     target = target || {};
-    target[fieldName] = restoreFieldData(subField, target[fieldName], data, pathIndex + 1, warnings, resolveLocale);
+    target[fieldName] = restoreFieldData(subField, target[fieldName], data, pathIndex + 1, warnings, unavailableByArray, resolveLocale);
     return target;
 };
-const restoreArrayFieldData = (field, target, data, pathIndex, warnings, resolveLocale) => {
+const restoreArrayFieldData = (field, target, data, pathIndex, warnings, unavailableByArray, resolveLocale) => {
     if (pathIndex >= data.field_path.length) {
         warnings.push(`Path ended unexpectedly while restoring array: ${data.field_path.join('.')}`);
         return target;
@@ -140,10 +181,10 @@ const restoreArrayFieldData = (field, target, data, pathIndex, warnings, resolve
         // Sub-field was removed from the schema; silently skip orphaned rows.
         return target;
     }
-    target[arrayIndex][fieldName] = restoreFieldData(subField, target[arrayIndex][fieldName], data, pathIndex + 2, warnings, resolveLocale);
+    target[arrayIndex][fieldName] = restoreFieldData(subField, target[arrayIndex][fieldName], data, pathIndex + 2, warnings, unavailableByArray, resolveLocale);
     return target;
 };
-const restoreBlocksFieldData = (field, target, data, pathIndex, warnings, resolveLocale) => {
+const restoreBlocksFieldData = (field, target, data, pathIndex, warnings, unavailableByArray, resolveLocale) => {
     const arrayIndex = Number.parseInt(data.field_path[pathIndex] ?? '', 10);
     if (Number.isNaN(arrayIndex) || arrayIndex < 0) {
         warnings.push(`Invalid block index '${data.field_path[pathIndex]}' in path ${data.field_path.join('.')}`);
@@ -156,7 +197,17 @@ const restoreBlocksFieldData = (field, target, data, pathIndex, warnings, resolv
     }
     const block = field.blocks.find((f) => f.blockType === blockType);
     if (block == null) {
-        // Block type was removed from the schema; silently skip orphaned rows.
+        target = target || [];
+        let issues = unavailableByArray.get(target);
+        if (!issues) {
+            issues = new Map();
+            unavailableByArray.set(target, issues);
+        }
+        const issue = issues.get(arrayIndex) ?? { storedIndex: arrayIndex, blockType };
+        if (data.field_type === 'meta' && pathIndex + 2 === data.field_path.length) {
+            issue.id = data.item_id;
+        }
+        issues.set(arrayIndex, issue);
         return target;
     }
     target = target || [];
@@ -178,7 +229,7 @@ const restoreBlocksFieldData = (field, target, data, pathIndex, warnings, resolv
         // Sub-field was removed from the schema; silently skip orphaned rows.
         return target;
     }
-    target[arrayIndex][fieldName] = restoreFieldData(subField, target[arrayIndex][fieldName], data, pathIndex + 3, warnings, resolveLocale);
+    target[arrayIndex][fieldName] = restoreFieldData(subField, target[arrayIndex][fieldName], data, pathIndex + 3, warnings, unavailableByArray, resolveLocale);
     return target;
 };
 const restoreMultiRelationFieldData = (target, data, pathIndex, warnings) => {

package/dist/storage/unavailable-blocks.test.node.d.ts

@@ -0,0 +1,8 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+export {};

package/dist/storage/unavailable-blocks.test.node.js

@@ -0,0 +1,132 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+import { describe, expect, it, vi } from 'vitest';
+import { getLogger } from '../lib/logger.js';
+import { applyPatches } from '../patches/apply-patches.js';
+import { validateDocumentFields } from '../validation/document-fields.js';
+import { flattenFieldSetData } from './storage-flatten.js';
+import { restoreFieldSetData } from './storage-restore.js';
+const logger = vi.hoisted(() => ({ warn: vi.fn() }));
+vi.mock('../lib/logger.js', () => ({ getLogger: () => logger }));
+const live = { blockType: 'live', fields: [{ name: 'text', type: 'text' }] };
+const retired = { ...live, blockType: 'retired' };
+const blocks = (optional = false) => ({
+    name: 'content',
+    type: 'blocks',
+    optional,
+    blocks: [live, retired],
+});
+const item = (_type, i) => ({ _id: `item-${i}`, _type, text: `${_type}-${i}` });
+describe('unavailable blocks', () => {
+    for (const optional of [false, true]) {
+        for (const types of [
+            ['retired', 'live'],
+            ['live', 'retired', 'live'],
+            ['live', 'retired'],
+            ['retired'],
+            ['retired', 'retired', 'live'],
+        ]) {
+            it(`compacts ${types.join(',')} (optional=${optional}) with an independent report`, () => {
+                const before = [blocks(optional)];
+                const after = [{ ...blocks(optional), blocks: [live] }];
+                const original = { content: types.map(item) };
+                const rows = flattenFieldSetData(before, original, 'en');
+                const restored = restoreFieldSetData(after, rows, 'en');
+                expect(restored.warnings).toEqual([]);
+                expect(restored.unavailableBlocks).toEqual(types.flatMap((type, storedIndex) => type === 'retired'
+                    ? [{ fieldPath: 'content', storedIndex, blockType: type, id: `item-${storedIndex}` }]
+                    : []));
+                expect(restored.data.content).toEqual(original.content.filter((item) => item._type === 'live'));
+                for (const data of [restored.data, JSON.parse(JSON.stringify(restored.data))]) {
+                    expect(validateDocumentFields(after, data)).toEqual([]);
+                    const saved = restoreFieldSetData(after, flattenFieldSetData(after, data, 'en'), 'en');
+                    expect(saved.unavailableBlocks).toEqual([]);
+                    expect(saved.data.content ?? []).toEqual(restored.data.content);
+                }
+            });
+        }
+    }
+    it('reports groups, arrays and nested blocks using compacted parent indices', () => {
+        const before = [
+            { name: 'group', type: 'group', fields: [blocks()] },
+            { name: 'sections', type: 'array', fields: [blocks()] },
+            {
+                name: 'outer',
+                type: 'blocks',
+                blocks: [retired, { blockType: 'container', fields: [blocks()] }],
+            },
+        ];
+        const after = [
+            { name: 'group', type: 'group', fields: [{ ...blocks(), blocks: [live] }] },
+            { name: 'sections', type: 'array', fields: [{ ...blocks(), blocks: [live] }] },
+            {
+                name: 'outer',
+                type: 'blocks',
+                blocks: [{ blockType: 'container', fields: [{ ...blocks(), blocks: [live] }] }],
+            },
+        ];
+        const content = [item('retired', 0), item('live', 1)];
+        const rows = flattenFieldSetData(before, {
+            group: { content },
+            sections: [{ content }],
+            outer: [item('retired', 0), { _type: 'container', content }],
+        }, 'en');
+        const result = restoreFieldSetData(after, rows.reverse(), 'en');
+        expect(result.unavailableBlocks.map((issue) => issue.fieldPath)).toEqual([
+            'group.content',
+            'sections[0].content',
+            'outer',
+            'outer[0].content',
+        ]);
+        expect(result.warnings).toEqual([]);
+        expect(result.data.outer[0].content).toEqual([item('live', 1)]);
+    });
+    it('applies a positional patch to a surviving block after the missing card', () => {
+        const before = [blocks()];
+        const after = [{ ...blocks(), blocks: [live] }];
+        const source = { content: ['retired', 'live', 'live'].map(item) };
+        const { data } = restoreFieldSetData(after, flattenFieldSetData(before, source, 'en'));
+        const definition = {
+            path: 'pages',
+            labels: { singular: 'Page', plural: 'Pages' },
+            fields: after,
+        };
+        const patched = applyPatches(definition, data, [
+            { kind: 'field.set', path: 'content[1].text', value: 'edited' },
+        ]);
+        expect(patched.errors).toEqual([]);
+        const saved = restoreFieldSetData(after, flattenFieldSetData(after, patched.doc, 'en'));
+        expect(saved.data.content.map((item) => [item._id, item.text])).toEqual([
+            ['item-1', 'live-1'],
+            ['item-2', 'edited'],
+        ]);
+    });
+    it.each([new Array(1), [null], [undefined], [42], [[]]].map((content) => ({ content })))('rejects malformed items without a destructure crash: $content', ({ content }) => {
+        expect(validateDocumentFields([blocks()], { content })).not.toEqual([]);
+        expect(() => flattenFieldSetData([blocks()], { content }, 'en')).toThrow(/Block item is missing or must be an object/);
+    });
+    it('logs once per document version without adding a strict-read warning', () => {
+        const spy = vi.spyOn(getLogger(), 'warn').mockImplementation(() => { });
+        try {
+            const rows = flattenFieldSetData([blocks()], { content: [item('retired', 0)] }, 'en');
+            const identity = { collectionPath: 'pages', documentId: crypto.randomUUID(), versionId: 'v1' };
+            for (let i = 0; i < 2; i++)
+                expect(restoreFieldSetData([{ ...blocks(), blocks: [live] }], rows, 'en', identity).warnings).toEqual([]);
+            expect(spy).toHaveBeenCalledTimes(1);
+            expect(spy).toHaveBeenCalledWith({ ...identity, blockTypes: ['retired'] }, expect.any(String));
+            restoreFieldSetData([{ ...blocks(), blocks: [live] }], rows, 'en', {
+                ...identity,
+                versionId: 'v2',
+            });
+            expect(spy).toHaveBeenCalledTimes(2);
+        }
+        finally {
+            spy.mockRestore();
+        }
+    });
+});

package/dist/validation/document-fields.js

@@ -119,13 +119,16 @@ export function validateDocumentFields(fields, data, options = {}) {
                     if (field.validation?.maxLength != null && value.length > field.validation.maxLength)
                         add(path, `${label} allows at most ${field.validation.maxLength} items`);
                 }
-                value.forEach((item, index) => {
+                for (let index = 0; index < value.length; index++) {
+                    const item = value[index];
                     const itemPath = record(item) && typeof item._id === 'string' && /^[\w-]+$/.test(item._id)
                         ? `${path}[id=${item._id}]`
                         : `${path}[${index}]`;
                     if (!record(item)) {
-                        add(itemPath, 'Item must be an object');
-                        return;
+                        add(itemPath, field.type === 'blocks'
+                            ? 'Block item is missing or must be an object'
+                            : 'Item must be an object');
+                        continue;
                     }
                     if (field.type === 'array')
                         visit(field.fields, item, itemPath);
@@ -136,7 +139,7 @@ export function validateDocumentFields(fields, data, options = {}) {
                         else
                             visit(block.fields, item, itemPath);
                     }
-                });
+                }
             }
         }
         else {

package/package.json

@@ -2,9 +2,9 @@
   "name": "@byline/core",
   "private": false,
   "license": "MPL-2.0",
-  "version": "6.7.2",
+  "version": "6.9.0",
   "engines": {
-    "node": ">=20.9.0"
+    "node": ">=22.22.2"
   },
   "description": "Byline CMS core package",
   "keywords": [
@@ -82,7 +82,7 @@
     "dist"
   ],
   "dependencies": {
-    "@byline/auth": "6.7.2",
+    "@byline/auth": "6.9.0",
     "dotenv": "^18.0.8",
     "pino": "^10.4.0",
     "sharp": "^0.35.5",