ghost 5.114.1 → 5.115.1

package/core/built/admin/index.html

@@ -8,7 +8,7 @@
     <title>Ghost Admin</title>
 
     
-<meta name="ghost-admin/config/environment" content="%7B%22modulePrefix%22%3A%22ghost-admin%22%2C%22environment%22%3A%22production%22%2C%22cdnUrl%22%3A%22%22%2C%22editorUrl%22%3A%22%22%2C%22rootURL%22%3A%22%22%2C%22locationType%22%3A%22trailing-hash%22%2C%22EmberENV%22%3A%7B%22FEATURES%22%3A%7B%7D%2C%22EXTEND_PROTOTYPES%22%3A%7B%22Date%22%3Afalse%2C%22Array%22%3Atrue%2C%22String%22%3Atrue%2C%22Function%22%3Afalse%7D%2C%22_APPLICATION_TEMPLATE_WRAPPER%22%3Afalse%2C%22_JQUERY_INTEGRATION%22%3Atrue%2C%22_TEMPLATE_ONLY_GLIMMER_COMPONENTS%22%3Atrue%7D%2C%22APP%22%3A%7B%22version%22%3A%225.114%22%2C%22name%22%3A%22ghost-admin%22%7D%2C%22ember-simple-auth%22%3A%7B%7D%2C%22%40sentry%2Fember%22%3A%7B%22disablePerformance%22%3Atrue%2C%22sentry%22%3A%7B%7D%7D%2C%22ember-cli-mirage%22%3A%7B%22usingProxy%22%3Afalse%2C%22useDefaultPassthroughs%22%3Atrue%7D%2C%22exportApplicationGlobal%22%3Afalse%2C%22ember-load%22%3A%7B%22loadingIndicatorClass%22%3A%22ember-load-indicator%22%7D%2C%22editorFilename%22%3A%22koenig-lexical.umd.js%22%2C%22editorHash%22%3A%22685db18f10%22%2C%22adminXDemoFilename%22%3A%22admin-x-demo.js%22%2C%22adminXDemoHash%22%3A%2215e68d0314%22%2C%22adminXSettingsFilename%22%3A%22admin-x-settings.js%22%2C%22adminXSettingsHash%22%3A%22a1e3245cf0%22%2C%22adminXActivitypubFilename%22%3A%22admin-x-activitypub.js%22%2C%22adminXActivitypubHash%22%3A%22c297b6fa8f%22%2C%22postsFilename%22%3A%22posts.js%22%2C%22postsHash%22%3A%22cc9549f09d%22%2C%22adminXActivitypubCustomUrl%22%3A%22https%3A%2F%2Fcdn.jsdelivr.net%2Fghost%2Fadmin-x-activitypub%400%2Fdist%2Fadmin-x-activitypub.js%22%7D" />
+<meta name="ghost-admin/config/environment" content="%7B%22modulePrefix%22%3A%22ghost-admin%22%2C%22environment%22%3A%22production%22%2C%22cdnUrl%22%3A%22%22%2C%22editorUrl%22%3A%22%22%2C%22rootURL%22%3A%22%22%2C%22locationType%22%3A%22trailing-hash%22%2C%22EmberENV%22%3A%7B%22FEATURES%22%3A%7B%7D%2C%22EXTEND_PROTOTYPES%22%3A%7B%22Date%22%3Afalse%2C%22Array%22%3Atrue%2C%22String%22%3Atrue%2C%22Function%22%3Afalse%7D%2C%22_APPLICATION_TEMPLATE_WRAPPER%22%3Afalse%2C%22_JQUERY_INTEGRATION%22%3Atrue%2C%22_TEMPLATE_ONLY_GLIMMER_COMPONENTS%22%3Atrue%7D%2C%22APP%22%3A%7B%22version%22%3A%225.115%22%2C%22name%22%3A%22ghost-admin%22%7D%2C%22ember-simple-auth%22%3A%7B%7D%2C%22%40sentry%2Fember%22%3A%7B%22disablePerformance%22%3Atrue%2C%22sentry%22%3A%7B%7D%7D%2C%22ember-cli-mirage%22%3A%7B%22usingProxy%22%3Afalse%2C%22useDefaultPassthroughs%22%3Atrue%7D%2C%22exportApplicationGlobal%22%3Afalse%2C%22ember-load%22%3A%7B%22loadingIndicatorClass%22%3A%22ember-load-indicator%22%7D%2C%22editorFilename%22%3A%22koenig-lexical.umd.js%22%2C%22editorHash%22%3A%22685db18f10%22%2C%22adminXDemoFilename%22%3A%22admin-x-demo.js%22%2C%22adminXDemoHash%22%3A%2253d9d1eb76%22%2C%22adminXSettingsFilename%22%3A%22admin-x-settings.js%22%2C%22adminXSettingsHash%22%3A%223812b50e35%22%2C%22adminXActivitypubFilename%22%3A%22admin-x-activitypub.js%22%2C%22adminXActivitypubHash%22%3A%220667582c6d%22%2C%22postsFilename%22%3A%22posts.js%22%2C%22postsHash%22%3A%22cc9549f09d%22%2C%22statsFilename%22%3A%22stats.js%22%2C%22statsHash%22%3A%2256566f9e00%22%2C%22adminXActivitypubCustomUrl%22%3A%22https%3A%2F%2Fcdn.jsdelivr.net%2Fghost%2Fadmin-x-activitypub%400%2Fdist%2Fadmin-x-activitypub.js%22%7D" />
 
     <meta name="HandheldFriendly" content="True" />
     <meta name="MobileOptimized" content="320" />
@@ -37,7 +37,7 @@
     </style>
 
     <link integrity="" rel="stylesheet" href="assets/vendor-0ede59da8efb5e28fa929557f7ff7154.css">
-    <link integrity="" rel="stylesheet" href="assets/ghost-c2a7c4a1b76550c4219adb2ed4124ce0.css" title="light">
+    <link integrity="" rel="stylesheet" href="assets/ghost-327b17ea23cb8c89bd7e6a51e18e8506.css" title="light">
 
     
 </head>
@@ -58,7 +58,7 @@
 
     <script src="assets/vendor-68a4aa424a179a90f5bbc2b750def576.js"></script>
 <script src="assets/chunk.874.461cb3cf5b6b36915f8c.js"></script>
-<script src="assets/chunk.524.85c5b32bd46b91c147b9.js"></script>
-    <script src="assets/ghost-c563138cc2c0767bf6eefc9a2587eaa4.js"></script>
+<script src="assets/chunk.524.2439684964c164c598ab.js"></script>
+    <script src="assets/ghost-df7b9558260aa27d18b195ee895b487d.js"></script>
 </body>
 </html>

package/core/frontend/helpers/ghost_head.js

@@ -151,6 +151,8 @@ function getWebmentionDiscoveryLink() {
 }
 
 function getTinybirdTrackerScript(dataRoot) {
+    const src = urlUtils.getSubdir() + '/public/ghost-stats.js';
+
     const endpoint = config.get('tinybird:tracker:endpoint');
     const token = config.get('tinybird:tracker:token');
     const datasource = config.get('tinybird:tracker:datasource');
@@ -162,7 +164,7 @@ function getTinybirdTrackerScript(dataRoot) {
         member_status: dataRoot.member?.status
     }, (value, key) => `tb_${key}="${value}"`).join(' ');
 
-    return `<script defer src="/public/ghost-stats.js" data-stringify-payload="false" ${datasource ? `data-datasource="${datasource}"` : ''} data-storage="localStorage" data-host="${endpoint}" data-token="${token}" ${tbParams}></script>`;
+    return `<script defer src="${src}" data-stringify-payload="false" ${datasource ? `data-datasource="${datasource}"` : ''} data-storage="localStorage" data-host="${endpoint}" data-token="${token}" ${tbParams}></script>`;
 }
 
 function getHCaptchaScript() {

package/core/frontend/src/cards/css/cta.css

@@ -95,7 +95,7 @@
 .kg-cta-content {
     display: flex;
     padding: 1.5em;
-    gap: 1.5em 1.5em 1.7em;
+    gap: 1.5em;
 }
 
 @media (max-width: 600px) {

package/core/server/api/endpoints/slugs.js

@@ -21,7 +21,8 @@ const controller = {
         },
         options: [
             'include',
-            'type'
+            'type',
+            'id'
         ],
         data: [
             'name'
@@ -32,6 +33,9 @@ const controller = {
                 type: {
                     required: true,
                     values: Object.keys(allowedTypes)
+                },
+                id: {
+                    required: false
                 }
             },
             data: {
@@ -41,7 +45,7 @@ const controller = {
             }
         },
         query(frame) {
-            return models.Base.Model.generateSlug(allowedTypes[frame.options.type], frame.data.name, {status: 'all'})
+            return models.Base.Model.generateSlug(allowedTypes[frame.options.type], frame.data.name, {status: 'all', modelId: frame.options.id})
                 .then((slug) => {
                     if (!slug) {
                         return Promise.reject(new errors.InternalServerError({

package/core/server/data/importer/import-manager.js

@@ -16,7 +16,7 @@ const RevueHandler = require('./handlers/revue');
 const JSONHandler = require('./handlers/json');
 const MarkdownHandler = require('./handlers/markdown');
 const ContentFileImporter = require('./importers/ContentFileImporter');
-const RevueImporter = require('@tryghost/importer-revue');
+const RevueImporter = require('./importers/importer-revue');
 const DataImporter = require('./importers/data');
 const urlUtils = require('../../../shared/url-utils');
 const {GhostMailer} = require('../../services/mail');
@@ -225,7 +225,7 @@ class ImportManager {
 
         try {
             await extract(filePath, tmpDir);
-            
+
             // Set permissions for all extracted files
             const files = glob.sync('**/*', {cwd: tmpDir, nodir: true});
             await Promise.all(files.map(file => fs.chmod(path.join(tmpDir, file), 0o644)));

package/core/server/data/importer/importers/importer-revue.js

@@ -0,0 +1,128 @@
+const debug = require('@tryghost/debug')('importer:revue');
+const {slugify} = require('@tryghost/string');
+const papaparse = require('papaparse');
+const _ = require('lodash');
+
+const JSONToHTML = require('./json-to-html');
+
+/**
+ * Build posts out of the issue and item data
+ *
+ * @param {Object} revueData
+ * @return {Array}
+ */
+const fetchPostsFromData = (revueData) => {
+    const itemData = JSON.parse(revueData.items);
+    const issueData = papaparse.parse(revueData.issues, {
+        header: true,
+        skipEmptyLines: true,
+        transform(value, header) {
+            if (header === 'id') {
+                return parseInt(value);
+            }
+            return value;
+        }
+    });
+
+    const posts = [];
+
+    issueData.data.forEach((postMeta) => {
+        // Convert issues to posts
+        if (!postMeta.subject) {
+            return;
+        }
+
+        const revuePostID = postMeta.id;
+        let postHTML = JSONToHTML.cleanCsvHTML(postMeta.description);
+
+        const postItems = _.filter(itemData, {issue_id: revuePostID});
+        const sortedPostItems = _.sortBy(postItems, o => o.order);
+
+        if (postItems) {
+            const convertedItems = JSONToHTML.itemsToHtml(sortedPostItems);
+            postHTML = `${postHTML}${convertedItems}`;
+        }
+
+        const postDate = JSONToHTML.getPostDate(postMeta);
+        const postSlug = slugify(postMeta.subject).slice(0, 190);
+
+        posts.push({
+            comment_id: revuePostID,
+            title: postMeta.subject,
+            slug: postSlug,
+            status: JSONToHTML.getPostStatus(postMeta),
+            visibility: 'public',
+            created_at: postDate,
+            published_at: postDate,
+            updated_at: postDate,
+            html: postHTML
+        });
+    });
+
+    return posts;
+};
+
+/**
+ *
+ * @param {*} revueData
+ */
+const buildSubscriberList = (revueData) => {
+    const subscribers = [];
+
+    const subscriberData = papaparse.parse(revueData.subscribers, {
+        header: true,
+        skipEmptyLines: true
+    });
+
+    subscriberData.data.forEach((subscriber) => {
+        subscribers.push({
+            email: subscriber.email,
+            name: `${subscriber.first_name} ${subscriber.last_name}`.trim(),
+            created_at: subscriber.created_at,
+            subscribed: true
+        });
+    });
+
+    return subscribers;
+};
+
+const RevueImporter = {
+    type: 'revue',
+    preProcess: function (importData) {
+        debug('preProcess');
+        importData.preProcessedByRevue = true;
+
+        // TODO: this should really be in doImport
+        // No posts to prePprocess, return early
+        if (!importData?.revue?.revue?.issues) {
+            return importData;
+        }
+
+        // This processed data goes to the data importer
+        importData.data = {
+            meta: {version: '5.0.0'},
+            data: {}
+        };
+
+        importData.data.data.posts = this.importPosts(importData.revue.revue);
+
+        // No subscribers to import, we're done
+        if (!importData?.revue?.revue?.subscribers) {
+            return importData;
+        }
+
+        importData.data.data.revue_subscribers = this.importSubscribers(importData.revue.revue);
+
+        return importData;
+    },
+    doImport: function (importData) {
+        debug('doImport');
+
+        return importData;
+    },
+
+    importPosts: fetchPostsFromData,
+    importSubscribers: buildSubscriberList
+};
+
+module.exports = RevueImporter;

package/core/server/data/importer/importers/json-to-html.js

@@ -0,0 +1,107 @@
+const SimpleDom = require('simple-dom');
+const serializer = new SimpleDom.HTMLSerializer(SimpleDom.voidMap);
+const imageCard = require('@tryghost/kg-default-cards/lib/cards/image.js');
+const embedCard = require('@tryghost/kg-default-cards/lib/cards/embed.js');
+
+// Take the array of items for a specific post and return the converted HTML
+const itemsToHtml = (items) => {
+    let itemHTMLChunks = [];
+    items.forEach((item) => {
+        let type = item.item_type;
+
+        if (type === 'header') {
+            itemHTMLChunks.push(`<h3>${item.title}</h3>`);
+        } else if (type === 'text') {
+            itemHTMLChunks.push(item.description); // THis is basic text HTML with <p>, <b>, <a>, etc (no media)
+        } else if (type === 'image') {
+            // We have 2 values to work with here. `image` is smaller and most suitable, and `original_image_url` is the full-res that would need to be resized
+            // - item.image (https://s3.amazonaws.com/revue/items/images/019/005/542/web/anita-austvika-C-JUrfmYqcw-unsplash.jpg?1667924147)
+            // - item.original_image_url (https://s3.amazonaws.com/revue/items/images/019/005/542/original/anita-austvika-C-JUrfmYqcw-unsplash.jpg?1667924147)
+            let cardOpts = {
+                env: {dom: new SimpleDom.Document()},
+                payload: {
+                    src: item.image,
+                    caption: item.description
+                }
+            };
+
+            itemHTMLChunks.push(serializer.serialize(imageCard.render(cardOpts)));
+        } else if (type === 'link') {
+            // This could be a bookmark, or it could be a paragraph of text with a linked header, there's no way to tell
+            // The safest option here is to output an image with text under it
+            let cardOpts = {
+                env: {dom: new SimpleDom.Document()},
+                payload: {
+                    src: item.image,
+                    caption: item.title,
+                    href: item.url
+                }
+            };
+            itemHTMLChunks.push(serializer.serialize(imageCard.render(cardOpts)));
+
+            let linkHTML = `<h4><a href="${item.url}">${item.title}</a></h4>${item.description}`;
+            itemHTMLChunks.push(linkHTML);
+        } else if (type === 'tweet') {
+            // Should this be an oEmbed call? Probably.
+            itemHTMLChunks.push(`<figure class="kg-card kg-embed-card">
+                <blockquote class="twitter-tweet"><a href="${item.url}"></a></blockquote>
+                <script async src="https://platform.twitter.com/widgets.js" charset="utf-8"></script>
+                </figure>`);
+        } else if (type === 'video') {
+            const isLongYouTube = /youtube.com/.test(item.url);
+            const isShortYouTube = /youtu.be/.test(item.url);
+            const isVimeo = /vimeo.com/.test(item.url);
+            let videoHTML = '';
+            if (isLongYouTube) {
+                let videoID = item.url.replace(/https?:\/\/(?:www\.)?youtube\.com\/watch\?v=([a-zA-Z0-9_-]*)/gi, '$1');
+                videoHTML = `<iframe width="200" height="113" src="https://www.youtube.com/embed/${videoID}?feature=oembed" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>`;
+            } else if (isShortYouTube) {
+                let videoID = item.url.replace(/https?:\/\/(?:www\.)?youtu\.be\/([a-zA-Z0-9_-]*)/gi, '$1');
+                videoHTML = `<iframe width="200" height="113" src="https://www.youtube.com/embed/${videoID}?feature=oembed" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>`;
+            } else if (isVimeo) {
+                let videoID = item.url.replace(/https?:\/\/(?:www\.)?vimeo\.com\/([0-9]+)/gi, '$1');
+                videoHTML = `<iframe src="https://player.vimeo.com/video/${videoID}" width="200" height="113" frameborder="0" allow="autoplay; fullscreen; picture-in-picture" allowfullscreen></iframe>`;
+            }
+            let cardOpts = {
+                env: {dom: new SimpleDom.Document()},
+                payload: {
+                    html: videoHTML,
+                    caption: item.description
+                }
+            };
+
+            itemHTMLChunks.push(serializer.serialize(embedCard.render(cardOpts)));
+        }
+    });
+    return itemHTMLChunks.join('\n');
+};
+
+const getPostDate = (data) => {
+    const isPublished = (data.sent_at) ? true : false; // This is how we determine is a post is published or not
+    const postDate = (isPublished) ? new Date(data.sent_at) : new Date();
+
+    return postDate.toISOString();
+};
+
+const getPostStatus = (data) => {
+    const isPublished = (data.sent_at) ? true : false; // This is how we determine is a post is published or not
+    return (isPublished) ? 'published' : 'draft';
+};
+
+const cleanCsvHTML = (data) => {
+    // Blockquotes need to have some sort of wrapping elements around all contents
+    // Wrap all content in <p> tags. The HTML to Mobiledoc parse can handle duplicate <p> tags.
+    data = data.replace(/<blockquote.*?>(.*?)<\/blockquote>/gm, '<blockquote><p>$1</p></blockquote>');
+
+    // These exports have a lot of <p><br></p> that we don't want
+    data = data.replace(/<p><br><\/p>/gm, '');
+
+    return data;
+};
+
+module.exports = {
+    itemsToHtml,
+    getPostDate,
+    getPostStatus,
+    cleanCsvHTML
+};

package/core/server/data/migrations/utils/tables.js

@@ -5,8 +5,7 @@ const {createIrreversibleMigration, createNonTransactionalMigration} = require('
 /**
  * Creates a migrations which will add a new table from schema.js to the database
  * @param {string} name - table name
- * @param {Object} tableSpec - copy of table schema definition as defined in schema.js at the moment of writing the migration,
- * this parameter MUST be present, otherwise @daniellockyer will hunt you down
+ * @param {Object} tableSpec - copy of table schema definition as defined in schema.js at the moment of writing the migration, this parameter MUST be present
  *
  * @returns {Object} migration object returning config/up/down properties
  */
@@ -60,8 +59,7 @@ function dropTables(names) {
 /**
  * Creates a migration which will drop an existing table and then re-add a new table based on provided spec
  * @param {string} name - table name
- * @param {Object} tableSpec - copy of table schema definition as defined in schema.js at the moment of writing the migration,
- * this parameter MUST be present, otherwise @daniellockyer will hunt you down
+ * @param {Object} tableSpec - copy of table schema definition as defined in schema.js at the moment of writing the migration, this parameter MUST be present
  *
  * @returns {Object} migration object returning config/up/down properties
  */

package/core/server/data/migrations/versions/5.115/2025-03-24-07-19-27-add-identity-read-permission-to-administrators.js

@@ -0,0 +1,6 @@
+const {addPermissionToRole} = require('../../utils');
+
+module.exports = addPermissionToRole({
+    permission: 'Read identities',
+    role: 'Administrator'
+});

package/core/server/data/schema/fixtures/fixtures.json

@@ -875,7 +875,8 @@
                     "link": "all",
                     "mention": "browse",
                     "collection": "all",
-                    "recommendation": "all"
+                    "recommendation": "all",
+                    "identity": "read"
                 },
                 "DB Backup Integration": {
                     "db": "all"

package/core/server/lib/bootstrap-socket.js

@@ -0,0 +1,87 @@
+const logging = require('@tryghost/logging');
+
+module.exports.connectAndSend = (socketAddress, message) => {
+    // Very basic guard against bad calls
+    if (!socketAddress || !socketAddress.host || !socketAddress.port || !logging || !logging.info || !logging.warn || !message) {
+        return Promise.resolve();
+    }
+
+    const net = require('net');
+    const client = new net.Socket();
+
+    return new Promise((resolve) => {
+        const connect = (options = {}) => {
+            let wasResolved = false;
+
+            const waitTimeout = setTimeout(() => {
+                logging.info('Bootstrap socket timed out.');
+
+                if (!client.destroyed) {
+                    client.destroy();
+                }
+
+                if (wasResolved) {
+                    return;
+                }
+
+                wasResolved = true;
+                resolve();
+            }, 1000 * 5);
+
+            client.connect(socketAddress.port, socketAddress.host, () => {
+                if (waitTimeout) {
+                    clearTimeout(waitTimeout);
+                }
+
+                client.write(JSON.stringify(message));
+
+                if (wasResolved) {
+                    return;
+                }
+
+                wasResolved = true;
+                resolve();
+            });
+
+            client.on('close', () => {
+                logging.info('Bootstrap client was closed.');
+
+                if (waitTimeout) {
+                    clearTimeout(waitTimeout);
+                }
+            });
+
+            client.on('error', (err) => {
+                logging.warn(`Can't connect to the bootstrap socket (${socketAddress.host} ${socketAddress.port}) ${err.code}.`);
+
+                client.removeAllListeners();
+
+                if (waitTimeout) {
+                    clearTimeout(waitTimeout);
+                }
+
+                if (options.tries < 3) {
+                    logging.warn(`Tries: ${options.tries}`);
+
+                    // retry
+                    logging.warn('Retrying...');
+
+                    options.tries = options.tries + 1;
+                    const retryTimeout = setTimeout(() => {
+                        clearTimeout(retryTimeout);
+                        connect(options);
+                    }, 150);
+                } else {
+                    if (wasResolved) {
+                        return;
+                    }
+
+                    wasResolved = true;
+                    resolve();
+                }
+            });
+        };
+
+        connect({tries: 0});
+    });
+};

package/core/server/lib/package-json/index.js

@@ -0,0 +1 @@
+module.exports = require('./package-json');

package/core/server/lib/package-json/package-json.js

@@ -0,0 +1,160 @@
+/*
+ * # Package Utils
+ *
+ * Ghost has support for several different types of sub-packages:
+ * - Themes: have always been packages, but we're going to lean more heavily on npm & package.json in future
+ * - Adapters: replace fundamental pieces like storage, will become npm modules
+ *
+ * These utils facilitate loading, reading, managing etc, packages from the file system.
+ *
+ */
+const _ = require('lodash');
+const fs = require('fs-extra');
+const join = require('path').join;
+const errors = require('@tryghost/errors');
+const parse = require('./parse');
+
+const notAPackageRegex = /^\.|_messages|README.md|node_modules|bower_components/i;
+const packageJSONPath = 'package.json';
+
+/**
+ * @typedef {object} PackageList
+ * @typedef {object} Package
+ */
+
+/**
+ * Recursively read directory and find the packages in it
+ *
+ * @param {string} absolutePath
+ * @param {string} packageName
+ * @returns {Promise<Package>}
+ */
+async function processPackage(absolutePath, packageName) {
+    const pkg = {
+        name: packageName,
+        path: absolutePath
+    };
+
+    try {
+        const packageJSON = await parse(join(absolutePath, packageJSONPath));
+        pkg['package.json'] = packageJSON;
+    } catch (err) {
+        // ignore invalid package.json for now,
+        // because Ghost does not rely/use them at the moment
+        // in the future, this .catch() will need to be removed,
+        // so that error is thrown on invalid json syntax
+        pkg['package.json'] = null;
+    }
+
+    return pkg;
+}
+
+/**
+ * ### Filter Packages
+ * Normalizes packages read by readPackages so that the themes module can use them.
+ * Iterates over each package and return an array of objects which are simplified representations of the package
+ * with 3 properties:
+ *
+ * @typedef {object} SimplePackage
+ * @prop {string} name    - the package name
+ * @prop {object|boolean} package - contents of the package.json or false if there isn't one
+ * @prop {boolean} active  - set to true if this package is active
+ *
+ * This data structure is used for listings of packages provided over the API and as such
+ * deliberately combines multiple sources of information in order to be efficient.
+ *
+ * TODO: simplify the package.json representation to contain only fields we use
+ *
+ * @param   {PackageList}       packages    object made up of packages keyed by name as returned by readPackages
+ * @param   {array|string}      [active]    optional set of names of packages that are active
+ * @returns {Array<SimplePackage>}          array of objects with useful info about themes
+ */
+function filter(packages, active) {
+    // turn active into an array if it isn't one, so this function can deal with lists and one-offs
+    if (!Array.isArray(active)) {
+        active = [active];
+    }
+
+    return _.reduce(packages, function (result, pkg, key) {
+        let item = {};
+        if (!key.match(notAPackageRegex)) {
+            item = {
+                name: key,
+                package: pkg['package.json'] || false,
+                active: _.indexOf(active, key) !== -1
+            };
+
+            result.push(item);
+        }
+
+        return result;
+    }, []);
+}
+
+/**
+ * @param {string} packagePath
+ * @param {string} packageName
+ * @returns {Promise<Package>}
+ */
+async function readPackage(packagePath, packageName) {
+    const absolutePath = join(packagePath, packageName);
+
+    try {
+        const stat = await fs.stat(absolutePath);
+        if (!stat.isDirectory()) {
+            return {};
+        }
+
+        const pkg = await processPackage(absolutePath, packageName);
+        const res = {};
+        res[packageName] = pkg;
+        return res;
+    } catch (err) {
+        return Promise.reject(new errors.NotFoundError({
+            message: 'Package not found',
+            err: err,
+            help: 'path: ' + packagePath,
+            context: 'name: ' + packageName
+        }));
+    }
+}
+
+/**
+ * @param {string} packagePath
+ * @returns {Promise<PackageList>}
+ */
+async function readPackages(packagePath) {
+    const files = await fs.promises.readdir(packagePath, {withFileTypes: true});
+    const packages = await Promise.all(files.map(async (file) => {
+        // Filter out things which are not packages by regex
+        if (file.name.match(notAPackageRegex)) {
+            return false;
+        }
+
+        if (file.isSymbolicLink()) {
+            try {
+                const packageFileOrig = await fs.stat(join(packagePath, file.name));
+                return packageFileOrig.isDirectory();
+            } catch (err) {
+                // if there's an error reading the symlink, we should just return false
+                return false;
+            }
+        }
+
+        // Check the remaining items to ensure they are a directory
+        return file.isDirectory();
+    }))
+        .then(results => files.filter((_v, index) => results[index]))
+        .then(packageFiles => Promise.all(packageFiles.map((packageFile) => {
+            const absolutePath = join(packagePath, packageFile.name);
+            return processPackage(absolutePath, packageFile.name);
+        })));
+
+    return _.keyBy(packages, 'name');
+}
+
+module.exports = {
+    filter,
+    readPackage,
+    readPackages
+};