@datawheel/bespoke 0.1.9 → 0.1.11

package/dist/server.js

@@ -12,7 +12,7 @@ import { strip, titleCase } from 'd3plus-text';
 import axios from 'axios';
 import slugifyFn from 'slugify';
 import lunr from 'lunr';
-import lunrStemmer from 'lunr-languages/lunr.stemmer.support.js';
+import lunrStemmer from 'lunr-languages/lunr.stemmer.support';
 import Base58 from 'base58';
 import FlickrSDK from 'flickr-sdk';
 import sharp from 'sharp';
@@ -1748,19 +1748,26 @@ function ingestMembersFactory(db) {
         attributes: [],
         required: false,
         right: true,
-        where: { dimension_id, slug }
+        where: { dimension_id, slug, id: { [Op.not]: id } }
       },
       group: "search.slug"
     }).then((arr) => Object.fromEntries(arr.map((item) => {
       const count = item.getDataValue("slugCount");
       return [item.slug, Number.parseInt(`${count}`, 10)];
     })));
+    const toIngestSlugs = [];
     const searchMembers = members.map((member) => {
-      const slug2 = slugify_default(member[labelKey]);
       const memberId = member[idKey];
+      const slug2 = slugify_default(member[labelKey]);
+      const hasDuplicateSlug = allSlugsFromOtherMembers[slug2] || toIngestSlugs.includes(slug2);
+      if (hasDuplicateSlug) {
+        console.log("Duplicate slug!", slug2, "replaced by", `${slug2}-${memberId}`);
+      }
+      const validateSlug = hasDuplicateSlug ? `${slug2}-${memberId}` : slug2;
+      toIngestSlugs.push(slug2);
       return {
         id: memberId,
-        slug: allSlugsFromOtherMembers[slug2] > 1 ? `${slug2}-${memberId}` : slug2,
+        slug: validateSlug,
         variant_id: id,
         dimension_id,
         report_id,
@@ -1780,30 +1787,39 @@ function readMemberFactory(db) {
   const { search: Search } = db;
   return readMember;
   async function readMember(params) {
-    const {
-      mode,
-      ids,
-      slugs,
-      all,
-      locale
-    } = params;
+    const { all, locale, mode } = params;
     const whereClause = {
       content_id: []
     };
-    if (mode === "ids") {
-      whereClause.content_id = ids;
+    if (mode === "content_ids") {
+      whereClause.content_id = params.content_ids;
+    } else if (mode === "ids") {
+      if (all) {
+        throw new BackendError(404, "Search by ids doesn't support locale all. Pick one locale.");
+      }
+      const entities = await Search.findAll({
+        attributes: ["content_id"],
+        where: {
+          id: params.ids
+        },
+        include: [{
+          association: "contentByLocale",
+          where: { locale }
+        }]
+      });
+      whereClause.content_id = entities.map((item) => item.content_id);
     } else if (mode === "slugs") {
       const entities = await Search.findAll({
         attributes: ["content_id"],
         where: {
-          slug: slugs.map((item) => item.memberSlug)
+          slug: params.slugs.map((item) => item.memberSlug)
         },
         include: {
           association: "variant",
           attributes: ["id", "name", "slug"],
           required: false,
           where: {
-            slug: slugs.map((item) => item.variantSlug)
+            slug: params.slugs.map((item) => item.variantSlug)
           }
         }
       });
@@ -1835,48 +1851,57 @@ function readMemberFactory(db) {
         where: contentWhereClause
       }]
     });
-    if (memberSearchResults.length !== (mode === "ids" ? ids.length : slugs.length)) {
-      throw new BackendError(404, "One of the members was not found.");
+    if (memberSearchResults.length !== params[mode].length) {
+      throw new BackendError(404, "One or more members were not found.");
     }
     memberSearchResults.sort(
       (a, b) => whereClause.content_id.indexOf(a.content_id) - whereClause.content_id.indexOf(b.content_id)
     );
     const normalizedResults = memberSearchResults.map((item) => {
-      const newItem = {
-        ...item.toJSON(),
+      const generalResult = {
+        id: item.id,
+        slug: item.slug,
+        visible: item.visible,
+        zvalue: item.zvalue,
+        content_id: item.content_id,
+        report_id: item.report_id,
         report_name: item.report.name,
+        dimension_id: item.dimension_id,
         dimension_name: item.dimension.name,
+        variant_id: item.variant_id,
         variant_name: item.variant.name,
         variant_slug: item.variant.slug
       };
       if (!all) {
         const localizedItem = item.getContent(locale);
-        newItem.locale = localizedItem.locale;
-        newItem.name = localizedItem.name;
-        newItem.attributes = localizedItem.attributes;
-        newItem.keywords = localizedItem.keywords;
-        delete newItem.contentByLocale;
-        if (item.image) {
-          const localizedImage = item.image.getContent(locale);
-          newItem.image = {
-            ...item.image,
-            meta: localizedImage.meta
-          };
-          delete newItem.image.contentByLocale;
-        }
+        const localizedImage = item.image?.getContent(locale);
+        const image = localizedImage && {
+          id: item.image.id,
+          author: item.image.author,
+          url: item.image.url,
+          license: item.image.license,
+          meta: localizedImage.meta
+        };
+        const localizedResult = {
+          ...generalResult,
+          locale: localizedItem.locale,
+          name: localizedItem.name,
+          attributes: localizedItem.attributes,
+          keywords: localizedItem.keywords,
+          image: image || null,
+          image_id: item.image_id
+        };
+        return localizedResult;
+      } else {
+        const contentByLocale = item.contentByLocale;
+        generalResult.contentByLocale = contentByLocale;
+        generalResult.image = item.image;
+        generalResult.image_id = item.image_id;
       }
-      delete newItem.report;
-      delete newItem.dimension;
-      delete newItem.variant;
-      return newItem;
+      return generalResult;
     });
     return {
-      meta: {
-        locale: stripHTML(locale),
-        mode,
-        ids,
-        slugs
-      },
+      meta: params,
       results: normalizedResults
     };
   }
@@ -1956,16 +1981,21 @@ var verbose3 = getLogging_default();
 var { locales } = getLocales_default();
 var enabledLocales = locales.filter((d) => whitelist.includes(d));
 var search;
+var initializing = false;
 var getSearchIndexByLocale = async (db, forceRegenerate = false) => {
   if (forceRegenerate) {
     if (verbose3)
       console.log("REGENERATING SEARCH INDEX");
     search = void 0;
+    initializing = false;
   }
   if (search)
     return search;
   try {
-    search = await newSearchIndex(db);
+    if (!initializing) {
+      initializing = true;
+      search = await newSearchIndex(db);
+    }
   } catch (e) {
     throw new Error(`
       Could not initialize Lunr search engine.
@@ -1983,7 +2013,7 @@ async function newSearchIndex(db) {
   });
   const setups = enabledLocales.map(async (locale) => {
     if (!lunr[locale] && whitelist.includes(locale) && locale !== "en") {
-      const lunrLang = await import(`lunr-languages/lunr.${locale}`);
+      const lunrLang = (await import(`lunr-languages/lunr.${locale}`)).default;
       lunrLang(lunr);
     }
     return [locale, lunr(function() {
@@ -2443,14 +2473,20 @@ function dbReportFactory(db) {
       sections,
       ...reportData
     } = data;
-    const entity = await Report.findByPk(id, {
-      include: reportQuery.include,
-      rejectOnEmpty: true
-    });
-    entity.set(reportData);
-    await entity.save();
-    await entity.reload();
-    return entity.toJSON();
+    try {
+      console.log("Update Report ID", id, reportData);
+      const entity = await Report.findByPk(id, {
+        include: reportQuery.include,
+        rejectOnEmpty: true
+      });
+      entity.set(reportData);
+      await entity.save();
+      await entity.reload();
+      return entity.toJSON();
+    } catch (error) {
+      console.log("Report id:", id, error);
+      throw new BackendError(500, "Update Report error");
+    }
   }
   async function deleteReport(id) {
     const entity = await Report.findByPk(id, { rejectOnEmpty: true });
@@ -2816,15 +2852,13 @@ function endpointMemberFactory(operations) {
   } = operations;
   return [
     endpoint("GET", "read/members", async (req) => {
-      const params = parseReadMemberParams(req.query);
-      if (params.ids.length > 1 && params.slugs.length > 1) {
-        throw new BackendError(400, "Both 'ids' and 'slugs' are filled. Pick one.");
+      const params = parseReadMembersParams(req.query);
+      const list = params[params.mode] || [];
+      if (list.length === 0) {
+        throw new BackendError(400, `The request is missing the '${params.mode}' parameter.`);
       }
-      if (params.ids.length === 0 && params.slugs.length === 0) {
-        throw new BackendError(400, "Params 'ids' and 'slugs' are missing. Fill one.");
-      }
-      if ((params.mode === "ids" ? params.ids.length : params.slugs.length) > 5) {
-        throw new BackendError(400, "Too many 'ids' and 'slugs' in the request. Try with less than 5.");
+      if (list.length > 20) {
+        throw new BackendError(400, `You can look for up to 20 '${params.mode}' in a single request.`);
       }
       return readMember(params);
     }),
@@ -2845,15 +2879,17 @@ function endpointMemberFactory(operations) {
     })
   ];
 }
-function parseReadMemberParams(query) {
-  const ids = normalizeList(query.ids).map(parseFiniteNumber);
+function parseReadMembersParams(query) {
+  const ids = normalizeList(query.ids);
+  const content_ids = normalizeList(query.content_ids).map(parseFiniteNumber);
   const locale = normalizeList(query.locale)[0] || localeDefault3 || "en";
   const localeIsAll = locale === "all";
   return {
     all: localeIsAll,
+    locale: localeIsAll ? localeDefault3 : stripHTML(locale),
+    mode: ids.length > 0 ? "ids" : content_ids.length > 0 ? "content_ids" : "slugs",
     ids,
-    locale: localeIsAll ? localeDefault3 : locale,
-    mode: ids.length > 0 ? "ids" : "slugs",
+    content_ids,
     slugs: normalizeList(query.slugs).map((token) => {
       const [variantSlug, memberSlug] = token.split("/");
       return { variantSlug, memberSlug };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@datawheel/bespoke",
-  "version": "0.1.9",
+  "version": "0.1.11",
   "description": "Content management system for creating automated data reports",
   "exports": {
     ".": {
@@ -10,8 +10,11 @@
       "import": "./dist/server.js"
     }
   },
-  "files": ["dist", "README.md"],
-  "style": "./dist/style.css",
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "style": "./dist/index.css",
   "scripts": {
     "prepublishOnly": "tsup",
     "build": "tsup",
@@ -81,14 +84,12 @@
     "normalizr": "^3.6.2",
     "pg": "^8.7.3",
     "pretty-format": "^28.1.1",
-    "react": "^17.0.2",
+    "react": "^17.0.0||^18.0.0",
     "react-beautiful-dnd": "^13.1.0",
     "react-clipboard.js": "^2.0.16",
     "react-icons": "^4.3.1",
     "react-redux": "^7.2.6",
-    "react-router-dom": "^6.3.0",
     "react-share": "^4.4.0",
-    "react-table-6": "^6.11.0",
     "sequelize": "^6.17.0",
     "sharp": "^0.31.0",
     "shelljs": "^0.8.5",