@datawheel/bespoke 0.1.26 → 0.1.28

package/dist/server.js

@@ -1,6 +1,6 @@
 import { initAuth0 } from '@auth0/nextjs-auth0';
 import * as pg from 'pg';
-import { Sequelize, Error as Error$1, fn, col, Op, DataTypes, literal, Model } from 'sequelize';
+import { Sequelize, Error as Error$1, fn, col, Op, DataTypes, QueryTypes, Model } from 'sequelize';
 import yn3 from 'yn';
 import * as d3Array from 'd3-array';
 import * as d3Collection from 'd3-collection';
@@ -1687,7 +1687,7 @@ var getDB = () => {
         ${e.message}
       `);
     }
-    Object.values(modelFactoryMap).map((modelFactory) => modelFactory(sequelize)).map((modelAssociator) => modelAssociator(sequelize.models));
+    Object.values(modelFactoryMap).map((modelFactory) => modelFactory(sequelize)).map((modelAssociator) => modelAssociator({ ...sequelize.models, sequelize }));
     global.sequelize = sequelize.sync({ force: shouldWipe }).catch((err) => {
       if (err instanceof Error$1) {
         console.error(
@@ -1697,7 +1697,14 @@ var getDB = () => {
         );
       }
       global.sequelize = void 0;
-    }).then(() => sequelize.models);
+    }).then(
+      () => ({
+        ...sequelize.models,
+        // All the models
+        sequelize
+        // Store the whole sequelize object in global to use for raw queries
+      })
+    );
   } else {
     if (verbose2)
       console.log("\u267B\uFE0F  RE-UTILIZING DB CONNECTION");
@@ -1830,7 +1837,7 @@ function ingestMembersFactory(db) {
       const contentHash = await Search.findAll({ where: { variant_id: id } }).then((arr) => Object.fromEntries(
         arr.map((content) => [content.id, content.content_id])
       ));
-      const contentMembers = members2.map((d) => ({
+      const contentMembers = members2.filter((member) => member[idKey2] && member[idKey2] !== "" && member[labelKey2] && member[labelKey2] !== "").map((d) => ({
         id: contentHash[d[idKey2]],
         locale,
         name: d[labelKey2],
@@ -1854,7 +1861,7 @@ function ingestMembersFactory(db) {
   async function ingestDefaultMembers(variant, locale, members) {
     const { id, dimension_id, slug } = variant;
     const { report_id } = variant.dimension;
-    const { idKey, labelKey } = variant.config[locale];
+    const { idKey, labelKey, zValueKey } = variant.config[locale];
     const allSlugsFromOtherMembers = await Search.findAll({
       attributes: ["slug", [fn("COUNT", col("variant.id")), "slugCount"]],
       include: {
@@ -1870,6 +1877,21 @@ function ingestMembersFactory(db) {
       return [item.slug, Number.parseInt(`${count}`, 10)];
     })));
     const toIngestSlugs = [];
+    let minValue = null;
+    let maxValue = null;
+    if (zValueKey && zValueKey !== "") {
+      for (const member of members) {
+        if (typeof member[zValueKey] === "number" && !isNaN(member[zValueKey])) {
+          const numericValue = member[zValueKey];
+          if (minValue === null || numericValue < minValue) {
+            minValue = numericValue;
+          }
+          if (maxValue === null || numericValue > maxValue) {
+            maxValue = numericValue;
+          }
+        }
+      }
+    }
     const searchMembers = members.map((member) => {
       const memberId = member[idKey];
       const slug2 = slugify_default(member[labelKey]);
@@ -1879,13 +1901,24 @@ function ingestMembersFactory(db) {
       }
       const validateSlug = hasDuplicateSlug ? `${slug2}-${memberId}` : slug2;
       toIngestSlugs.push(slug2);
+      let zvalue = Math.random();
+      if (zValueKey && zValueKey !== "" && minValue && maxValue && member[zValueKey]) {
+        let value;
+        try {
+          value = parseInt(member[zValueKey], 10);
+        } catch (error) {
+          value = 0;
+        }
+        const indexedValue = maxValue !== null && minValue !== null && maxValue !== minValue ? (value - minValue) / (maxValue - minValue) : 0;
+        zvalue = maxValue === minValue ? 1 : parseFloat(indexedValue.toPrecision(5));
+      }
       return {
         id: memberId,
         slug: validateSlug,
         variant_id: id,
         dimension_id,
         report_id,
-        zvalue: Math.random()
+        zvalue
       };
     });
     try {
@@ -2151,7 +2184,7 @@ var search_default2 = getSearchIndexByLocale;
 // api/db/member/searchMember.ts
 var { localeDefault: localeDefault2 } = getLocales_default();
 function dbSearchMemberFactory(db) {
-  const { search: Search } = db;
+  const { search: Search, sequelize } = db;
   return searchMember2;
   async function searchMember2(params) {
     const searchIndexByLocale = await search_default2(db);
@@ -2186,21 +2219,19 @@ function dbSearchMemberFactory(db) {
     const contentWhereClause = {
       locale: all ? { [Op.ne]: null } : locale
     };
+    const calculatedLimit = limit && !isNaN(limit) ? limit : 20;
     if (query === "") {
-      const rank = await Search.findAll({
-        attributes: [
-          "id",
-          "content_id",
-          "report_id",
-          [literal("rank() over(partition by report_id order by zvalue desc)"), "rank"]
-        ],
-        where: whereClause,
-        include: [{
-          association: "contentByLocale",
-          where: contentWhereClause
-        }]
-      }).then((arr) => arr.map((d) => d.toJSON()));
-      resultsIds = rank.filter((item) => parseInt(item["rank"], 10) <= limit).map((d) => d.content_id);
+      const rank = await sequelize.query(`
+      SELECT t.content_id
+      FROM (
+        SELECT id, content_id, variant_id, zvalue,
+          RANK() OVER (PARTITION BY variant_id ORDER BY zvalue DESC) AS rank
+        FROM bespoke_reports_search
+      ) t
+      WHERE t.rank <= ${calculatedLimit}
+      ORDER BY t.variant_id, t.zvalue DESC;
+        `, { type: QueryTypes.SELECT });
+      resultsIds = rank.map((d) => d.content_id);
     }
     whereClause.content_id = resultsIds;
     const includeClause = [{
@@ -3416,6 +3447,7 @@ var addProfilesData = (items, metadata) => {
             profileResults = profileResults.concat(members.map((member) => ({
               path: `/${variant.slug}/${member.slug}`,
               name: member.name,
+              confidence: member.confidence,
               report: {
                 id: report.id,
                 name: report.name,
@@ -3429,7 +3461,7 @@ var addProfilesData = (items, metadata) => {
       } else if (report.mode === REPORT_MODES.MULTILATERAL) ;
     }
   });
-  return profileResults;
+  return profileResults.sort((a, b) => a.confidence < b.confidence ? 1 : -1);
 };
 var formatNested = (items, metadata) => {
   if (!items)
@@ -3471,23 +3503,18 @@ function dbSearchReportFactory(db) {
       locale: stripHTML(locale),
       visible: true
     });
-    let reportSearchResults = memberSearchResponse.results ? memberSearchResponse.results : [];
-    let tempResults;
+    const reportSearchResults = memberSearchResponse.results ? memberSearchResponse.results : [];
     const metadataResponse = await dbReadMetadata();
     const metadata = metadataResponse.data;
-    switch (format) {
-      case "profiles":
-        tempResults = formatNested(reportSearchResults, metadata);
-        reportSearchResults = addProfilesData(tempResults, metadata);
-        break;
-    }
+    const tempResults = formatNested(reportSearchResults, metadata);
     return {
       meta: {
         ...memberSearchResponse.meta,
         format
         // Override format
       },
-      results: reportSearchResults
+      results: addProfilesData(tempResults, metadata)
+      // Group entities as profiles
     };
   }
 }
@@ -4303,7 +4330,6 @@ function httpImageSearchFactory(axios6, provider) {
 
 // api/http/index.ts
 var transformDeletePayload = (id) => {
-  console.log("transformDeletePayload", id);
   return { id };
 };
 var transformReadMembers = (params) => {
@@ -4521,6 +4547,7 @@ var runSelector_default = (logic, formatterFunctions, blockContext) => {
   const evalResult = mortarEval_default("variables", variables, transpiledLogic, formatterFunctions, void 0, blockContext);
   const { vars, log, error } = evalResult;
   const type = vars.type || SELECTOR_TYPES.SINGLE;
+  const component = vars.component || "Selector";
   const name = vars.name || "Unlabeled Selector";
   const options = scaffoldDynamic_default(vars.options || []);
   const maybeFixForMulti = (defaultValue2) => type === SELECTOR_TYPES.MULTI && !Array.isArray(defaultValue2) ? [defaultValue2] : defaultValue2;
@@ -4539,7 +4566,8 @@ var runSelector_default = (logic, formatterFunctions, blockContext) => {
     name,
     type,
     options,
-    defaultValue
+    defaultValue,
+    component
   };
   return { config, log, error };
 };
@@ -4683,11 +4711,26 @@ async function apiFetch(url, settings, readMemberFn, locale = "en") {
 }
 async function runSingleBlock(block, formatterFunctions, blockContext, readMemberFn) {
   const { locale, variables, query } = parseBlockContext(blockContext);
+  const allowed = isBlockAllowed(block, blockContext, variables, formatterFunctions);
   let resp = null;
   const unswappedAPI = block.contentByLocale?.logic?.content?.api;
   let apiResponse;
   let duration = null;
   let api = null;
+  if (!allowed) {
+    return {
+      outputVariables: {},
+      renderVariables: {},
+      status: {
+        api: null,
+        duration: null,
+        resp: null,
+        log: [],
+        error: null,
+        allowed: false
+      }
+    };
+  }
   if (unswappedAPI) {
     api = swapAPI(unswappedAPI, formatterFunctions, blockContext);
     const isAbsolute = new RegExp("^(?:[a-z+]+:)?//", "i");
@@ -4700,7 +4743,7 @@ async function runSingleBlock(block, formatterFunctions, blockContext, readMembe
       return result;
     }, (e) => {
       if (verbose7)
-        console.error(`Error fetching ${api}: ${e.message}`);
+        console.error(`Error fetching ${api} block ${block.id}: ${e.message}`);
       return { data: {}, requestDuration: 0 };
     });
     resp = apiResponse.data;
@@ -4720,7 +4763,8 @@ async function runSingleBlock(block, formatterFunctions, blockContext, readMembe
         duration: null,
         resp: null,
         log: log2,
-        error: error2
+        error: error2,
+        allowed
       }
     };
   }
@@ -4734,7 +4778,8 @@ async function runSingleBlock(block, formatterFunctions, blockContext, readMembe
         duration: null,
         resp: null,
         log: [],
-        error: null
+        error: null,
+        allowed
       }
     };
   }
@@ -4751,7 +4796,8 @@ async function runSingleBlock(block, formatterFunctions, blockContext, readMembe
         log,
         duration,
         resp,
-        api
+        api,
+        allowed
       }
     };
   }
@@ -4764,31 +4810,56 @@ async function runSingleBlock(block, formatterFunctions, blockContext, readMembe
       error,
       duration,
       resp,
-      api
+      api,
+      allowed
     }
   };
 }
-var getDependencies = (bid, blocks, acc = [], crawlUp = true, crawlDown2 = true) => {
+var getDependencies = (bid, blocks, acc = [], crawlUp = true, crawlDown2 = true, withinSection = true, visited = []) => {
+  if (visited.includes(bid))
+    return [];
+  visited.push(bid);
   const rootBlock = blocks[bid];
   if (rootBlock.inputs.length && crawlUp) {
     rootBlock.inputs.forEach((iid) => {
       const rel = `${iid}-${bid}`;
       if (!acc.includes(rel))
         acc.push(rel);
-      rootBlock.inputs.forEach((iid2) => getDependencies(iid2, blocks, acc, crawlUp, false));
+      rootBlock.inputs.forEach((iid2) => getDependencies(iid2, blocks, acc, crawlUp, false, withinSection, visited));
     });
   }
   if (rootBlock.consumers.length && crawlDown2) {
     rootBlock.consumers.forEach((cid) => {
       const rel = `${bid}-${cid}`;
-      if (!acc.includes(rel) && blocks[cid].section_id === blocks[bid].section_id)
+      if (!acc.includes(rel) && (blocks[cid].section_id === blocks[bid].section_id || !withinSection))
         acc.push(rel);
-      rootBlock.consumers.filter((cid2) => blocks[cid2].section_id === blocks[bid].section_id).forEach((cid2) => getDependencies(cid2, blocks, acc, crawlUp, crawlDown2));
+      rootBlock.consumers.filter((cid2) => blocks[cid2].section_id === blocks[bid].section_id || !withinSection).forEach((cid2) => getDependencies(cid2, blocks, acc, crawlUp, crawlDown2, withinSection, visited));
     });
   }
   return acc;
 };
-var runConsumersV2 = async (blocks, sections, bid, formatterFunctions, blockContext, initialVariables = {}, readMemberFn) => {
+var isBlockAllowed = (block, blockContext, variables, formatterFunctions) => {
+  let allowed = true;
+  if (block && "allowed" in block.settings && block.settings.allowed !== "always") {
+    if (block.settings.allowed === "never") {
+      allowed = false;
+    } else {
+      const parsedBlockContext = parseBlockContext(blockContext);
+      const { error, output } = mortarEval_default(
+        "variables",
+        variables,
+        block.settings.allowedLogic || "return false;",
+        formatterFunctions,
+        void 0,
+        parsedBlockContext
+      );
+      if (!error)
+        allowed = Boolean(output);
+    }
+  }
+  return allowed;
+};
+var runConsumersV2 = async (blocks, sections, bid, formatterFunctions, blockContext, initialVariables = {}, readMemberFn, mode = "section") => {
   if (!bid && !sections)
     return { variables: { ...initialVariables } };
   const variablesById = { ...initialVariables };
@@ -4796,8 +4867,9 @@ var runConsumersV2 = async (blocks, sections, bid, formatterFunctions, blockCont
   const parsedBlockContext = parseBlockContext(blockContext);
   const attributes = parsedBlockContext.variables;
   const rootBlocks = bid ? { [bid]: blocks[bid] } : sections.reduce((rootBlocks2, { id }) => ({ ...rootBlocks2, ...getRootBlocksForSection_default(id, blocks) }), {});
+  const withinSection = mode === "section";
   const blockDeps = Object.keys(rootBlocks).reduce((deps, id) => {
-    const dependencies = getDependencies(Number(id), blocks, [], !bid).map((rel) => rel.split("-"));
+    const dependencies = getDependencies(Number(id), blocks, [], !bid, true, withinSection).map((rel) => rel.split("-"));
     return [...deps, ...dependencies];
   }, []);
   const orderedDAG = Object.keys(rootBlocks).reduce(
@@ -4813,6 +4885,19 @@ var runConsumersV2 = async (blocks, sections, bid, formatterFunctions, blockCont
       );
       const runningTask = dependenciesDone.then(() => {
         const variables = block.inputs.reduce((acc, d) => ({ ...acc, ...variablesById[d] }), attributes);
+        const inputNotAllowed = block.inputs.find((iid) => !statusById[iid].allowed || statusById[iid].hiddenByCascade);
+        if (inputNotAllowed) {
+          statusById[bid2] = {
+            api: null,
+            duration: null,
+            resp: null,
+            log: [],
+            error: null,
+            allowed: false,
+            hiddenByCascade: statusById[inputNotAllowed].hiddenByCascade || inputNotAllowed
+          };
+          return;
+        }
         return runSingleBlock(
           block,
           formatterFunctions,
@@ -5718,6 +5803,7 @@ var storeFactory = (_context) => configureStore({
     return getDefaultMiddleware({
       thunk: {
         extraArgument: apiFactory2("/api/cms/")
+        // serializableCheck: false,
       }
     });
   }
@@ -5767,7 +5853,8 @@ function parseReportId(value) {
 var { locales: locales8 } = getLocales_default();
 var initialContext = {
   pathSegment: "path",
-  formatterFunctions: locales8.reduce((acc, d) => ({ ...acc, [d]: {} }), {})
+  formatterFunctions: locales8.reduce((acc, d) => ({ ...acc, [d]: {} }), {}),
+  profilePrefix: "path"
 };
 createContext(initialContext);
 
@@ -5898,7 +5985,8 @@ function BespokeRendererStaticProps(options) {
             variables: attributes
           },
           void 0,
-          readMemberFn
+          readMemberFn,
+          "report"
         ).then((data) => {
           dispatch(variablesActions.setVariableChange({ ...data, attributes }));
         });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@datawheel/bespoke",
-  "version": "0.1.26",
+  "version": "0.1.28",
   "description": "Content management system for creating automated data reports",
   "exports": {
     ".": {
@@ -59,8 +59,10 @@
     "base58": "^2.0.1",
     "buble": "^0.20.0",
     "comment-parser": "^1.3.1",
+    "d3": "^7.8.4",
     "d3-array": "^3.1.6",
     "d3-collection": "^1.0.7",
+    "d3-dag": "^0.11.5",
     "d3-dsv": "^1.2.0",
     "d3-format": "^3.1.0",
     "d3-selection": "^3.0.0",