@dittowords/cli 4.0.0-alpha.0 → 4.0.1-alpha.0

package/lib/config.ts

@@ -5,12 +5,12 @@ import yaml from "js-yaml";
 import * as Sentry from "@sentry/node";
 
 import consts from "./consts";
-import { Project, ConfigYAML } from "./types";
 import { createSentryContext } from "./utils/createSentryContext";
+import { Project, ConfigYAML, SourceInformation } from "./types";
 
 export const DEFAULT_CONFIG_JSON: ConfigYAML = {
   sources: {
-    components: { enabled: true },
+    components: true,
   },
   variants: true,
   format: "flat",
@@ -188,7 +188,7 @@ function dedupeProjectName(projectNames: Set<string>, projectName: string) {
  * - an array of valid, deduped projects
  * - the `variants` and `format` config options
  */
-function parseSourceInformation(file?: string) {
+function parseSourceInformation(file?: string): SourceInformation {
   const {
     sources,
     variants,
@@ -222,8 +222,20 @@ function parseSourceInformation(file?: string) {
     validProjects.push(project);
   });
 
-  const shouldFetchComponentLibrary = Boolean(sources?.components?.enabled);
-
+  const shouldFetchComponentLibrary = Boolean(sources?.components);
+  const componentRoot =
+    typeof sources?.components === "object"
+      ? sources.components.root
+      : undefined;
+  const componentFolders =
+    typeof sources?.components === "object"
+      ? sources.components.folders
+      : undefined;
+
+  /**
+   * If it's not specified to fetch projects or the component library, then there
+   * is no source data to pull.
+   */
   const hasSourceData = !!validProjects.length || shouldFetchComponentLibrary;
 
   const result = {
@@ -237,7 +249,8 @@ function parseSourceInformation(file?: string) {
     hasTopLevelProjectsField: !!projectsRoot,
     hasTopLevelComponentsField: !!componentsRoot,
     hasComponentLibraryInProjects,
-    componentFolders: sources?.components?.folders || null,
+    componentRoot,
+    componentFolders,
   };
 
   Sentry.setContext("config", createSentryContext(result));

package/lib/init/project.ts

@@ -17,7 +17,7 @@ import { quit } from "../utils/quit";
 function saveProject(file: string, name: string, id: string) {
   if (id === "components") {
     config.writeProjectConfigData(file, {
-      sources: { components: { enabled: true } },
+      sources: { components: true },
     });
     return;
   }

package/lib/pull.ts

@@ -12,11 +12,25 @@ import { collectAndSaveToken } from "./init/token";
 import sourcesToText from "./utils/sourcesToText";
 import { generateJsDriver } from "./utils/generateJsDriver";
 import { cleanFileName } from "./utils/cleanFileName";
-import { SourceInformation, Token, Project, SupportedFormat } from "./types";
+import {
+  SourceInformation,
+  Token,
+  Project,
+  SupportedFormat,
+  ComponentFolder,
+  ComponentSource,
+  Source,
+} from "./types";
 import { fetchVariants } from "./http/fetchVariants";
-import { kMaxLength } from "buffer";
 import { quit } from "./utils/quit";
 import { AxiosError } from "axios";
+import { fetchComponentFolders } from "./http/fetchComponentFolders";
+
+const ensureEndsWithNewLine = (str: string) =>
+  str + (/[\r\n]$/.test(str) ? "" : "\n");
+
+const writeFile = (path: string, data: string) =>
+  new Promise((r) => fs.writeFile(path, ensureEndsWithNewLine(data), r));
 
 const SUPPORTED_FORMATS: SupportedFormat[] = [
   "flat",
@@ -41,7 +55,7 @@ const FORMAT_EXTENSIONS = {
 const getJsonFormatIsValid = (data: string) => {
   try {
     return Object.keys(JSON.parse(data)).some(
-      (k) => !k.startsWith("__variant")
+      (k) => !k.startsWith("__variant"),
     );
   } catch {
     return false;
@@ -59,12 +73,12 @@ export const getFormatDataIsValid = {
 };
 
 const getFormat = (
-  formatFromSource: string | string[] | undefined
+  formatFromSource: string | string[] | undefined,
 ): SupportedFormat[] => {
   const formats = (
     Array.isArray(formatFromSource) ? formatFromSource : [formatFromSource]
   ).filter((format) =>
-    SUPPORTED_FORMATS.includes(format as SupportedFormat)
+    SUPPORTED_FORMATS.includes(format as SupportedFormat),
   ) as SupportedFormat[];
 
   if (formats.length) {
@@ -109,18 +123,26 @@ async function downloadAndSaveVariant(
   format: SupportedFormat,
   status: string | undefined,
   richText: boolean | undefined,
-  token?: Token
+  token?: Token,
 ) {
   const api = createApiClient();
   const params: Record<string, string | null> = { variant: variantApiId };
   if (format) params.format = format;
-  if (status) params.status = status;
   if (richText) params.includeRichText = richText.toString();
 
+  // Root-level status gets set as the default if specified
+  if (status) params.status = status;
+
   const savedMessages = await Promise.all(
-    projects.map(async ({ id, fileName }: Project) => {
-      const { data } = await api.get(`/projects/${id}`, {
-        params,
+    projects.map(async (project) => {
+      const projectParams = { ...params };
+      // If project-level status is specified, overrides root-level status
+      if (project.status) projectParams.status = project.status;
+      if (project.exclude_components)
+        projectParams.exclude_components = String(project.exclude_components);
+
+      const { data } = await api.get(`/projects/${project.id}`, {
+        params: projectParams,
         headers: { Authorization: `token ${token}` },
       });
 
@@ -131,7 +153,7 @@ async function downloadAndSaveVariant(
       const extension = getFormatExtension(format);
 
       const filename = cleanFileName(
-        fileName + ("__" + (variantApiId || "base")) + extension
+        project.fileName + ("__" + (variantApiId || "base")) + extension,
       );
       const filepath = path.join(consts.TEXT_DIR, filename);
 
@@ -145,9 +167,9 @@ async function downloadAndSaveVariant(
         return "";
       }
 
-      fs.writeFileSync(filepath, dataString);
+      await writeFile(filepath, dataString);
       return getSavedMessage(filename);
-    })
+    }),
   );
 
   return savedMessages.join("");
@@ -159,12 +181,12 @@ async function downloadAndSaveVariants(
   format: SupportedFormat,
   status: string | undefined,
   richText: boolean | undefined,
-  token?: Token
+  token?: Token,
 ) {
   const messages = await Promise.all([
     downloadAndSaveVariant(null, projects, format, status, richText, token),
     ...variants.map(({ apiID }: { apiID: string }) =>
-      downloadAndSaveVariant(apiID, projects, format, status, richText, token)
+      downloadAndSaveVariant(apiID, projects, format, status, richText, token),
     ),
   ]);
 
@@ -177,23 +199,31 @@ async function downloadAndSaveBase(
   status: string | undefined,
   richText?: boolean | undefined,
   token?: Token,
-  options?: PullOptions
+  options?: PullOptions,
 ) {
   const api = createApiClient();
   const params = { ...options?.meta };
   if (format) params.format = format;
-  if (status) params.status = status;
   if (richText) params.includeRichText = richText.toString();
 
+  // Root-level status gets set as the default if specified
+  if (status) params.status = status;
+
   const savedMessages = await Promise.all(
-    projects.map(async ({ id, fileName }: Project) => {
-      const { data } = await api.get(`/projects/${id}`, {
-        params,
+    projects.map(async (project) => {
+      const projectParams = { ...params };
+      // If project-level status is specified, overrides root-level status
+      if (project.status) projectParams.status = project.status;
+      if (project.exclude_components)
+        projectParams.exclude_components = String(project.exclude_components);
+
+      const { data } = await api.get(`/projects/${project.id}`, {
+        params: projectParams,
         headers: { Authorization: `token ${token}` },
       });
 
       const extension = getFormatExtension(format);
-      const filename = cleanFileName(`${fileName}__base${extension}`);
+      const filename = cleanFileName(`${project.fileName}__base${extension}`);
       const filepath = path.join(consts.TEXT_DIR, filename);
 
       let dataString = data;
@@ -206,9 +236,9 @@ async function downloadAndSaveBase(
         return "";
       }
 
-      fs.writeFileSync(filepath, dataString);
+      await writeFile(filepath, dataString);
       return getSavedMessage(filename);
-    })
+    }),
   );
 
   return savedMessages.join("");
@@ -236,16 +266,27 @@ function cleanOutputFiles() {
 async function downloadAndSave(
   source: SourceInformation,
   token?: Token,
-  options?: PullOptions
+  options?: PullOptions,
 ) {
   const api = createApiClient();
+
+  if (process.env.DEBUG_CLI) {
+    try {
+      await api.get("/health");
+      console.debug("Can connect to api.dittowords.com");
+    } catch {
+      console.debug("CANNOT connect to api.dittowords.com");
+    }
+  }
+
   const {
     validProjects,
     format: formatFromSource,
     shouldFetchComponentLibrary,
     status,
     richText,
-    componentFolders,
+    componentFolders: specifiedComponentFolders,
+    componentRoot,
   } = source;
 
   const formats = getFormat(formatFromSource);
@@ -254,17 +295,81 @@ async function downloadAndSave(
   const spinner = ora(msg);
   spinner.start();
 
-  const variants = await fetchVariants(source);
+  const [variants, allComponentFoldersResponse] = await Promise.all([
+    fetchVariants(source),
+    fetchComponentFolders(),
+  ]);
+
+  const allComponentFolders = Object.entries(
+    allComponentFoldersResponse,
+  ).reduce(
+    (acc, [id, name]) => acc.concat([{ id, name }]),
+    [] as ComponentFolder[],
+  );
 
   try {
     msg += cleanOutputFiles();
     msg += `\nFetching the latest text from ${sourcesToText(
       validProjects,
-      shouldFetchComponentLibrary
+      shouldFetchComponentLibrary,
     )}\n`;
 
     const meta = options ? options.meta : {};
 
+    const rootRequest = {
+      id: "__root__",
+      name: "Root",
+      // componentRoot can be a boolean or an object
+      status:
+        typeof source.componentRoot === "object"
+          ? source.componentRoot.status
+          : undefined,
+    };
+
+    let componentFolderRequests: ComponentFolder[] = [];
+
+    // there's a lot of complex logic here, and it's tempting to want to
+    // simplify it. however, it's difficult to get rid of the complexity
+    // without sacrificing specificity and expressiveness.
+    //
+    // if folders specified..
+    if (specifiedComponentFolders) {
+      switch (componentRoot) {
+        // .. and no root specified, you only get components in the specified folders
+        case undefined:
+        case false:
+          componentFolderRequests.push(...specifiedComponentFolders);
+          break;
+        // .. and root specified, you get components in folders and the root
+        default:
+          componentFolderRequests.push(...specifiedComponentFolders);
+          componentFolderRequests.push(rootRequest);
+          break;
+      }
+    }
+    // if no folders specified..
+    else {
+      switch (componentRoot) {
+        // .. and no root specified, you get all components including those in folders
+        case undefined:
+          componentFolderRequests.push(...allComponentFolders);
+          componentFolderRequests.push(rootRequest);
+          break;
+        // .. and root specified as false, you only get components in folders
+        case false:
+          componentFolderRequests.push(...allComponentFolders);
+          break;
+        // .. and root specified as true or config object, you only get components in the root
+        default:
+          componentFolderRequests.push(rootRequest);
+          break;
+      }
+    }
+
+    // this array is populated while fetching from the component library and is used when
+    // generating the index.js driver file
+    const componentSources: ComponentSource[] = [];
+
     async function fetchComponentLibrary(format: SupportedFormat) {
       // Always include a variant with an apiID of undefined to ensure that we
       // fetch the base text for the component library.
@@ -274,41 +379,70 @@ async function downloadAndSave(
       if (options?.meta)
         Object.entries(options.meta).forEach(([k, v]) => params.append(k, v));
       if (format) params.append("format", format);
-      if (status) params.append("status", status);
       if (richText) params.append("includeRichText", richText.toString());
-      if (componentFolders) {
-        componentFolders.forEach(({ id }) => params.append("folder_id[]", id));
-      }
 
-      const messages = await Promise.all(
-        componentVariants.map(async ({ apiID: variantApiId }) => {
-          const p = new URLSearchParams(params);
-          if (variantApiId) p.append("variant", variantApiId);
+      // Root-level status gets set as the default if specified
+      if (status) params.append("status", status);
 
-          const { data } = await api.get(`/components`, { params: p });
+      const messagePromises: Promise<string>[] = [];
 
-          const nameExt = getFormatExtension(format);
-          const nameBase = "ditto-component-library";
-          const namePostfix = `__${variantApiId || "base"}`;
+      componentVariants.forEach(({ apiID: variantApiId }) => {
+        messagePromises.push(
+          ...componentFolderRequests.map(async (componentFolder) => {
+            const componentFolderParams = new URLSearchParams(params);
 
-          const fileName = cleanFileName(`${nameBase}${namePostfix}${nameExt}`);
-          const filePath = path.join(consts.TEXT_DIR, fileName);
+            if (variantApiId)
+              componentFolderParams.append("variant", variantApiId);
 
-          let dataString = data;
-          if (nameExt === ".json") {
-            dataString = JSON.stringify(data, null, 2);
-          }
+            // If folder-level status is specified, overrides root-level status
+            if (componentFolder.status)
+              componentFolderParams.append("status", componentFolder.status);
 
-          const dataIsValid = getFormatDataIsValid[format];
-          if (!dataIsValid(dataString)) {
-            return "";
-          }
+            const url =
+              componentFolder.id === "__root__"
+                ? "/components?root_only=true"
+                : `/component-folders/${componentFolder.id}/components`;
 
-          await new Promise((r) => fs.writeFile(filePath, dataString, r));
-          return getSavedMessage(fileName);
-        })
-      );
+            const { data } = await api.get(url, {
+              params: componentFolderParams,
+            });
+
+            const nameExt = getFormatExtension(format);
+            const nameBase = "components";
+            const nameFolder = `__${componentFolder.name}`;
+            const namePostfix = `__${variantApiId || "base"}`;
+
+            const fileName = cleanFileName(
+              `${nameBase}${nameFolder}${namePostfix}${nameExt}`,
+            );
+            const filePath = path.join(consts.TEXT_DIR, fileName);
+
+            let dataString = data;
+            if (nameExt === ".json") {
+              dataString = JSON.stringify(data, null, 2);
+            }
+
+            const dataIsValid = getFormatDataIsValid[format];
+            if (!dataIsValid(dataString)) {
+              return "";
+            }
 
+            await writeFile(filePath, dataString);
+
+            componentSources.push({
+              type: "components",
+              id: "ditto_component_library",
+              name: "ditto_component_library",
+              fileName,
+              variant: variantApiId || "base",
+            });
+
+            return getSavedMessage(fileName);
+          }),
+        );
+      });
+
+      const messages = await Promise.all(messagePromises);
       msg += messages.join("");
     }
 
@@ -326,7 +460,7 @@ async function downloadAndSave(
             format,
             status,
             richText,
-            token
+            token,
           )
         : await downloadAndSaveBase(
             validProjects,
@@ -336,7 +470,7 @@ async function downloadAndSave(
             token,
             {
               meta,
-            }
+            },
           );
     }
 
@@ -346,14 +480,7 @@ async function downloadAndSave(
       }
     }
 
-    const sources = [...validProjects];
-    if (shouldFetchComponentLibrary) {
-      sources.push({
-        id: "ditto_component_library",
-        name: "Ditto Component Library",
-        fileName: "ditto-component-library",
-      });
-    }
+    const sources: Source[] = [...validProjects, ...componentSources];
 
     if (formats.some((f) => JSON_FORMATS.includes(f)))
       msg += generateJsDriver(sources);
@@ -375,7 +502,7 @@ async function downloadAndSave(
     if (e.response && e.response.status === 401) {
       error = "You don't have access to the selected projects";
       msg = `${output.errorText(error)}.\nChoose others using the ${output.info(
-        "project"
+        "project",
       )} command, or update your API key.`;
       return console.log(msg);
     }
@@ -383,11 +510,11 @@ async function downloadAndSave(
       error =
         "One or more of the requested projects don't have Developer Mode enabled";
       msg = `${output.errorText(
-        error
+        error,
       )}.\nPlease choose different projects using the ${output.info(
-        "project"
+        "project",
       )} command, or turn on Developer Mode for all selected projects. Learn more here: ${output.subtle(
-        "https://www.dittowords.com/docs/ditto-developer-mode"
+        "https://www.dittowords.com/docs/ditto-developer-mode",
       )}.`;
       return console.log(msg);
     }
@@ -395,7 +522,7 @@ async function downloadAndSave(
       error = "projects not found";
     }
     msg = `We hit an error fetching text from the projects: ${output.errorText(
-      error
+      error,
     )}.\nChoose others using the ${output.info("project")} command.`;
     return console.log(msg);
   }
@@ -410,22 +537,27 @@ export const pull = async (options?: PullOptions) => {
   const token = config.getToken(consts.CONFIG_FILE, consts.API_HOST);
   const sourceInformation = config.parseSourceInformation();
 
+  if (process.env.DEBUG_CLI === "true") {
+    console.debug(`Token: ${token}`);
+  }
+
   try {
     return await downloadAndSave(sourceInformation, token, { meta });
   } catch (e) {
-    Sentry.captureException(e);
+    const eventId = Sentry.captureException(e);
+    const eventStr = `\n\nError ID: ${output.info(eventId)}`;
     if (e instanceof AxiosError) {
       return quit(
         output.errorText(
-          "Something went wrong connecting to Ditto servers. Please contact support or try again later."
-        )
+          "Something went wrong connecting to Ditto servers. Please contact support or try again later.",
+        ) + eventStr,
       );
     }
 
     return quit(
       output.errorText(
-        "Something went wrong. Please contact support or try again later."
-      )
+        "Something went wrong. Please contact support or try again later.",
+      ) + eventStr,
     );
   }
 };

package/lib/types.ts

@@ -3,13 +3,22 @@ export interface Project {
   id: string;
   url?: string;
   fileName?: string;
+  status?: string;
+  exclude_components?: boolean;
 }
 
-export type Source = Project;
+export type ComponentSource = ComponentFolder & {
+  type: "components";
+  fileName: string;
+  variant: string;
+};
+
+export type Source = (Project & { type?: undefined }) | ComponentSource;
 
-interface ComponentFolder {
+export interface ComponentFolder {
   id: string;
   name: string;
+  status?: string;
 }
 
 export type SupportedFormat =
@@ -20,12 +29,16 @@ export type SupportedFormat =
   | "ios-stringsdict"
   | "icu";
 
+type ComponentsSourceBool = boolean;
+type ComponentsSourceConfig = {
+  root?: boolean | { status: string };
+  folders?: ComponentFolder[];
+};
+type ComponentsSource = ComponentsSourceBool | ComponentsSourceConfig;
+
 export interface ConfigYAML {
   sources?: {
-    components?: {
-      enabled?: boolean;
-      folders?: ComponentFolder[];
-    };
+    components?: ComponentsSource;
     projects?: Project[];
   };
   format?: SupportedFormat;
@@ -42,13 +55,17 @@ export interface ConfigYAML {
 
 export interface SourceInformation {
   hasSourceData: boolean;
+  hasTopLevelProjectsField: boolean;
+  hasTopLevelComponentsField: boolean;
+  hasComponentLibraryInProjects: boolean;
   validProjects: Project[];
   shouldFetchComponentLibrary: boolean;
   variants: boolean;
   format: string | string[] | undefined;
   status: string | undefined;
   richText: boolean | undefined;
-  componentFolders: ComponentFolder[] | null;
+  componentRoot: boolean | { status: string } | undefined;
+  componentFolders: ComponentFolder[] | undefined;
 }
 
 export type Token = string | undefined;

package/lib/utils/generateJsDriver.ts

@@ -24,10 +24,6 @@ const stringifySourceId = (projectId: string) =>
 
 // TODO: support ESM
 export function generateJsDriver(sources: Source[]) {
-  const fileNames = fs
-    .readdirSync(consts.TEXT_DIR)
-    .filter((fileName) => /\.json$/.test(fileName));
-
   const sourceIdsByName: Record<string, string> = sources.reduce(
     (agg, source) => {
       if (source.fileName) {
@@ -39,7 +35,14 @@ export function generateJsDriver(sources: Source[]) {
     {}
   );
 
-  const data = fileNames.reduce(
+  const projectFileNames = fs
+    .readdirSync(consts.TEXT_DIR)
+    .filter(
+      (fileName) => /\.json$/.test(fileName) && !/^components__/.test(fileName)
+    );
+
+  type DriverFile = Record<string, Record<string, string | object>>;
+  const data: DriverFile = projectFileNames.reduce(
     (obj: Record<string, Record<string, string>>, fileName) => {
       const [sourceId, rest] = fileName.split("__");
       const [variantApiId] = rest.split(".");
@@ -57,9 +60,40 @@ export function generateJsDriver(sources: Source[]) {
     {}
   );
 
+  // Create arrays of stringified "...require()" statements,
+  // each of which corresponds to one of the component files
+  // (which are created on a per-component-folder basis)
+  const componentData: Record<string, string[]> = {};
+  sources
+    .filter((s) => s.type === "components")
+    .forEach((componentSource) => {
+      if (componentSource.type !== "components") return;
+      componentData[componentSource.variant] ??= [];
+      componentData[componentSource.variant].push(
+        `...require('./${componentSource.fileName}')`
+      );
+    });
+  // Convert each array of stringified "...require()" statements
+  // into a unified string, and set it on the final data object
+  // that will be written to the driver file
+  Object.keys(componentData).forEach((key) => {
+    data.ditto_component_library ??= {};
+
+    let str = "{";
+    componentData[key].forEach((k, i) => {
+      str += k;
+      if (i < componentData[key].length - 1) str += ", ";
+    });
+    str += "}";
+    data.ditto_component_library[key] = str;
+  });
+
   let dataString = `module.exports = ${JSON.stringify(data, null, 2)}`
     // remove quotes around require statements
-    .replace(/"require\((.*)\)"/g, "require($1)");
+    .replace(/"require\((.*)\)"/g, "require($1)")
+    // remove quotes around opening & closing curlies
+    .replace(/"\{/g, "{")
+    .replace(/\}"/g, "}");
 
   const filePath = path.resolve(consts.TEXT_DIR, "index.js");
   fs.writeFileSync(filePath, dataString, { encoding: "utf8" });

package/package.json

@@ -1,11 +1,11 @@
 {
   "name": "@dittowords/cli",
-  "version": "4.0.0-alpha.0",
+  "version": "4.0.1-alpha.0",
   "description": "Command Line Interface for Ditto (dittowords.com).",
   "license": "MIT",
   "main": "bin/index.js",
   "scripts": {
-    "prepublish": "etsc && sentry-cli sourcemaps inject ./bin && npx sentry-cli sourcemaps upload ./bin --release=\"$(cat package.json | jq -r '.version')\"",
+    "prepublishOnly": "ENV=production etsc && sentry-cli sourcemaps inject ./bin && npx sentry-cli sourcemaps upload ./bin --release=\"$(cat package.json | jq -r '.version')\"",
     "prepare": "husky install",
     "start": "etsc && node bin/ditto.js",
     "sync": "etsc && node bin/ditto.js pull",