@catladder/cli 0.0.0-refactore-catenv-command-validates-argument-78d05852 → 0.0.0-reference-database-url-1befbbee

package/package.json

@@ -1,9 +1,10 @@
 {
   "name": "@catladder/cli",
-  "description": "Panter cli tool for Kubernetes",
+  "description": "Panter cli tool for cloud CI/CD and DevOps",
   "author": "Marco Wettstein <maw@panter.ch>",
-  "homepage": "https://git.panter.ch/catladder/catladder/#readme",
+  "homepage": "https://catladder.git.panter.biz/catladder/docs",
   "bugs": "https://git.panter.ch/catladder/catladder/issues",
+  "readme": "https://git.panter.ch/catladder/catladder/-/blob/main/cli/README.md",
   "repository": {
     "type": "git",
     "url": "git+https://git.panter.ch/catladder/catladder.git",
@@ -52,14 +53,14 @@
     }
   ],
   "license": "MIT",
-  "version": "0.0.0-refactore-catenv-command-validates-argument-78d05852",
+  "version": "0.0.0-reference-database-url-1befbbee",
   "scripts": {
     "lint": "eslint \"src/**/*.ts\"",
     "lint:fix": "eslint \"src/**/*.ts\" --fix",
     "test": "yarn run lint",
     "build": "yarn build:compile && yarn build:bundle",
     "build:bundle": "./scripts/bundle",
-    "build:compile": "yarn tsc",
+    "build:compile": "yarn tsc && yarn tsc-alias",
     "build:watch": "yarn tsc -w"
   },
   "bin": {
@@ -67,49 +68,48 @@
     "catenv": "./bin/catenv",
     "catladder": "./bin/catladder"
   },
+  "types": "dist/cli/src/index.d.ts",
+  "main": "dist/cli/src/index.js",
   "engines": {
-    "node": ">=12.0.0"
-  },
-  "dependencies": {
-    "ts-node": "^10.9.1"
+    "node": ">=18.0.0"
   },
   "devDependencies": {
-    "@catladder/pipeline": "0.0.0-refactore-catenv-command-validates-argument-78d05852",
     "@gitbeaker/rest": "^39.28.0",
     "@kubernetes/client-node": "^0.16.2",
     "@tsconfig/node14": "^1.0.1",
-    "@types/common-tags": "^1.8.0",
+    "@types/common-tags": "^1.8.4",
     "@types/fs-extra": "^5.0.4",
-    "@types/js-yaml": "^4.0.5",
-    "@types/lodash": "^4.14.185",
+    "@types/lodash": "^4.17.7",
     "@types/memoizee": "^0.4.2",
-    "@types/node-fetch": "^2.1.4",
-    "@types/request": "^2.48.1",
+    "@types/node-fetch": "^2.6.11",
+    "@types/request": "^2.48.12",
     "@types/tmp": "^0.2.3",
     "@types/update-notifier": "^2.5.0",
     "@types/vorpal": "^1.12.0",
     "@types/websocket": "^0.0.40",
-    "@types/yargs": "^13.0.0",
+    "@types/yargs": "^17.0.33",
     "child-process-promise": "^2.2.1",
     "clipboardy": "^2.0.0",
     "command-exists-promise": "^2.0.2",
-    "common-tags": "^1.8.0",
+    "common-tags": "^1.8.2",
     "connection-string-parser": "^1.0.4",
-    "dayjs": "^1.10.4",
-    "eslint": "^8.7.0",
+    "date-fns": "^3.6.0",
     "fs-extra": "^7.0.1",
     "gitlab": "^4.4.1",
-    "js-yaml": "^4.1.0",
     "lodash": "^4.17.21",
-    "memoizee": "^0.4.14",
+    "memoizee": "^0.4.17",
     "node-fetch": "^2.3.0",
     "open": "^8.4.0",
-    "prettier": "^2.5.1",
-    "prettier-plugin-sh": "^0.14.0",
+    "prettier": "^3.3.3",
     "tmp-promise": "^2.0.2",
-    "ts-results-es": "^4.1.0-alpha.1",
-    "typescript": "^4.5.4",
-    "vorpal": "^1.12.0"
+    "ts-results-es": "^4.2.0",
+    "tsc-alias": "^1.8.10",
+    "typescript": "^5.4.5",
+    "vorpal": "^1.12.0",
+    "yaml": "^2.5.0"
+  },
+  "dependencies": {
+    "tsx": "^4.19.0"
   },
   "stableVersion": "0.0.0"
 }

package/scripts/bundle

@@ -1,6 +1,6 @@
 #!/bin/bash
-ncc build dist/catenv.js -o dist/bundles/catenv -e 'ts-node' $( ((SHOULD_MINIFY == 1)) && printf %s '-m')
+ncc build dist/cli/src/catenv.js -o dist/bundles/catenv -e 'tsx/cjs/api' $( ((SHOULD_MINIFY == 1)) && printf %s '-m')
 
-ncc build dist/cli.js -o dist/bundles/cli -e 'ts-node' $( ((SHOULD_MINIFY == 1)) && printf %s '-m')
+ncc build dist/cli/src/cli.js -o dist/bundles/cli -e 'tsx/cjs/api' $( ((SHOULD_MINIFY == 1)) && printf %s '-m')
 
-ncc build dist/catci.js -o dist/bundles/catci -e 'ts-node' $( ((SHOULD_MINIFY == 1)) && printf %s '-m')
+ncc build dist/cli/src/catci.js -o dist/bundles/catci -e 'tsx/cjs/api' $( ((SHOULD_MINIFY == 1)) && printf %s '-m')

package/src/apps/catci/commands/security/auditDocument.ts

@@ -22,7 +22,7 @@ const rows = [
     t.description,
     "",
     t.more,
-  ])
+  ]),
 );
 
 function makeTable(rows: string[][]) {
@@ -33,7 +33,7 @@ ${makeRow(rows[0], colWidths, " ")}
 ${makeRow(
   rows[0].map(() => ""),
   colWidths,
-  "-"
+  "-",
 )}
 ${rows
   .slice(1)
@@ -45,7 +45,7 @@ ${rows
 function calculateColumnWidths(rows: string[][]) {
   const columnCount = rows[0].length;
   return Array.from({ length: columnCount }, (_, i) => i).map((columnIndex) =>
-    Math.max(...rows.map((row) => row[columnIndex].length))
+    Math.max(...rows.map((row) => row[columnIndex].length)),
   );
 }
 
@@ -115,7 +115,7 @@ export function evaluateDocument(document: string): SecurityEvaluation {
       !isUnknown &&
       !answer.includes(checkPlaceholder) &&
       !responsibles.some((responsible) =>
-        responsible.includes(responsiblePlaceholder)
+        responsible.includes(responsiblePlaceholder),
       );
     const isSecured = !isUnknown && isAnswered && answer.includes(checkYes);
 

package/src/apps/catci/commands/security/commands.ts

@@ -21,7 +21,7 @@ export default function (vorpal: Vorpal) {
 type ActionFunc = (args: Vorpal.Args) => Promise<void>;
 
 function resultAsExitCode(
-  func: (args: Vorpal.Args) => Promise<Result<unknown, unknown>>
+  func: (args: Vorpal.Args) => Promise<Result<unknown, unknown>>,
 ): ActionFunc {
   return async (args: Vorpal.Args) => {
     const result = await func(args);
@@ -40,7 +40,7 @@ async function commandCiJob(vorpal: Vorpal) {
 <main-branch> main branch name
 <project-id> project id to create security audit for
 <user-id> gitlab user id that will be assignee of the audit
-`
+`,
     )
     .action(
       resultAsExitCode(async (args) => {
@@ -49,7 +49,7 @@ async function commandCiJob(vorpal: Vorpal) {
         if (evaluation.isErr()) {
           console.log("could not evaluate security audit document");
           console.log(
-            "creating new merge request with security audit template..."
+            "creating new merge request with security audit template...",
           );
 
           const { token, mainBranch, projectId, userId } = args;
@@ -67,14 +67,14 @@ async function commandCiJob(vorpal: Vorpal) {
 
           if (mr.isErr()) {
             console.error(
-              `could not create merge request with security audit template: ${mr.error}`
+              `could not create merge request with security audit template: ${mr.error}`,
             );
             return mr;
           }
 
           console.log("security audit merge request created successfully");
           console.log(
-            `please finish the MR by updating SECURITY.md document: ${mr.value.web_url}`
+            `please finish the MR by updating SECURITY.md document: ${mr.value.web_url}`,
           );
           return Err("merge request created" as const);
         }
@@ -82,14 +82,14 @@ async function commandCiJob(vorpal: Vorpal) {
         if (evaluation.value.score.answeredTopics === 0) {
           console.error("audit document has no answered topics");
           console.error(
-            `please answer security topics in ${SECURITY_AUDIT_FILE_NAME} by adding responsible people and check/cross in the table`
+            `please answer security topics in ${SECURITY_AUDIT_FILE_NAME} by adding responsible people and check/cross in the table`,
           );
           return Err("audit document has no answered topics" as const);
         }
 
         console.log(makeSecurityAuditOverview(evaluation.value));
         return Ok({});
-      })
+      }),
     );
 }
 
@@ -97,7 +97,7 @@ async function commandEvaluate(vorpal: Vorpal) {
   vorpal
     .command(
       "security-audit-evaluate <path>",
-      "Evaluates security audit document in given <path>"
+      "Evaluates security audit document in given <path>",
     )
     .action(
       resultAsExitCode(async (args) => {
@@ -107,13 +107,13 @@ async function commandEvaluate(vorpal: Vorpal) {
         if (result.isErr()) {
           console.error(result.error);
           console.error(
-            `please make sure the security audit document ${SECURITY_AUDIT_FILE_NAME} is in the repository`
+            `please make sure the security audit document ${SECURITY_AUDIT_FILE_NAME} is in the repository`,
           );
         } else {
           console.log(makeSecurityAuditOverview(result.value));
         }
         return result;
-      })
+      }),
     );
 }
 
@@ -127,7 +127,7 @@ async function commandCreate(vorpal: Vorpal) {
 <main-branch> main branch name
 <project-id> project id to create security audit for
 <user-id> gitlab user id that will be assignee of the audit
-`
+`,
     )
     .action(
       resultAsExitCode(async (args) => {
@@ -147,15 +147,15 @@ async function commandCreate(vorpal: Vorpal) {
 
         if (result.isErr()) {
           console.error(
-            `could not create security audit merge request: ${result.error}`
+            `could not create security audit merge request: ${result.error}`,
           );
         } else {
           console.log("security audit merge request created successfully");
           console.log(
-            `please finish the MR by updating SECURITY.md document: ${result.value.web_url}`
+            `please finish the MR by updating SECURITY.md document: ${result.value.web_url}`,
           );
         }
         return result;
-      })
+      }),
     );
 }

package/src/apps/catci/commands/security/createSecurityAuditMergeRequest.ts

@@ -27,7 +27,7 @@ export async function createSecurityAuditMergeRequest({
         state: "opened",
         wip: "yes",
         labels: "security-audit",
-      })
+      }),
     )
   ).mapErr(() => `could not search for existing merge requests` as const);
   if (mrs.isErr()) return mrs;
@@ -35,11 +35,11 @@ export async function createSecurityAuditMergeRequest({
   const existingMr = mrs.value[0];
   if (existingMr)
     return Err(
-      `open merge request with security audit already exists: ${existingMr.web_url}`
+      `open merge request with security audit already exists: ${existingMr.web_url}`,
     );
 
   const auditTemplate = Result.wrap(() => makeTemplate()).mapErr(
-    () => "could not make security audit template document" as const
+    () => "could not make security audit template document" as const,
   );
   if (auditTemplate.isErr()) return auditTemplate;
 
@@ -48,8 +48,8 @@ export async function createSecurityAuditMergeRequest({
       api.Branches.create(
         projectId,
         makeDatedBranchName("chore/security-audit"),
-        mainBranch
-      )
+        mainBranch,
+      ),
     )
   ).mapErr((e) => {
     console.log(e);
@@ -70,8 +70,8 @@ export async function createSecurityAuditMergeRequest({
             content: auditTemplate.value,
             encoding: "text",
           },
-        ]
-      )
+        ],
+      ),
     )
   ).mapErr(() => "could not create commit" as const);
   if (commit.isErr()) return commit;
@@ -89,8 +89,8 @@ export async function createSecurityAuditMergeRequest({
           squash: true,
           labels: "security-audit",
           removeSourceBranch: true,
-        }
-      )
+        },
+      ),
     )
   ).mapErr(() => "could not create merge request" as const);
 

package/src/apps/catci/commands/security/evaluateSecurityAudit.ts

@@ -25,6 +25,6 @@ export function makeSecurityAuditOverview(evaluation: SecurityEvaluation) {
  📢 Answered topics: ${evaluation.score.answeredTopics}
  ❔ Unknown topics: ${evaluation.score.unknownTopics}
  📊 Rating: ${ratingToEmo(evaluation.score.rating)} ${
-    evaluation.score.rating
-  }/100`;
+   evaluation.score.rating
+ }/100`;
 }

package/src/apps/catci/commands/security/topics.json

@@ -15,8 +15,12 @@
     "more": ""
   },
   {
-    "description":
-      "Passwords are stored hashed with salt and salt is not stored in the repository",
+    "description": "No sensitive information (passwords, keys, user data, ...) is logged or traced",
+    "responsibles": 1,
+    "more": "[Logging guide](https://www.notion.so/panterch/Long-story-logging-022722bb878f4724ae5b49e17667b630?pvs=4#9e5a36b7158a4953b73ec6a345bd8989), [Tracing guide](https://www.notion.so/panterch/Long-story-tracing-d8a9ec1ac2ff4fa78cefa8991233224e?pvs=4#535121b5bf9741fbaf8654b4b64d879d)"
+  },
+  {
+    "description": "Passwords are stored hashed with salt and salt is not stored in the repository",
     "responsibles": 1,
     "more": "[guide](https://git.panter.ch/panter/security-guide/-/blob/main/docs/audit/hash.md)"
   },
@@ -41,8 +45,7 @@
     "more": "[guide](https://git.panter.ch/panter/security-guide/-/blob/main/docs/audit/cookies.md)"
   },
   {
-    "description":
-      "The app uses JWT with a secret and the secret is not stored in the repository",
+    "description": "The app uses JWT with a secret and the secret is not stored in the repository",
     "responsibles": 1,
     "more": "[guide](https://git.panter.ch/panter/security-guide/-/blob/main/docs/audit/cookies.md)"
   },
@@ -57,8 +60,7 @@
     "more": "[guide](https://git.panter.ch/panter/security-guide/-/blob/main/docs/audit/cors.md)"
   },
   {
-    "description":
-      "CSP headers are properly configured (no `unsafe-inline` or `unsafe-eval`)",
+    "description": "CSP headers are properly configured (no `unsafe-inline` or `unsafe-eval`)",
     "responsibles": 1,
     "more": "[guide](https://git.panter.ch/panter/security-guide/-/blob/main/docs/audit/csp.md)"
   },
@@ -68,8 +70,7 @@
     "more": "[guide](https://git.panter.ch/panter/security-guide/-/blob/main/docs/audit/dos.md)"
   },
   {
-    "description":
-      "YAML/XML parsing is not used or used YAML/XML parsers have disabled DTD",
+    "description": "YAML/XML parsing is not used or used YAML/XML parsers have disabled DTD",
     "responsibles": 1,
     "more": "[guide](https://git.panter.ch/panter/security-guide/-/blob/main/docs/audit/dos.md)"
   },
@@ -84,8 +85,7 @@
     "more": ""
   },
   {
-    "description":
-      "The app has disabled GraphQL introspection and schema registry",
+    "description": "The app has disabled GraphQL introspection and schema registry",
     "responsibles": 1,
     "more": "[guide](https://git.panter.ch/panter/security-guide/-/blob/main/docs/audit/graphql.md)"
   },
@@ -100,20 +100,17 @@
     "more": ""
   },
   {
-    "description":
-      "Cloud storage is (private) configured to not leak any sensitive data publicly",
+    "description": "Cloud storage is (private) configured to not leak any sensitive data publicly",
     "responsibles": 1,
     "more": ""
   },
   {
-    "description":
-      "Security Dashboard checks weekly vulnerable dependencies https://dep.panter.swiss/",
+    "description": "Security Dashboard checks weekly vulnerable dependencies https://dep.panter.swiss/",
     "responsibles": 1,
     "more": ""
   },
   {
-    "description":
-      "The app has `.well-known/security.txt` https://securitytxt.org/",
+    "description": "The app has `.well-known/security.txt` https://securitytxt.org/",
     "responsibles": 1,
     "more": ""
   }

package/src/apps/catenv/catenv.ts

@@ -1,20 +1,22 @@
-import { getProjectConfig } from '../../config/getProjectConfig';
-import { parseChoiceArgument } from '../../config/parseChoiceArgument';
-import { printVariables } from './printVariables';
-import { writeDotEnvFiles } from './writeDotEnvFiles';
-import { writeDTsFiles } from './writeEnvDTs';
+import { getProjectConfig } from "../../config/getProjectConfig";
+import { printVariables } from "./printVariables";
+import type { Choice } from "./types";
+import { writeDotEnvFiles } from "./writeDotEnvFiles";
+import { writeDTsFiles } from "./writeEnvDTs";
+import { generatePipelineFiles } from "@catladder/pipeline";
 
-export default async () => {
+export default async (choice?: Choice) => {
   const config = await getProjectConfig();
   if (!config) {
     return;
   }
 
-  const choice = parseChoiceArgument(config);
+  await Promise.all([
+    generatePipelineFiles(config, config.pipelineType ?? "gitlab"),
+    writeDotEnvFiles(config, choice),
 
-  await printVariables(config, choice);
+    writeDTsFiles(config, choice),
 
-  await writeDotEnvFiles(config, choice);
-
-  await writeDTsFiles(config, choice);
+    printVariables(config, choice),
+  ]);
 };

package/src/apps/catenv/printVariables.ts

@@ -1,6 +1,6 @@
 import type { Config } from "@catladder/pipeline";
 import { getEnvVarsResolved } from "../../config/getProjectConfig";
-import type { Choice, Variables } from "./types";
+import type { Choice } from "./types";
 import {
   getCurrentComponentAndEnvFromChoice,
   makeKeyValueString,
@@ -10,12 +10,12 @@ import {
 const getAllVariablesToPrint = async (config: Config, choice?: Choice) => {
   const { env, currentComponent } = await getCurrentComponentAndEnvFromChoice(
     config,
-    choice
+    choice,
   );
 
   if (currentComponent) {
     // don't print vars if dotenv is enabled
-    if (config.components[currentComponent].dotEnv) {
+    if (config.components[currentComponent].dotEnv ?? true) {
       return {};
     }
     return await getEnvVarsResolved(null, env, currentComponent);
@@ -26,7 +26,7 @@ const getAllVariablesToPrint = async (config: Config, choice?: Choice) => {
     return await Object.keys(config.components).reduce(
       async (acc, componentName) => {
         // don't print vars if dotenv is enabled
-        if (config.components[componentName].dotEnv) {
+        if (config.components[componentName].dotEnv ?? true) {
           return await acc;
         }
         const subappvars = await getEnvVarsResolved(null, env, componentName);
@@ -39,11 +39,11 @@ const getAllVariablesToPrint = async (config: Config, choice?: Choice) => {
             Object.entries(subappvars).map(([key, value]) => [
               `${sanitizeEnvVarName(componentName.toUpperCase())}_${key}`,
               value,
-            ])
+            ]),
           ),
         };
       },
-      Promise.resolve({} as Record<string, string>)
+      Promise.resolve({} as Record<string, string>),
     );
   }
 };
@@ -54,5 +54,8 @@ export const printVariables = async (config: Config, choice?: Choice) => {
   console.log(makeExportKeyValuestring(variables));
 };
 
-const makeExportKeyValuestring = (variables: Variables) =>
-  makeKeyValueString(variables, "export ");
+const makeExportKeyValuestring = (variables: Record<string, string>) =>
+  makeKeyValueString(variables, {
+    keyPrefix: "export ",
+    escapeOptions: { quoteMode: "always" },
+  });

package/src/apps/catenv/types.ts

@@ -1,6 +1,6 @@
+import type { VariableValue } from "@catladder/pipeline";
+
 export type Choice = {
   env?: string;
   componentName?: string;
 };
-
-export type Variables = Record<string, string>;

package/src/apps/catenv/utils.ts

@@ -1,11 +1,12 @@
-import type { Config } from "@catladder/pipeline";
+import type { Config, EscapeForDotEnvOptions } from "@catladder/pipeline";
 import { join } from "path";
 import { getGitRoot } from "../../utils/projects";
-import type { Choice, Variables } from "./types";
+import type { Choice } from "./types";
+import { escapeForDotEnv } from "@catladder/pipeline";
 export const getComponentFullPath = (
   gitRoot: string,
   config: Config,
-  componentName: string
+  componentName: string,
 ) => {
   return join(gitRoot, config.components[componentName].dir);
 };
@@ -13,13 +14,13 @@ const getCurrentComponentName = async (config: Config) => {
   const gitRoot = await getGitRoot();
   const currentDir = process.cwd();
   return Object.keys(config.components).find((c) =>
-    currentDir.startsWith(getComponentFullPath(gitRoot, config, c))
+    currentDir.startsWith(getComponentFullPath(gitRoot, config, c)),
   );
 };
 
 export const getCurrentComponentAndEnvFromChoice = async (
   config: Config,
-  choice?: Choice
+  choice?: Choice,
 ) => {
   const env = choice?.env ?? "local";
   const currentComponent =
@@ -31,19 +32,23 @@ export const getCurrentComponentAndEnvFromChoice = async (
   };
 };
 
-export const makeKeyValueString = (variables: Variables, keyPrefix = "") =>
+export type MakeKeyValueStringOptions = {
+  keyPrefix?: string;
+  escapeOptions?: EscapeForDotEnvOptions;
+};
+export const makeKeyValueString = (
+  variables: Record<string, string>,
+  { keyPrefix, escapeOptions }: MakeKeyValueStringOptions = {
+    keyPrefix: "",
+  },
+) =>
   Object.entries(variables)
-    .map(([key, value]) => `${keyPrefix}${key}='${value}'`)
+    // quotes are important, otherwise line breaks don't work properly
+    .map(
+      ([key, value]) =>
+        `${keyPrefix}${key}=${escapeForDotEnv(value, escapeOptions)}`,
+    )
     .join("\n");
 
-export const sanitizeMultiLine = (variables: Variables) => {
-  return Object.fromEntries(
-    Object.entries(variables).map(([key, value]) => [
-      key,
-      value.replaceAll("\n", "\\n"),
-    ])
-  );
-};
-
 export const sanitizeEnvVarName = (name: string) =>
   name.replace(/[\s\-.]+/g, "_");

package/src/apps/catenv/writeDotEnvFiles.ts

@@ -1,5 +1,4 @@
-import type { Config } from "@catladder/pipeline";
-import { writeFile } from "fs-extra";
+import { writeGeneratedFile, type Config } from "@catladder/pipeline";
 import { join } from "path";
 import { getEnvVarsResolved } from "../../config/getProjectConfig";
 import { getGitRoot } from "../../utils/projects";
@@ -8,17 +7,16 @@ import {
   getComponentFullPath,
   getCurrentComponentAndEnvFromChoice,
   makeKeyValueString,
-  sanitizeMultiLine,
 } from "./utils";
 
 export const writeDotEnvFiles = async (config: Config, choice?: Choice) => {
   const { env, currentComponent } = await getCurrentComponentAndEnvFromChoice(
     config,
-    choice
+    choice,
   );
 
   const componentsWithEnabledDotEnvWrite = Object.entries(config.components)
-    .filter(([, component]) => component?.dotEnv)
+    .filter(([, component]) => component?.dotEnv ?? true) // when set to true or "local"
     .map(([componentName]) => componentName);
 
   const componentsToActuallyWriteDotEnvNow = currentComponent
@@ -33,12 +31,9 @@ export const writeDotEnvFiles = async (config: Config, choice?: Choice) => {
     delete variables["_ALL_ENV_VAR_KEYS"];
     const componentDir = getComponentFullPath(gitRoot, config, componentName);
     const filePath = join(componentDir, ".env");
-    // many .dotenv don't like multiline values, so we sanitize them here
-    const variablesSanitized = sanitizeMultiLine(variables);
-    await writeFile(
-      filePath,
-      "# automatically created by catladder. Do not modify!\n\n" +
-        makeKeyValueString(variablesSanitized)
-    );
+
+    await writeGeneratedFile(filePath, makeKeyValueString(variables), {
+      commentChar: "#",
+    });
   }
 };