@platforma-sdk/tengo-builder 4.0.28 → 4.1.1

package/src/commands/imports.ts

@@ -0,0 +1,110 @@
+import { Command, Option } from "commander";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { createLogger, getTengoFiles, pathType } from "../compiler/util";
+import { dropImportLines, findUnusedImports } from "../compiler/imports";
+import { resolvePackageJsonRoot } from "../compiler/main";
+import * as opts from "../shared/basecmd";
+
+// The same limit the tags generation uses: deep enough for any real source
+// tree, shallow enough to keep a scan started in a wrong place cheap.
+const maxScanDepth = 9;
+
+export default function importsCommand(): Command {
+  const cmd = new Command("imports").description("find imports tengo sources do not use");
+
+  cmd.argument("[paths...]", "source paths to scan (defaults to ./src)");
+  opts.addOptions(cmd, opts.GlobalOptions());
+  cmd.addOption(new Option("--check", "report unused imports and fail, change no file"));
+  cmd.addOption(new Option("--fix", "remove unused imports from the sources"));
+
+  cmd.action(async (paths: string[], o) => {
+    const logger = createLogger(o.logLevel as string);
+
+    if (o.check && o.fix) {
+      cmd.error("error: options '--check' and '--fix' cannot be used together");
+    }
+
+    // Without an explicit mode: fix the sources when a human runs the command,
+    // report and fail when CI does. CI must never commit a change of its own.
+    const fix = o.fix ? true : o.check ? false : !process.env.CI;
+
+    const roots = paths.length == 0 ? ["./src"] : paths;
+    const localPackageName = getLocalPackageName();
+
+    let hasUnusedImports = false;
+    let hasBrokenSources = false;
+
+    for (const file of collectSources(roots)) {
+      const src = fs.readFileSync(file).toString();
+
+      let unused;
+      try {
+        unused = findUnusedImports(logger, src, localPackageName, file);
+      } catch (err: unknown) {
+        // A source we cannot parse is a problem for 'pl-tengo check' to
+        // report in full. Here we keep the file as it is and move on, so a
+        // single broken source does not block the cleanup of all the others.
+        logger.error(err);
+        hasBrokenSources = true;
+        continue;
+      }
+
+      if (unused.length == 0) {
+        continue;
+      }
+
+      hasUnusedImports = true;
+      logger.warn(`${file}: unused imports: ${unused.map((im) => im.alias).join(", ")}`);
+
+      if (fix) {
+        logger.info(`fixing ${file}...`);
+        fs.writeFileSync(
+          file,
+          dropImportLines(
+            src,
+            unused.map((im) => im.lineNo),
+          ),
+        );
+      }
+    }
+
+    if (hasBrokenSources || (hasUnusedImports && !fix)) {
+      process.exit(1);
+    }
+  });
+
+  return cmd;
+}
+
+/** Package name of the sources we scan. Used to resolve local ':<item>'
+ *  imports. An empty name keeps the parser working for a directory that is
+ *  not a package. */
+function getLocalPackageName(): string {
+  try {
+    const packageJson = fs.readFileSync(resolvePackageJsonRoot(process.cwd())).toString();
+    return (JSON.parse(packageJson) as { name?: string }).name ?? "";
+  } catch {
+    return "";
+  }
+}
+
+function collectSources(roots: string[]): string[] {
+  const sources: string[] = [];
+
+  for (const root of roots) {
+    const type = pathType(root);
+    if (type === "dir") {
+      sources.push(...getTengoFiles(path.resolve(root), maxScanDepth));
+      continue;
+    }
+    if (type === "file") {
+      sources.push(path.resolve(root));
+      continue;
+    }
+
+    throw new Error(`source path not found: ${root}`);
+  }
+
+  return sources;
+}

package/src/compiler/imports.test.ts

@@ -0,0 +1,181 @@
+import { expect, describe, test } from "vitest";
+import { dropImportLines, findUnusedImports } from "./imports";
+import { createLogger } from "./util";
+
+const logger = createLogger("error");
+
+const unusedAliases = (src: string, localPackageName = "current-package"): string[] =>
+  findUnusedImports(logger, src, localPackageName, "test.lib.tengo").map((im) => im.alias);
+
+describe("findUnusedImports", () => {
+  test("keeps an import the code uses", () => {
+    const src = `ll := import(":ll")
+
+doSomething := func() {
+	ll.assert(true, "always")
+}
+`;
+    expect(unusedAliases(src)).toEqual([]);
+  });
+
+  test("reports an import nothing uses", () => {
+    const src = `ll := import(":ll")
+sets := import(":sets")
+
+doSomething := func() {
+	ll.assert(true, "always")
+}
+`;
+    expect(unusedAliases(src)).toEqual(["sets"]);
+  });
+
+  test("reports standard library imports too", () => {
+    const src = `json := import("json")
+text := import("text")
+
+doSomething := func(s) {
+	return text.split(s, ",")
+}
+`;
+    expect(unusedAliases(src)).toEqual(["json"]);
+  });
+
+  test("a single line comment is not a usage", () => {
+    const src = `validation := import(":validation")
+
+doSomething := func(spec) {
+	// @TODO fix validation
+	// validation.assertValidJson(spec, schema)
+	return spec
+}
+`;
+    expect(unusedAliases(src)).toEqual(["validation"]);
+  });
+
+  test("a comment block is not a usage", () => {
+    const src = `maps := import(":maps")
+
+/*
+	maps.clone(m) is what this used to do.
+*/
+doSomething := func(m) {
+	return m
+}
+`;
+    expect(unusedAliases(src)).toEqual(["maps"]);
+  });
+
+  test("a compiler option line is not a usage", () => {
+    const src = `//tengo:hash_override 8ad4b0c4-1f31-4c4a-9b53-c2e50d2f0d9e
+ll := import(":ll")
+
+doSomething := func() {
+	return 1
+}
+`;
+    expect(unusedAliases(src)).toEqual(["ll"]);
+  });
+
+  test("a longer name that ends with the alias is not a usage", () => {
+    const src = `ll := import(":ll")
+
+doSomething := func(xll) {
+	return xll.value
+}
+`;
+    expect(unusedAliases(src)).toEqual(["ll"]);
+  });
+
+  test("a usage inside a multiline statement counts", () => {
+    const src = `ll := import(":ll")
+exec := import(":exec")
+
+doSomething := func() {
+	return exec.builder().
+		cmd(ll.toStrings(["a"])).
+		run()
+}
+`;
+    expect(unusedAliases(src)).toEqual([]);
+  });
+
+  test("a module the code only passes around counts as unused", () => {
+    // We ask for the dot on purpose: keeping an import too long costs
+    // nothing, dropping one the source needs breaks the build.
+    const src = `text := import("text")
+
+doSomething := func() {
+	return apply(text)
+}
+`;
+    expect(unusedAliases(src)).toEqual(["text"]);
+  });
+
+  test("reports the line of each import", () => {
+    const src = `ll := import(":ll")
+sets := import(":sets")
+`;
+    expect(findUnusedImports(logger, src, "current-package", "test.lib.tengo")).toEqual([
+      { alias: "ll", module: ":ll", lineNo: 0 },
+      { alias: "sets", module: ":sets", lineNo: 1 },
+    ]);
+  });
+
+  test("a source without imports gives nothing", () => {
+    expect(unusedAliases("doSomething := func() { return 1 }\n")).toEqual([]);
+  });
+
+  test("throws with the source name and the line for a source it cannot parse", () => {
+    const src = `assets := import("@platforma-sdk/workflow-tengo:assets")
+
+softwareName := "sw:main"
+sw := assets.importSoftware(softwareName)
+`;
+    expect(() => unusedAliases(src)).toThrow("[line 4 in test.lib.tengo]");
+  });
+});
+
+describe("dropImportLines", () => {
+  test("removes only the given lines", () => {
+    const src = `ll := import(":ll")
+sets := import(":sets")
+
+doSomething := func() {
+	ll.assert(true, "always")
+}
+`;
+    expect(dropImportLines(src, [1])).toEqual(`ll := import(":ll")
+
+doSomething := func() {
+	ll.assert(true, "always")
+}
+`);
+  });
+
+  test("keeps comments, blank lines and the trailing newline", () => {
+    const src = `// header comment
+
+json := import("json")
+
+/* a block
+   comment */
+doSomething := func() {
+	return 1
+}
+`;
+    expect(dropImportLines(src, [2])).toEqual(`// header comment
+
+
+/* a block
+   comment */
+doSomething := func() {
+	return 1
+}
+`);
+  });
+
+  test("keeps the source as it is when there is nothing to remove", () => {
+    const src = `ll := import(":ll")\n`;
+    expect(dropImportLines(src, [])).toEqual(src);
+  });
+});

package/src/compiler/imports.ts

@@ -0,0 +1,96 @@
+// Unused import detection for Tengo sources.
+//
+// The check runs on the parser's own line results, not on the raw text: a
+// comment line comes back empty, so an alias that appears only in a comment
+// correctly counts as unused.
+
+import type { MiLogger } from "@milaboratories/ts-helpers";
+import type { ImportInfo, sourceParserContext } from "./source";
+import { parseSingleSourceLine } from "./source";
+
+/** A module import statement and the line it is on. */
+export type TengoImport = ImportInfo & {
+  /** Line index in the source, counted from 0. */
+  lineNo: number;
+};
+
+function newParserContext(): sourceParserContext {
+  return {
+    isInCommentBlock: false,
+    canDetectOptions: true,
+    artifactImportREs: new Map(),
+    importLikeREs: new Map(),
+    multilineStatement: "",
+    lineNo: 0,
+  };
+}
+
+function escapeForRegExp(s: string): string {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/**
+ * Find imports the source does not use.
+ *
+ * An import is used when its alias appears with a dot somewhere else in the
+ * code: 'text.split(...)'. A module passed around as a value, say
+ * 'myFunc(text)', counts as unused. This is deliberate: the cost of keeping an
+ * import the source no longer needs is nothing, while the cost of dropping one
+ * it does need is a broken build.
+ *
+ * Throws when the source does not parse, with the same line context the
+ * compiler reports.
+ */
+export function findUnusedImports(
+  logger: MiLogger,
+  src: string,
+  localPackageName: string,
+  srcName: string,
+): TengoImport[] {
+  const imports: TengoImport[] = [];
+
+  // Lines as the parser sees them: comments are empty, code is untouched.
+  const codeLines: string[] = [];
+
+  let context = newParserContext();
+
+  const lines = src.split("\n");
+  for (const line of lines) {
+    context.lineNo++;
+
+    try {
+      const result = parseSingleSourceLine(logger, line, context, localPackageName, false);
+      context = result.context;
+      codeLines.push(result.line);
+
+      if (result.moduleImport) {
+        imports.push({ ...result.moduleImport, lineNo: context.lineNo - 1 });
+      }
+    } catch (error: unknown) {
+      const err = error as Error;
+      throw new Error(`[line ${context.lineNo} in ${srcName}]: ${err.message}\n\t${line}`, {
+        cause: err,
+      });
+    }
+  }
+
+  return imports.filter((im) => {
+    const usageRE = new RegExp(`\\b${escapeForRegExp(im.alias)}\\s*\\.`);
+    return !codeLines.some((code, lineNo) => lineNo !== im.lineNo && usageRE.test(code));
+  });
+}
+
+/**
+ * Drop the given lines from the source.
+ *
+ * The source is cut as it was written: comments, blank lines and formatting
+ * outside the dropped lines stay byte for byte the same. An import is always a
+ * single line, as the parser accepts it only as an assignment.
+ */
+export function dropImportLines(src: string, lineNos: number[]): string {
+  const drop = new Set(lineNos);
+  return src
+    .split("\n")
+    .filter((_, lineNo) => !drop.has(lineNo))
+    .join("\n");
+}

package/src/compiler/source.ts

@@ -238,6 +238,9 @@ export type lineProcessingResult = {
   context: sourceParserContext;
   artifacts: TypedArtifactName[];
   option: CompilerOption | undefined;
+  /** Set when the line is a module import, for every module: Platforma
+   *  artifacts, `plapi` and the Tengo standard library alike. */
+  moduleImport?: ImportInfo;
 };
 
 export function parseSingleSourceLine(
@@ -347,7 +350,7 @@ function processModuleImport(
         ["software", newGetSoftwareInfoRE(iInfo.alias)],
       ]);
     }
-    return { line: originalLine, context, artifacts: [], option: undefined };
+    return { line: originalLine, context, artifacts: [], option: undefined, moduleImport: iInfo };
   }
 
   if (
@@ -391,7 +394,7 @@ function processModuleImport(
   const artifact = parseArtifactName(iInfo.module, "library", localPackageName);
   if (!artifact) {
     // not a Platforma Tengo library import
-    return { line: originalLine, context, artifacts: [], option: undefined };
+    return { line: originalLine, context, artifacts: [], option: undefined, moduleImport: iInfo };
   }
 
   if (globalizeImports) {
@@ -401,7 +404,13 @@ function processModuleImport(
     );
   }
 
-  return { line: originalLine, context, artifacts: [artifact], option: undefined };
+  return {
+    line: originalLine,
+    context,
+    artifacts: [artifact],
+    option: undefined,
+    moduleImport: iInfo,
+  };
 }
 
 function processAssetImport(
@@ -495,7 +504,7 @@ function processAssetImport(
   return { line: originalLine, context, artifacts: [], option: undefined };
 }
 
-interface ImportInfo {
+export interface ImportInfo {
   module: string; // the module name without wrapping quotes: import("<module>")
   alias: string; // the name of variable that keeps imported module: <alias> := import("<module>")
 }

package/src/compiler/util.ts

@@ -64,3 +64,33 @@ export function isUUID(uuid: string): boolean {
   const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
   return uuidRegex.test(uuid.toLowerCase());
 }
+
+/**
+ * Collect all '.tengo' files in the directory and its subdirectories.
+ *
+ * The depth limit keeps the walk cheap: without it a scan started in a wrong
+ * place can become unbearably slow and even get the node process OOM killed.
+ * Increase the limit if you need to.
+ */
+export function getTengoFiles(dir: string, depth: number): string[] {
+  if (depth === 0) {
+    return [];
+  }
+
+  const files = fs.readdirSync(dir, { withFileTypes: true });
+
+  let tengoFiles: string[] = [];
+  files.forEach((file) => {
+    const absPath = path.join(dir, file.name);
+
+    if (file.isDirectory()) {
+      tengoFiles = tengoFiles.concat(getTengoFiles(absPath, depth - 1));
+    }
+
+    if (!file.isDirectory() && file.name.endsWith(".tengo")) {
+      tengoFiles.push(absPath);
+    }
+  });
+
+  return tengoFiles;
+}