lage 1.6.4 → 1.6.7

package/CHANGELOG.json

@@ -2,7 +2,52 @@
   "name": "lage",
   "entries": [
     {
-      "date": "Fri, 01 Jul 2022 16:00:36 GMT",
+      "date": "Fri, 01 Jul 2022 16:25:06 GMT",
+      "tag": "lage_v1.6.7",
+      "version": "1.6.7",
+      "comments": {
+        "patch": [
+          {
+            "author": "renovate@whitesourcesoftware.com",
+            "package": "lage",
+            "commit": "3cee217a37ec7d80a90f9434f39826e758bbdd2a",
+            "comment": "Update dependency @types/node to v12.20.55"
+          }
+        ]
+      }
+    },
+    {
+      "date": "Fri, 01 Jul 2022 16:15:39 GMT",
+      "tag": "lage_v1.6.6",
+      "version": "1.6.6",
+      "comments": {
+        "patch": [
+          {
+            "author": "email not defined",
+            "package": "lage",
+            "commit": "7ecee913b1cd4e6427aeb2c5f6f1e0e1f3225713",
+            "comment": "Pin dependencies"
+          }
+        ]
+      }
+    },
+    {
+      "date": "Fri, 01 Jul 2022 16:11:45 GMT",
+      "tag": "lage_v1.6.5",
+      "version": "1.6.5",
+      "comments": {
+        "patch": [
+          {
+            "author": "renovate@whitesourcesoftware.com",
+            "package": "lage",
+            "commit": "d849123a63fd58a413980f4be7937dddf40d3bc2",
+            "comment": "Update dependency workspace-tools to ^0.21.0"
+          }
+        ]
+      }
+    },
+    {
+      "date": "Fri, 01 Jul 2022 16:00:40 GMT",
       "tag": "lage_v1.6.4",
       "version": "1.6.4",
       "comments": {

package/CHANGELOG.md

@@ -1,12 +1,36 @@
 # Change Log - lage
 
-This log was last generated on Fri, 01 Jul 2022 16:00:36 GMT and should not be manually modified.
+This log was last generated on Fri, 01 Jul 2022 16:25:06 GMT and should not be manually modified.
 
 <!-- Start content -->
 
+## 1.6.7
+
+Fri, 01 Jul 2022 16:25:06 GMT
+
+### Patches
+
+- Update dependency @types/node to v12.20.55 (renovate@whitesourcesoftware.com)
+
+## 1.6.6
+
+Fri, 01 Jul 2022 16:15:39 GMT
+
+### Patches
+
+- Pin dependencies (email not defined)
+
+## 1.6.5
+
+Fri, 01 Jul 2022 16:11:45 GMT
+
+### Patches
+
+- Update dependency workspace-tools to ^0.21.0 (renovate@whitesourcesoftware.com)
+
 ## 1.6.4
 
-Fri, 01 Jul 2022 16:00:36 GMT
+Fri, 01 Jul 2022 16:00:40 GMT
 
 ### Patches
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lage",
-  "version": "1.6.4",
+  "version": "1.6.7",
   "description": "A monorepo task runner",
   "repository": {
     "url": "https://github.com/microsoft/lage"
@@ -47,26 +47,26 @@
     "p-graph": "^1.1.1",
     "p-limit": "^3.1.0",
     "p-profiler": "^0.2.1",
-    "workspace-tools": "^0.18.4",
+    "workspace-tools": "^0.21.0",
     "yargs-parser": "^18.1.3"
   },
   "devDependencies": {
-    "@marp-team/marp-cli": "^0.20.0",
-    "@types/chalk": "^2.2.0",
-    "@types/cosmiconfig": "^6.0.0",
-    "@types/execa": "^2.0.0",
-    "@types/git-url-parse": "^9.0.0",
-    "@types/ioredis": "^4.27.7",
+    "@marp-team/marp-cli": "0.20.0",
+    "@types/chalk": "2.2.0",
+    "@types/cosmiconfig": "6.0.0",
+    "@types/execa": "2.0.0",
+    "@types/git-url-parse": "9.0.1",
+    "@types/ioredis": "4.28.10",
     "@types/jest": "^27.0.1",
-    "@types/node": "^12.0.0",
-    "@types/npmlog": "^4.1.2",
-    "@types/p-queue": "^3.2.1",
-    "@types/yargs-parser": "^15.0.0",
+    "@types/node": "12.20.55",
+    "@types/npmlog": "4.1.4",
+    "@types/p-queue": "3.2.1",
+    "@types/yargs-parser": "15.0.0",
     "beachball": "2.23.0",
-    "gh-pages": "^2.2.0",
+    "gh-pages": "2.2.0",
     "jest": "^27.2.0",
     "ts-jest": "^27.0.5",
-    "ts-node": "^8.10.2",
+    "ts-node": "8.10.2",
     "typescript": "^4.4.4"
   }
 }

package/src/args.ts

@@ -1,107 +0,0 @@
-import yargsParser, { Arguments } from "yargs-parser";
-
-export function arrifyArgs(args: { [key: string]: string | string[] }) {
-  const argsArray: string[] = [];
-  for (const [key, val] of Object.entries(args)) {
-    if (key === "--" && Array.isArray(val)) {
-      val.forEach((arg) => argsArray.push(arg));
-    } else if (Array.isArray(val)) {
-      for (const item of val) {
-        pushValue(key, item);
-      }
-    } else {
-      pushValue(key, val);
-    }
-  }
-
-  return argsArray;
-
-  function pushValue(key: string, value: string) {
-    let keyArg = "";
-
-    if (typeof value === "boolean") {
-      if (key.length === 1 && value) {
-        keyArg = `-${key}`;
-      } else if (value) {
-        keyArg = `--${key}`;
-      } else {
-        keyArg = `--no-${key}`;
-      }
-
-      argsArray.push(keyArg);
-    } else {
-      if (key.length === 1 && value) {
-        keyArg = `-${key}`;
-      } else if (value) {
-        keyArg = `--${key}`;
-      }
-
-      argsArray.push(keyArg, value);
-    }
-  }
-}
-
-export function getPassThroughArgs(
-  command: string[],
-  args: { [key: string]: string | string[] }
-) {
-  let result: string[] = [];
-
-  let lageArgs = [
-    "node",
-    "scope",
-    "since",
-    "cache",
-    "deps",
-    "resetCache",
-    "ignore",
-    "verbose",
-    "only",
-    "concurrency",
-    "profile",
-    "grouped",
-    "reporter",
-    "to",
-    "parallel",
-    "continue",
-    "safeExit",
-    "includeDependencies",
-    "logLevel",
-    "cacheKey",
-    "dist",
-    "experimentDist",
-    "skipLocalCache",
-    "_",
-  ];
-
-  if (command[0] === "cache") {
-    lageArgs = [...lageArgs, "clear", "prune"];
-  }
-
-  const filtered: { [key: string]: string | string[] } = {};
-
-  for (let [key, value] of Object.entries(args)) {
-    if (!lageArgs.includes(key)) {
-      filtered[key] = value;
-    }
-  }
-
-  result = result.concat(arrifyArgs(filtered));
-
-  return result;
-}
-
-export function parseArgs() {
-  return yargsParser(process.argv.slice(2), {
-    array: ["scope", "node", "ignore", "to"],
-    configuration: {
-      "populate--": true,
-      "strip-dashed": true,
-    },
-    string: ["cacheKey"],
-  });
-}
-
-export function validateInput(parsedArgs: Arguments) {
-  return parsedArgs._ && parsedArgs._.length > 0;
-}

package/src/cache/RemoteFallbackCacheProvider.ts

@@ -1,100 +0,0 @@
-import { ICacheStorage } from "backfill-config";
-import { getCacheStorageProvider, isCustomProvider } from "backfill-cache";
-import { Logger } from "backfill-logger";
-import { CacheOptions } from "../types/CacheOptions";
-import { logger } from "../logger";
-
-export type RemoteFallbackCacheProviderOptions = Pick<
-  CacheOptions,
-  "internalCacheFolder" | "cacheStorageConfig" | "writeRemoteCache" | "skipLocalCache"
->;
-
-/**
- * Remote Fallback Cache Provider
- *
- * This backfill cache provider will fallback to a remote cache provider if the local cache does not contain the item.
- * It will also automatically populate the local cache with the remote cache.
- */
-export class RemoteFallbackCacheProvider implements ICacheStorage {
-  private localCacheStorageProvider: ICacheStorage;
-  private remoteCacheStorageProvider?: ICacheStorage;
-
-  private static localHits: { [hash: string]: boolean } = {};
-  private static remoteHits: { [hash: string]: boolean } = {};
-
-  constructor(private cacheOptions: RemoteFallbackCacheProviderOptions, logger: Logger, cwd: string) {
-    this.localCacheStorageProvider = getCacheStorageProvider(
-      {
-        provider: "local",
-      },
-      cacheOptions.internalCacheFolder,
-      logger,
-      cwd
-    );
-
-    // Remote providers should have a provider name of something other than "local" OR it is
-    // a custom provider (currently S3 would be a custom provider)
-    const isRemoteProvider =
-      isCustomProvider(cacheOptions.cacheStorageConfig) ||
-      (typeof cacheOptions.cacheStorageConfig.provider === "string" &&
-        !cacheOptions.cacheStorageConfig.provider.includes("local"));
-
-    if (isRemoteProvider) {
-      logger.silly("remote provider enabled");
-
-      this.remoteCacheStorageProvider = getCacheStorageProvider(
-        cacheOptions.cacheStorageConfig,
-        cacheOptions.internalCacheFolder,
-        logger,
-        cwd
-      );
-    }
-  }
-
-  async fetch(hash: string) {
-    if (!this.cacheOptions.skipLocalCache) {
-      RemoteFallbackCacheProvider.localHits[hash] = await this.localCacheStorageProvider.fetch(hash);
-      logger.silly(`local cache fetch: ${hash} ${RemoteFallbackCacheProvider.localHits[hash]}`);
-    }
-
-    if (!RemoteFallbackCacheProvider.localHits[hash] && this.remoteCacheStorageProvider) {
-      RemoteFallbackCacheProvider.remoteHits[hash] = await this.remoteCacheStorageProvider.fetch(hash);
-      logger.silly(`remote fallback fetch: ${hash} ${RemoteFallbackCacheProvider.remoteHits[hash]}`);
-      return RemoteFallbackCacheProvider.remoteHits[hash];
-    }
-
-    return RemoteFallbackCacheProvider.localHits[hash];
-  }
-
-  async put(hash: string, filesToCache: string[]) {
-    const putPromises: Promise<void>[] = [];
-
-    // Write local cache if it doesn't already exist, or if the the hash isn't in the localHits
-    const shouldWriteLocalCache = !this.isLocalHit(hash) && !this.cacheOptions.skipLocalCache;
-
-    if (shouldWriteLocalCache) {
-      logger.silly(`local cache put: ${hash}`);
-      putPromises.push(this.localCacheStorageProvider.put(hash, filesToCache));
-    }
-
-    // Write to remote if there is a no hit in the remote cache, and remote cache storage provider, and that the "writeRemoteCache" config flag is set to true
-    const shouldWriteRemoteCache =
-      !this.isRemoteHit(hash) && !!this.remoteCacheStorageProvider && this.cacheOptions.writeRemoteCache;
-
-    if (shouldWriteRemoteCache) {
-      logger.silly(`remote fallback put: ${hash}`);
-      const remotePut = this.remoteCacheStorageProvider!.put(hash, filesToCache);
-      putPromises.push(remotePut);
-    }
-
-    await Promise.all(putPromises);
-  }
-
-  private isRemoteHit(hash) {
-    return hash in RemoteFallbackCacheProvider.remoteHits && RemoteFallbackCacheProvider.remoteHits[hash];
-  }
-
-  private isLocalHit(hash) {
-    return hash in RemoteFallbackCacheProvider.localHits && RemoteFallbackCacheProvider.localHits[hash];
-  }
-}

package/src/cache/backfill.ts

@@ -1,93 +0,0 @@
-import { getCacheConfig } from "./cacheConfig";
-import { logger } from "../logger";
-import { salt } from "./salt";
-import * as backfill from "backfill/lib/api";
-import { CacheOptions } from "../types/CacheOptions";
-
-export async function cacheHash(
-  id: string,
-  cwd: string,
-  root: string,
-  cacheOptions: CacheOptions,
-  args: any
-) {
-  const cacheConfig = getCacheConfig(cwd, cacheOptions);
-
-  const backfillLogger = backfill.makeLogger(
-    "error",
-    process.stdout,
-    process.stderr
-  );
-
-  const hashKey = salt(
-    cacheOptions.environmentGlob || ["lage.config.js"],
-    `${id}|${JSON.stringify(args)}`,
-    root,
-    cacheOptions.cacheKey
-  );
-
-  backfillLogger.setName(id);
-
-  try {
-    return await backfill.computeHash(
-      cwd,
-      backfillLogger,
-      hashKey,
-      cacheConfig
-    );
-  } catch {
-    // computeHash can throw exception when git is not installed or the repo hashes cannot be calculated with a staged file that is deleted
-    // lage will continue as if this package cannot be cached
-  }
-
-  return null;
-}
-
-export async function cacheFetch(
-  hash: string | null,
-  id: string,
-  cwd: string,
-  cacheOptions: CacheOptions
-) {
-  if (!hash) {
-    return false;
-  }
-
-  const cacheConfig = getCacheConfig(cwd, cacheOptions);
-  const backfillLogger = backfill.makeLogger(
-    "error",
-    process.stdout,
-    process.stderr
-  );
-
-  try {
-    return await backfill.fetch(cwd, hash, backfillLogger, cacheConfig);
-  } catch (e) {
-    logger.error(`${id} fetchBackfill ${e && (e as any).stack || e && (e as any).message || e}`);
-  }
-
-  return false;
-}
-
-export async function cachePut(
-  hash: string | null,
-  cwd: string,
-  cacheOptions: CacheOptions
-) {
-  if (!hash) {
-    return;
-  }
-
-  const cacheConfig = getCacheConfig(cwd, cacheOptions);
-  const backfillLogger = backfill.makeLogger(
-    "warn",
-    process.stdout,
-    process.stderr
-  );
-
-  try {
-    await backfill.put(cwd, hash, backfillLogger, cacheConfig);
-  } catch (e) {
-    // sometimes outputGlob don't match any files, so skipping this
-  }
-}

package/src/cache/cacheConfig.ts

@@ -1,33 +0,0 @@
-import { getEnvConfig, createDefaultConfig } from "backfill-config";
-import { Logger, makeLogger } from "backfill-logger";
-import { CacheOptions } from "../types/CacheOptions";
-import { RemoteFallbackCacheProvider } from "./RemoteFallbackCacheProvider";
-
-export function getCacheConfig(cwd: string, cacheOptions: CacheOptions) {
-  const defaultCacheConfig = createDefaultConfig(cwd);
-
-  // in lage, default mode is to CACHE locally
-  defaultCacheConfig.cacheStorageConfig.provider = "local";
-  
-
-  const logger = makeLogger("warn");
-  const envConfig = getEnvConfig(logger);
-
-  const configWithEnvOverrides: CacheOptions = {
-    ...defaultCacheConfig,
-    ...cacheOptions,
-    ...envConfig,
-    writeRemoteCache: cacheOptions.writeRemoteCache || !!process.env.LAGE_WRITE_REMOTE_CACHE
-  };
-  
-  const configWithFallback: CacheOptions = {
-    ...configWithEnvOverrides,
-    cacheStorageConfig: {
-      ...configWithEnvOverrides.cacheStorageConfig,
-      provider: (logger: Logger, cwd: string) => new RemoteFallbackCacheProvider(configWithEnvOverrides, logger, cwd),
-      name: "remote-fallback-provider"
-    },
-  };
-
-  return configWithFallback;
-}

package/src/cache/cacheTasks.ts

@@ -1,9 +0,0 @@
-export const CacheHashTask = "backfillHash";
-export const CacheFetchTask = "backfillFetch";
-export const CachePutTask = "backfillPut";
-
-export function isCacheTask(task: string) {
-  return (
-    task === CacheHashTask || task === CacheFetchTask || task === CachePutTask
-  );
-}

package/src/cache/salt.ts

@@ -1,52 +0,0 @@
-import * as path from "path";
-import * as crypto from "crypto";
-import * as fg from "fast-glob";
-import * as fs from "fs";
-
-let envHash: string[];
-
-export function salt(
-  environmentGlobFiles: string[],
-  command: string,
-  repoRoot: string,
-  customKey: string = ""
-): string {
-  return hashStrings([...getEnvHash(environmentGlobFiles, repoRoot), command, customKey]);
-}
-
-function getEnvHash(environmentGlobFiles: string[], repoRoot: string) {
-  if (!envHash) {
-    const newline = /\r\n|\r|\n/g;
-    const LF = "\n";
-    const files = fg.sync(environmentGlobFiles, {
-      cwd: repoRoot,
-    });
-
-    files.sort((a, b) => a.localeCompare(b));
-
-    const hashes = files.map((file) => {
-      const hasher = crypto.createHash("sha1");
-      hasher.update(file);
-
-      const fileBuffer = fs.readFileSync(path.join(repoRoot, file));
-      const data = fileBuffer.toString().replace(newline, LF);
-      hasher.update(data);
-
-      return hasher.digest("hex");
-    });
-
-    envHash = hashes;
-  }
-
-  return envHash;
-}
-
-function hashStrings(strings: string | string[]): string {
-  const hasher = crypto.createHash("sha1");
-  const anArray = typeof strings === "string" ? [strings] : strings;
-  const elements = [...anArray];
-  elements.sort((a, b) => a.localeCompare(b));
-  elements.forEach((element) => hasher.update(element));
-
-  return hasher.digest("hex");
-}

package/src/cli.ts

@@ -1,47 +0,0 @@
-import { getConfig } from "./config/getConfig";
-import { init } from "./command/init";
-import { run } from "./command/run";
-import { showHelp } from "./showHelp";
-import { logger } from "./logger";
-import { info } from "./command/info";
-import { initReporters } from "./logger/initReporters";
-import { version } from "./command/version";
-import { cache } from "./command/cache";
-import { worker } from "./command/worker";
-
-// Parse CLI args
-const cwd = process.cwd();
-try {
-  const config = getConfig(cwd);
-  const reporters = initReporters(config);
-
-  switch (config.command[0]) {
-    case "cache":
-      cache(cwd, config);
-      break;
-
-    case "init":
-      init(cwd);
-      break;
-
-    case "info":
-      info(cwd, config);
-      break;
-
-    case "version":
-      version();
-      break;
-
-    case "experiment-worker":
-      logger.info(`Lage worker - let's make it`);
-      worker(cwd, config);
-      break;
-
-    default:
-      logger.info(`Lage task runner - let's make it`);
-      run(cwd, config, reporters);
-      break;
-  }
-} catch (e) {
-  showHelp(e && (e as any).message);
-}

package/src/command/cache.ts

@@ -1,81 +0,0 @@
-import fs, { Stats } from "fs";
-import path from "path";
-import { PackageInfo } from "workspace-tools";
-import { logger } from "../logger";
-import { Config } from "../types/Config";
-import { getWorkspace } from "../workspace/getWorkspace";
-
-const MS_IN_A_DAY = 1000 * 60 * 60 * 24;
-
-export async function cache(cwd: string, config: Config) {
-  if (config.clear) {
-    logger.info("clearing cache, this may take a while");
-    clearCache(cwd, config);
-    logger.info("done!");
-  } else if (config.prune) {
-    logger.info("pruning cache, this may take a while");
-    pruneCache(cwd, config);
-    logger.info("done!");
-  } else {
-    logger.info("No options given to cache command. Try --clear or --prune");
-  }
-}
-
-function clearCache(cwd: string, config: Config) {
-  const workspace = getWorkspace(cwd, config);
-  const { allPackages } = workspace;
-  for (const info of Object.values(allPackages)) {
-    const cachePath = getCachePath(info);
-
-    if (fs.existsSync(cachePath)) {
-      const entries = fs.readdirSync(cachePath);
-      for (const entry of entries) {
-        logger.verbose(`clearing cache for ${info.name}`);
-
-        const entryPath = path.join(cachePath, entry);
-        const entryStat = fs.statSync(entryPath);
-        remove(entryPath, entryStat);
-      }
-    }
-  }
-}
-
-function pruneCache(cwd: string, config: Config) {
-  const prunePeriod = parseInt(config.prune) || 30;
-  const now = new Date();
-  const workspace = getWorkspace(cwd, config);
-  const { allPackages } = workspace;
-  for (const info of Object.values(allPackages)) {
-    const cachePath = getCachePath(info);
-
-    if (fs.existsSync(cachePath)) {
-      const entries = fs.readdirSync(cachePath);
-
-      for (const entry of entries) {
-        const entryPath = path.join(cachePath, entry);
-        const entryStat = fs.statSync(entryPath);
-
-        logger.verbose(`clearing cache for ${info.name}`);
-
-        if (
-          now.getTime() - entryStat.mtime.getTime() >
-          prunePeriod * MS_IN_A_DAY
-        ) {
-          remove(entryPath, entryStat);
-        }
-      }
-    }
-  }
-}
-
-function getCachePath(info: PackageInfo) {
-  return path.join(info.packageJsonPath, "../node_modules/.cache/backfill");
-}
-
-function remove(entryPath: string, entryStat: Stats) {
-  if (entryStat.isDirectory()) {
-    fs.rmdirSync(entryPath, { recursive: true });
-  } else {
-    fs.unlinkSync(entryPath);
-  }
-}