@wmfs/lerna-sync 1.38.0 → 1.39.0

package/lib/index.js

@@ -1,373 +1,106 @@
 const jsonfile = require('jsonfile')
 const path = require('path')
-const fs = require('fs')
-const glob = require('glob')
 const humanizeDuration = require('humanize-duration')
 const chalk = require('chalk')
-const getLocalGitInfo = require('git-repo-info')
-const dottie = require('dottie')
 const { graphql } = require('@octokit/graphql')
-const _ = require('lodash')
-const query = require('./github-graphql-query')
-const actionFunctions = require('./actions')
-const Messages = require('./utils/Messages')
+const discover = require('./discover')
+const inspect = require('./inspect')
+const planSync = require('./plan-sync')
+const { report, printPlan, describeResult } = require('./report')
+const { executePlan } = require('./execute-plan')
+const packageFoldersFromPatterns = require('./utils/package-folders-from-patterns')
 
 class LernaSync {
   constructor (options) {
-    this.options = options
-    this.lernaJson = jsonfile.readFileSync(path.join(this.options.monorepoPath, 'lerna.json'))
-  }
-
-  async discoverGithubRepos () {
-    const gitHubOrgName = this.options.gitHubOrgName
-    console.log(chalk.gray(`Downloading repo information from https://github.com/${gitHubOrgName}`))
-    let afterValue = null
-    let hasNextPage = true
-    while (hasNextPage) {
-      const result = await graphql(
-        query,
-        {
-          afterValue,
-          queryString: `org:${this.options.gitHubOrgName}`,
-          headers: {
-            authorization: `token ${this.options.gitHubToken}`
-          }
-        })
-      result.search.edges.forEach(
-        (edge) => {
-          const repoName = dottie.get(edge, 'node.name')
-          const historyEdges = dottie.get(edge, 'node.ref.target.history.edges')
-          if (historyEdges && historyEdges.length > 0) {
-            let packageJson = dottie.get(edge, 'node.object.text')
-            if (packageJson) {
-              packageJson = JSON.parse(packageJson)
-            }
-            this.applyRepo(
-              repoName,
-              {
-                gitHubSha: historyEdges[0].node.oid,
-                gitHubMessageHeadline: historyEdges[0].node.messageHeadline,
-                gitHubAuthor: historyEdges[0].node.author,
-                gitHubPackageJson: packageJson,
-                gitHubLernaPackageName: this.options.lernaPackageRouterFunction(packageJson)
-              }
-            )
-          }
-        }
-      )
-      hasNextPage = result.search.pageInfo.hasNextPage
-      afterValue = result.search.pageInfo.endCursor
-    }
-  }
-
-  collectLocalGitInfo () {
-    console.log(chalk.gray('Collecting local Git information'))
-    this.lernaJson.packages.forEach(
-      (relPackagePath) => {
-        const fullPackagePath = path.resolve(this.options.monorepoPath, relPackagePath + '/')
-        const packageRepoPaths = glob.sync(fullPackagePath, {
-          windowsPathsNoEscape: true
-        })
-        packageRepoPaths.forEach(
-          (repoPath) => {
-            const localRepoInfo = {
-              localGit: getLocalGitInfo(repoPath),
-              localLernaPackageName: path.basename(path.resolve(repoPath, '..'))
-            }
-
-            const packageJsonPath = path.join(repoPath, 'package.json')
-            try {
-              localRepoInfo.localPackageJson = jsonfile.readFileSync(packageJsonPath)
-            } catch (e) {
-              // Just ignore failure to grab package.json.
-              console.error(e)
-            }
-            this.applyRepo(
-              path.basename(repoPath),
-              localRepoInfo
-            )
-          }
-        )
+    this.lernaPackageRouterFunction = options.lernaPackageRouterFunction
+    this.cloneUrl = options.cloneUrl
+    this.updateLocalDefaultBranch = options.updateLocalDefaultBranch
+    this.verbose = options.verbose
+    this.dryRun = options.dryRun
+    this.monorepoPath = options.monorepoPath
+    this.concurrency = validateConcurrency(options.concurrency)
+    this.gitHubOrgName = options.gitHubOrgName
+    this.lernaJson = jsonfile.readFileSync(path.join(options.monorepoPath, 'lerna.json'))
+    this.graphql = graphql.defaults({
+      headers: {
+        authorization: `token ${options.gitHubToken}`
       }
-    )
-  }
-
-  applyRepo (repoName, info) {
-    if (Object.prototype.hasOwnProperty.call(this.repos, repoName)) {
-      const existingInfo = this.repos[repoName]
-      this.repos[repoName] = _.defaults(info, existingInfo)
-    } else {
-      this.repos[repoName] = info
-    }
+    })
   }
 
-  process () {
-    const processed = {
-      containingDirName: path.basename(this.options.monorepoPath),
-      packages: {},
-      flattenedPackages: [],
-      totalPackageCount: 0,
-      rejectedCount: 0,
-      actions: [],
-      allNpmPackageNames: [],
-      gitHubDeps: {
-        dependencyPackageNames: [],
-        devDependencyPackageNames: []
-      },
-      existingLocalDeps: {
-        dependencyPackageNames: [],
-        devDependencyPackageNames: []
-      }
-    }
-
-    function applyDeps (target, packageJson) {
-      if (packageJson) {
-        if (packageJson.dependencies) {
-          target.dependencyPackageNames = _.union(
-            target.dependencyPackageNames,
-            _.keys(packageJson.dependencies)
-          )
-        }
-        if (packageJson.devDependencies) {
-          target.devDependencyPackageNames = _.union(
-            target.devDependencyPackageNames,
-            _.keys(packageJson.devDependencies)
-          )
-        }
-      }
-    }
-
-    function applyNpmPackageName (packageJson) {
-      if (packageJson) {
-        const name = packageJson.name
-        if (processed.allNpmPackageNames.indexOf(name) === -1) {
-          processed.allNpmPackageNames.push(name)
-        }
-      }
-    }
+  async sync () {
+    const start = Date.now()
 
-    for (const [repoName, rawRepo] of Object.entries(this.repos)) {
-      const pkgName = rawRepo.gitHubLernaPackageName
-      if (pkgName) {
-        processed.flattenedPackages.push({ repoName, pkgName })
+    const packagePatterns = this.lernaJson.packages
+    const packageFolders = packageFoldersFromPatterns(packagePatterns)
 
-        let pkgInfo
-        if (!Object.prototype.hasOwnProperty.call(processed.packages, pkgName)) {
-          pkgInfo = {
-            name: pkgName,
-            repos: {}
-          }
-          processed.packages[pkgName] = pkgInfo
-        } else {
-          pkgInfo = processed.packages[pkgName]
-        }
+    // 1. Discover (remote)
+    // Make one paginated GraphQL query that returns the name,
+    // default branch, head commit and package.json
+    // for every repo in the org.
 
-        const rep = {
-          name: repoName,
-          gitHubSha: rawRepo.gitHubSha,
-          gitHubMessageHeadline: rawRepo.gitHubMessageHeadline,
-          gitHubAuthor: rawRepo.gitHubAuthor,
-          localSha: dottie.get(rawRepo, 'localGit.sha'),
-          localLernaPackageName: rawRepo.localLernaPackageName,
-          gitHubLernaPackageName: rawRepo.gitHubLernaPackageName,
-          githubPackageJsonDescription: dottie.get(rawRepo, 'gitHubPackageJson.description')
-        }
+    const remoteRepos = await discover(this.graphql, this.gitHubOrgName)
 
-        applyDeps(processed.existingLocalDeps, rawRepo.localPackageJson)
-        applyNpmPackageName(rawRepo.localPackageJson)
-        if (rep.gitHubSha === rep.localSha) {
-          rep.syncAction = 'SKIP'
-        } else if (!rep.localSha) {
-          rep.syncAction = 'CLONE'
-          applyDeps(processed.gitHubDeps, rawRepo.gitHubPackageJson)
-          applyNpmPackageName(rawRepo.gitHubPackageJson)
-          processed.actions.push(
-            {
-              type: 'CLONE',
-              config: {
-                packageName: rep.gitHubLernaPackageName,
-                repoName
-              }
-            }
-          )
-        } else {
-          rep.syncAction = 'PULL'
-          applyDeps(processed.gitHubDeps, rawRepo.gitHubPackageJson)
-          applyNpmPackageName(rawRepo.gitHubPackageJson)
-          processed.actions.push(
-            {
-              type: 'PULL',
-              config: {
-                packageName: rep.localLernaPackageName,
-                repoName
-              }
-            }
-          )
-        }
-        pkgInfo.repos[repoName] = rep
-        processed.totalPackageCount++
-      } else {
-        processed.rejectedCount++
-      }
-    }
+    // 2. Inspect (local)
+    // For each folder that already exists, read its
+    // current branch, whether it has uncommitted changes,
+    // its HEAD SHA and its folder name.
 
-    processed.newDependencies = _.difference(
-      _.union(processed.gitHubDeps.dependencyPackageNames, processed.gitHubDeps.devDependencyPackageNames),
-      _.union(processed.existingLocalDeps.dependencyPackageNames, processed.existingLocalDeps.devDependencyPackageNames)
+    const localRepos = await inspect.inspectLocalRepos(
+      this.monorepoPath,
+      this.lernaJson.packages,
+      { concurrency: this.concurrency }
     )
-    processed.newDependenciesWithoutLinkables = _.difference(
-      processed.newDependencies,
-      processed.allNpmPackageNames
-    )
-    return processed
-  }
 
-  outputSummary (info) {
-    console.log(chalk.underline('Lerna Sync summary'))
-    console.log('')
-    console.log(`/${info.containingDirName}`)
-
-    for (const [packageName, packageInfo] of Object.entries(info.packages)) {
-      console.log('')
-      console.log(`  /${packageName}`)
-      let sameCount = 0
-      for (const [repoName, repoInfo] of Object.entries(packageInfo.repos)) {
-        // Running an action might have revised its type, so find it in actions array
-        let syncAction
-        info.actions.forEach(
-          action => {
-            if (action.config.repoName === repoName) {
-              syncAction = action.type
-            }
-          }
-        )
-
-        if (syncAction && syncAction !== 'SKIP') {
-          let repoLine = `    /${chalk.white(chalk.bold(repoName))} `
-
-          if (syncAction === 'CLONE') {
-            repoLine += chalk.green(chalk.inverse('Clone'))
-          } else {
-            repoLine += chalk.cyan(chalk.inverse('Pull'))
-          }
-          console.log(repoLine)
-
-          if (syncAction === 'CLONE') {
-            console.log(`      ${chalk.green(repoInfo.githubPackageJsonDescription)}`)
-          }
-
-          let agoLabel
-          const rawAuthorDate = dottie.get(repoInfo, 'gitHubAuthor.date')
-          if (rawAuthorDate) {
-            const authorDate = new Date(rawAuthorDate)
-            agoLabel = humanizeDuration(
-              new Date() - authorDate,
-              {
-                largest: 1
-              }
-            )
-            agoLabel = `{${agoLabel} ago)`
-          }
-          console.log(`      ${chalk.yellow(repoInfo.gitHubAuthor.name + ':')} ${chalk.gray(agoLabel)}`)
-          console.log(chalk.magenta(`      ${repoInfo.gitHubMessageHeadline}`))
-        } else {
-          sameCount++
-        }
-      }
-      let sameLine = chalk.gray('    OK count: ')
-      sameLine += chalk.green(sameCount)
-      console.log(sameLine)
+    // 3. Plan (pure function)
+    // Take `(remoteRepos, localRepos, router)` and
+    // return a list of decisions, each with a reason.
+    // This stage does no I/O.
+
+    const plan = planSync.planSync({
+      remoteRepos,
+      localRepos,
+      router: this.lernaPackageRouterFunction,
+      orgName: this.gitHubOrgName,
+      packageFolders
+    })
+
+    // 4. Execute and report
+    // Run the decisions, then print a summary
+    // grouped by outcome.
+
+    if (this.dryRun) {
+      printPlan(plan, { verbose: this.verbose })
+      return { plan, results: [], errorCount: 0, attentionCount: 0 }
     }
-  }
 
-  outputSuggestedNextSteps (newDependencies, newDependenciesWithoutLinkables) {
-    console.log('')
-    if (newDependenciesWithoutLinkables.length === 0) {
-      if (newDependencies.length === 0) {
-        console.log(chalk.green('Good times, no Lerna bootstrapping/linking required!'))
-      } else {
-        console.log(chalk.yellow('You\'ll need to \'lerna link\' because synced-packages need links to:'))
-        newDependencies.forEach(
-          (packageName) => {
-            console.log(chalk.yellow(` - ${packageName}`))
-          }
-        )
+    const results = await executePlan(plan, {
+      monorepoPath: this.monorepoPath,
+      gitHubOrgName: this.gitHubOrgName,
+      concurrency: this.concurrency,
+      cloneUrl: this.cloneUrl,
+      updateLocalDefaultBranch: this.updateLocalDefaultBranch,
+      onResult: r => {
+        const { label, colour, detail } = describeResult(r)
+        const name = `${r.decision.folder}/${r.decision.repoName}`
+        console.log(`  ${colour(label.padEnd(11))} ${name}${detail ? chalk.gray(`  (${detail})`) : ''}`)
       }
-    } else {
-      console.log(chalk.red('Oh no! You need to \'lerna bootstrap\', because these are unheard of:'))
-      newDependenciesWithoutLinkables.forEach(
-        (packageName) => {
-          console.log(chalk.red(` - ${packageName}`))
-        }
-      )
-    }
-  }
+    })
 
-  async performActions (actions, messages) {
-    console.log('')
-    if (actions.length > 0) {
-      console.log(chalk.underline(`Identified ${actions.length} action${actions.length === 1 ? '' : 's'} to run`))
-      console.log('')
-      // let idx = 0
-      // const totalActionCount = actions.length
-      for (const action of actions) {
-        await actionFunctions[action.type](action, this.options, messages)
-      }
-    }
-    console.log('')
-  }
+    const { errorCount, attentionCount } = report(plan, results, { verbose: this.verbose })
+    console.log(`\nDone (${humanizeDuration(Date.now() - start, { largest: 2, round: true })}).`)
 
-  async sync () {
-    const messages = new Messages()
-    this.syncStart = new Date()
-    this.repos = {}
-    await this.discoverGithubRepos()
-    this.collectLocalGitInfo()
-    this.processed = this.process()
-    if (this.processed.actions.length > 0) {
-      await this.performActions(
-        this.processed.actions,
-        messages
-      )
-    }
-    this.outputSummary(this.processed)
-    messages.output()
-
-    const localNonMasterBranches = this.processed.flattenedPackages
-      .map(({ repoName, pkgName }) => {
-        const destPath = path.join(this.options.monorepoPath, pkgName, repoName)
-        const gitHeadPath = path.join(destPath, '.git', 'HEAD')
-
-        if (fs.existsSync(destPath)) {
-          if (fs.existsSync(gitHeadPath)) {
-            const currentBranch = fs.readFileSync(gitHeadPath, 'utf-8').trim().split('refs/heads/')[1]
-
-            if (currentBranch !== 'master') {
-              return { repoName, pkgName, currentBranch }
-            }
-          }
-        }
-
-        return null
-      })
-      .filter(p => p)
-
-    if (localNonMasterBranches.length > 0) {
-      console.log(chalk.bgYellow.gray('\n\n ----- Found some non-master branches, best double check that you should be on these! ----- '))
-      localNonMasterBranches.forEach(branch => {
-        console.log(chalk.yellow.underline(`${branch.repoName}:`))
-        console.log(chalk.yellow(`Branch name: ${branch.currentBranch}\n`))
-      })
-    }
+    return { plan, results, errorCount, attentionCount }
+  }
+}
 
-    this.outputSuggestedNextSteps(
-      this.processed.newDependencies,
-      this.processed.newDependenciesWithoutLinkables
-    )
-    this.syncEnd = new Date()
-    console.log('')
-    console.log(`Done (${humanizeDuration(this.syncEnd - this.syncStart)}).`)
+// undefined/null means "use each stage's default"; anything else must be a positive integer
+function validateConcurrency (value) {
+  if (value == null) return undefined
+  if (!Number.isInteger(value) || value < 1) {
+    throw new TypeError(`concurrency must be a positive integer, got ${JSON.stringify(value)}`)
   }
+  return value
 }
 
 module.exports = LernaSync

package/lib/inspect.js

@@ -0,0 +1,99 @@
+const fs = require('fs')
+const path = require('path')
+const { globSync } = require('glob')
+const jsonfile = require('jsonfile')
+const { tryGit } = require('./utils/git')
+const mapLimit = require('./utils/map-limit')
+
+async function inspectLocalRepos (monorepoPath, lernaPackagePatterns, { concurrency = 8 } = {}) {
+  const dirs = new Set()
+
+  for (const pattern of lernaPackagePatterns) {
+    // A trailing slash makes glob match directories only
+    const matches = globSync(`${pattern.replace(/\/+$/, '')}/`, { cwd: monorepoPath, absolute: true })
+    matches.forEach(m => dirs.add(path.resolve(m)))
+  }
+
+  return mapLimit([...dirs].sort(), concurrency, dir => inspectRepo(monorepoPath, dir))
+}
+
+async function inspectRepo (monorepoPath, repoPath) {
+  const record = {
+    path: repoPath,
+    repoName: path.basename(repoPath),
+    folder: path.relative(monorepoPath, path.dirname(repoPath)).split(path.sep).join('/'),
+    isGitRepo: false
+  }
+
+  // Each package folder sits inside the husk's own repo, so in a folder that
+  // isn't a repo of its own, git would happily answer for the husk instead.
+  // Only trust the result if this folder is the top level.
+  const toplevel = await tryGit(['rev-parse', '--show-toplevel'], repoPath)
+  if (!toplevel || !samePath(toplevel, repoPath)) {
+    return record
+  }
+
+  const [
+    branch,
+    headSha,
+    status,
+    originUrl,
+    gitPaths
+  ] = await Promise.all([
+    tryGit(['symbolic-ref', '--quiet', '--short', 'HEAD'], repoPath),
+    tryGit(['rev-parse', '--verify', '--quiet', 'HEAD'], repoPath),
+    tryGit(['status', '--porcelain'], repoPath),
+    tryGit(['remote', 'get-url', 'origin'], repoPath),
+    tryGit(['rev-parse', '--git-path', 'MERGE_HEAD', '--git-path', 'rebase-merge', '--git-path', 'rebase-apply'], repoPath)
+  ])
+
+  const statusLines = status === null ? null : status.split('\n').filter(Boolean)
+
+  const operationInProgress = (gitPaths || '')
+    .split('\n')
+    .filter(Boolean)
+    .some(p => fs.existsSync(path.resolve(repoPath, p)))
+
+  return {
+    ...record,
+    isGitRepo: true,
+    branch, // null when HEAD is detached
+    isDetached: branch === null,
+    headSha, // null in a repo with no commits yet
+    statusKnown: statusLines !== null,
+    trackedChanges: statusLines ? statusLines.filter(l => !l.startsWith('??')).length : 0,
+    untrackedFiles: statusLines ? statusLines.filter(l => l.startsWith('??')).length : 0,
+    operationInProgress, // mid-merge or mid-rebase
+    origin: parseGitHubRemote(originUrl),
+    packageJson: readJsonOrNull(path.join(repoPath, 'package.json'))
+  }
+}
+
+function samePath (a, b) {
+  const norm = p => {
+    const real = path.resolve(fs.realpathSync.native(p))
+    return process.platform === 'win32' ? real.toLowerCase() : real
+  }
+  try {
+    return norm(a) === norm(b)
+  } catch (e) {
+    return false
+  }
+}
+
+function parseGitHubRemote (url) {
+  if (!url) return null
+  // https://github.com/wmfs/repo(.git) or git@github.com:wmfs/repo(.git)
+  const m = url.match(/github\.com[/:]([^/]+)\/(.+?)(?:\.git)?\/?$/i)
+  return m ? { owner: m[1], name: m[2], url } : { owner: null, name: null, url }
+}
+
+function readJsonOrNull (file) {
+  try {
+    return jsonfile.readFileSync(file)
+  } catch (e) {
+    return null
+  }
+}
+
+module.exports = { inspectLocalRepos, inspectRepo }