@netlify/cache-utils 4.1.5-rc → 4.1.6-rc

package/lib/dir.js

@@ -0,0 +1,8 @@
+import { resolve } from 'path'
+
+// Retrieve the cache directory location
+export const getCacheDir = function ({ cacheDir = DEFAULT_CACHE_DIR, cwd = '.' } = {}) {
+  return resolve(cwd, cacheDir)
+}
+
+const DEFAULT_CACHE_DIR = '.netlify/cache/'

package/lib/expire.js

@@ -0,0 +1,15 @@
+// Retrieve the expiration date when caching a file
+export const getExpires = function (ttl) {
+  if (!Number.isInteger(ttl) || ttl < 1) {
+    return
+  }
+
+  return Date.now() + ttl * SECS_TO_MSECS
+}
+
+const SECS_TO_MSECS = 1e3
+
+// Check if a file about to be restored is expired
+export const checkExpires = function (expires) {
+  return expires !== undefined && Date.now() > expires
+}

package/lib/fs.js

@@ -0,0 +1,60 @@
+import { promises as fs } from 'fs'
+import { basename, dirname } from 'path'
+
+import cpy from 'cpy'
+import { globby } from 'globby'
+import { isNotJunk } from 'junk'
+import { moveFile } from 'move-file'
+
+// Move or copy a cached file/directory from/to a local one
+export const moveCacheFile = async function (src, dest, move) {
+  // Moving is faster but removes the source files locally
+  if (move) {
+    return moveFile(src, dest, { overwrite: false })
+  }
+
+  const { srcGlob, cwd } = await getSrcGlob(src)
+  return cpy(srcGlob, dirname(dest), { cwd, parents: true, overwrite: false })
+}
+
+// Non-existing files and empty directories are always skipped
+export const hasFiles = async function (src) {
+  const { srcGlob, cwd, isDir } = await getSrcGlob(src)
+  return srcGlob !== undefined && !(await isEmptyDir({ srcGlob, cwd, isDir }))
+}
+
+// Replicates what `cpy` is doing under the hood.
+const isEmptyDir = async function ({ srcGlob, cwd, isDir }) {
+  if (!isDir) {
+    return false
+  }
+
+  const files = await globby(srcGlob, { cwd })
+  const filteredFiles = files.filter((file) => isNotJunk(basename(file)))
+  return filteredFiles.length === 0
+}
+
+// Get globbing pattern with files to move/copy
+const getSrcGlob = async function (src) {
+  const srcStat = await getStat(src)
+
+  if (srcStat === undefined) {
+    return {}
+  }
+
+  const isDir = srcStat.isDirectory()
+  const srcBasename = basename(src)
+  const cwd = dirname(src)
+
+  if (isDir) {
+    return { srcGlob: `${srcBasename}/**`, cwd, isDir }
+  }
+
+  return { srcGlob: srcBasename, cwd, isDir }
+}
+
+const getStat = async function (src) {
+  try {
+    return await fs.stat(src)
+  } catch {}
+}

package/lib/hash.js

@@ -0,0 +1,36 @@
+import { createHash } from 'crypto'
+import { createReadStream } from 'fs'
+
+import getStream from 'get-stream'
+import { locatePath } from 'locate-path'
+
+// Caching a big directory like `node_modules` is slow. However those can
+// sometime be represented by a digest file such as `package-lock.json`. If this
+// has not changed, we don't need to save cache again.
+export const getHash = async function (digests, move) {
+  // Moving files is faster than computing hashes
+  if (move || digests.length === 0) {
+    return
+  }
+
+  const digestPath = await locatePath(digests)
+  if (digestPath === undefined) {
+    return
+  }
+
+  const hash = await hashFile(digestPath)
+  return hash
+}
+
+// Hash a file's contents
+const hashFile = async function (path) {
+  const contentStream = createReadStream(path, 'utf8')
+  const hashStream = createHash(HASH_ALGO, { encoding: 'hex' })
+  contentStream.pipe(hashStream)
+  const hash = await getStream(hashStream)
+  return hash
+}
+
+// We need a hashing algoritm that's as fast as possible.
+// Userland CRC32 implementations are actually slower than Node.js SHA1.
+const HASH_ALGO = 'sha1'

package/lib/list.js

@@ -0,0 +1,29 @@
+import { join } from 'path'
+
+import readdirp from 'readdirp'
+
+import { getCacheDir } from './dir.js'
+import { isManifest } from './manifest.js'
+import { getBases } from './path.js'
+
+// List all cached files/directories, at the top-level
+export const list = async function ({ cacheDir, cwd: cwdOpt, depth = DEFAULT_DEPTH } = {}) {
+  const bases = await getBases(cwdOpt)
+  const cacheDirA = getCacheDir({ cacheDir, cwd: cwdOpt })
+  const files = await Promise.all(bases.map(({ name, base }) => listBase({ name, base, cacheDir: cacheDirA, depth })))
+  const filesA = files.flat()
+  return filesA
+}
+
+const DEFAULT_DEPTH = 1
+
+// TODO: the returned paths are missing the Windows drive
+const listBase = async function ({ name, base, cacheDir, depth }) {
+  const files = await readdirp.promise(`${cacheDir}/${name}`, { fileFilter, depth, type: 'files_directories' })
+  const filesA = files.map(({ path }) => join(base, path))
+  return filesA
+}
+
+const fileFilter = function ({ basename }) {
+  return !isManifest(basename)
+}

package/lib/main.js

@@ -0,0 +1,103 @@
+import del from 'del'
+
+import { getCacheDir } from './dir.js'
+import { moveCacheFile, hasFiles } from './fs.js'
+import { list } from './list.js'
+import { getManifestInfo, writeManifest, removeManifest, isExpired } from './manifest.js'
+import { parsePath } from './path.js'
+
+export { getCacheDir } from './dir.js'
+export { list } from './list.js'
+
+// Cache a file
+const saveOne = async function (
+  path,
+  { move = DEFAULT_MOVE, ttl = DEFAULT_TTL, digests = [], cacheDir, cwd: cwdOpt } = {},
+) {
+  const { srcPath, cachePath } = await parsePath({ path, cacheDir, cwdOpt })
+
+  if (!(await hasFiles(srcPath))) {
+    return false
+  }
+
+  const { manifestInfo, identical } = await getManifestInfo({ cachePath, move, ttl, digests })
+  if (identical) {
+    return true
+  }
+
+  await del(cachePath, { force: true })
+  await moveCacheFile(srcPath, cachePath, move)
+  await writeManifest(manifestInfo)
+
+  return true
+}
+
+// Restore a cached file
+const restoreOne = async function (path, { move = DEFAULT_MOVE, cacheDir, cwd: cwdOpt } = {}) {
+  const { srcPath, cachePath } = await parsePath({ path, cacheDir, cwdOpt })
+
+  if (!(await hasFiles(cachePath))) {
+    return false
+  }
+
+  if (await isExpired(cachePath)) {
+    return false
+  }
+
+  await del(srcPath, { force: true })
+  await moveCacheFile(cachePath, srcPath, move)
+
+  return true
+}
+
+// Remove the cache of a file
+const removeOne = async function (path, { cacheDir, cwd: cwdOpt } = {}) {
+  const { cachePath } = await parsePath({ path, cacheDir, cwdOpt })
+
+  if (!(await hasFiles(cachePath))) {
+    return false
+  }
+
+  await del(cachePath, { force: true })
+  await removeManifest(cachePath)
+
+  return true
+}
+
+// Check if a file is cached
+const hasOne = async function (path, { cacheDir, cwd: cwdOpt } = {}) {
+  const { cachePath } = await parsePath({ path, cacheDir, cwdOpt })
+
+  return (await hasFiles(cachePath)) && !(await isExpired(cachePath))
+}
+
+const DEFAULT_MOVE = false
+const DEFAULT_TTL = undefined
+
+// Allow each of the main functions to take either a single path or an array of
+// paths as arguments
+const allowMany = async function (func, paths, ...args) {
+  if (!Array.isArray(paths)) {
+    return func(paths, ...args)
+  }
+
+  const results = await Promise.all(paths.map((path) => func(path, ...args)))
+  return results.some(Boolean)
+}
+
+export const save = allowMany.bind(null, saveOne)
+export const restore = allowMany.bind(null, restoreOne)
+export const remove = allowMany.bind(null, removeOne)
+export const has = allowMany.bind(null, hasOne)
+
+// Change `opts` default values
+export const bindOpts = function (opts) {
+  return {
+    save: (paths, optsA) => save(paths, { ...opts, ...optsA }),
+    restore: (paths, optsA) => restore(paths, { ...opts, ...optsA }),
+    remove: (paths, optsA) => remove(paths, { ...opts, ...optsA }),
+    has: (paths, optsA) => has(paths, { ...opts, ...optsA }),
+    list: (optsA) => list({ ...opts, ...optsA }),
+    getCacheDir: (optsA) => getCacheDir({ ...opts, ...optsA }),
+  }
+}

package/lib/manifest.js

@@ -0,0 +1,71 @@
+import { promises as fs } from 'fs'
+import { dirname } from 'path'
+
+import del from 'del'
+import { pathExists } from 'path-exists'
+
+import { getExpires, checkExpires } from './expire.js'
+import { getHash } from './hash.js'
+
+// Retrieve cache manifest of a file to cache, which contains the file/directory
+// contents hash and the `expires` date.
+export const getManifestInfo = async function ({ cachePath, move, ttl, digests }) {
+  const manifestPath = getManifestPath(cachePath)
+  const expires = getExpires(ttl)
+  const hash = await getHash(digests, move)
+  const manifest = { expires, hash }
+  const manifestString = `${JSON.stringify(manifest, null, 2)}\n`
+  const identical = await isIdentical({ hash, manifestPath, manifestString })
+  return { manifestInfo: { manifestPath, manifestString }, identical }
+}
+
+// Whether the cache manifest has changed
+const isIdentical = async function ({ hash, manifestPath, manifestString }) {
+  if (hash === undefined || !(await pathExists(manifestPath))) {
+    return false
+  }
+
+  const oldManifestString = await fs.readFile(manifestPath, 'utf8')
+  return oldManifestString === manifestString
+}
+
+// Persist the cache manifest to filesystem
+export const writeManifest = async function ({ manifestPath, manifestString }) {
+  await fs.mkdir(dirname(manifestPath), { recursive: true })
+  await fs.writeFile(manifestPath, manifestString)
+}
+
+// Remove the cache manifest from filesystem
+export const removeManifest = async function (cachePath) {
+  const manifestPath = getManifestPath(cachePath)
+  await del(manifestPath, { force: true })
+}
+
+// Retrieve the cache manifest filepath
+const getManifestPath = function (cachePath) {
+  return `${cachePath}${CACHE_EXTENSION}`
+}
+
+export const isManifest = function (filePath) {
+  return filePath.endsWith(CACHE_EXTENSION)
+}
+
+const CACHE_EXTENSION = '.netlify.cache.json'
+
+// Check whether a file/directory is expired by checking its cache manifest
+export const isExpired = async function (cachePath) {
+  const manifestPath = getManifestPath(cachePath)
+  if (!(await pathExists(manifestPath))) {
+    return false
+  }
+
+  const { expires } = await readManifest(cachePath)
+  return checkExpires(expires)
+}
+
+const readManifest = async function (cachePath) {
+  const manifestPath = getManifestPath(cachePath)
+  const manifestString = await fs.readFile(manifestPath)
+  const manifest = JSON.parse(manifestString)
+  return manifest
+}

package/lib/path.js

@@ -0,0 +1,107 @@
+import { homedir } from 'os'
+import { resolve, isAbsolute, join, sep } from 'path'
+
+import { getCacheDir } from './dir.js'
+import { safeGetCwd } from './utils/cwd.js'
+
+// Find the paths of the file before/after caching
+export const parsePath = async function ({ path, cacheDir, cwdOpt }) {
+  const srcPath = await getSrcPath(path, cwdOpt)
+  const cachePath = await getCachePath({ srcPath, cacheDir, cwdOpt })
+  return { srcPath, cachePath }
+}
+
+// Retrieve absolute path to the local file to cache/restore
+const getSrcPath = async function (path, cwdOpt) {
+  const cwd = await safeGetCwd(cwdOpt)
+  const srcPath = resolvePath(path, cwd)
+  checkSrcPath(srcPath, cwd)
+  return srcPath
+}
+
+const resolvePath = function (path, cwd) {
+  if (isAbsolute(path)) {
+    return resolve(path)
+  }
+
+  if (cwd !== '') {
+    return resolve(cwd, path)
+  }
+
+  throw new Error(`Current directory does not exist: ${cwd}`)
+}
+
+// Caching the whole repository creates many issues:
+//  - It caches many directories that are not related to Gatsby but take lots of
+//    space, such as node_modules
+//  - It caches directories that are not meant to restored across builds. For
+//    example .git (beside being big).
+//  - It undoes any build operations inside the repository that might have
+//    happened before this plugin starts restoring the cache, leading to
+//    conflicts with other plugins, Netlify Build or the buildbot.
+const checkSrcPath = function (srcPath, cwd) {
+  if (cwd !== '' && isParentPath(srcPath, cwd)) {
+    throw new Error(`Cannot cache ${srcPath} because it is the current directory (${cwd}) or a parent directory`)
+  }
+}
+
+// Note: srcPath and cwd are already normalized and absolute
+const isParentPath = function (srcPath, cwd) {
+  return `${cwd}${sep}`.startsWith(`${srcPath}${sep}`)
+}
+
+const getCachePath = async function ({ srcPath, cacheDir, cwdOpt }) {
+  const cacheDirA = getCacheDir({ cacheDir, cwd: cwdOpt })
+  const { name, relPath } = await findBase(srcPath, cwdOpt)
+  const cachePath = join(cacheDirA, name, relPath)
+  return cachePath
+}
+
+// The cached path is the path relative to the base which can be either the
+// current directory, the home directory or the root directory. Each is tried
+// in order.
+const findBase = async function (srcPath, cwdOpt) {
+  const bases = await getBases(cwdOpt)
+  const srcPathA = normalizeWindows(srcPath)
+  return bases.map(({ name, base }) => parseBase(name, base, srcPathA)).find(Boolean)
+}
+
+// Windows drives are problematic:
+//  - they cannot be used in `relPath` since directories cannot be called `C:`
+//    for example.
+//  - they make comparing between drives harder
+// For the moment, we simply strip them. This does mean two files with the same
+// paths but inside different drives would be cached together, which is not
+// correct.
+// This also means `cache.list()` always assumes the source files are on the
+// current drive.
+// TODO: fix it.
+const normalizeWindows = function (srcPath) {
+  return srcPath.replace(WINDOWS_DRIVE_REGEX, '\\')
+}
+
+const WINDOWS_DRIVE_REGEX = /^[a-zA-Z]:\\/
+
+// This logic works when `base` and `path` are on different Windows drives
+const parseBase = function (name, base, srcPath) {
+  if (srcPath === base || !srcPath.startsWith(base)) {
+    return
+  }
+
+  const relPath = srcPath.replace(base, '')
+  return { name, relPath }
+}
+
+export const getBases = async function (cwdOpt) {
+  const cwdBase = await getCwdBase(cwdOpt)
+  return [...cwdBase, { name: 'home', base: homedir() }, { name: 'root', base: sep }]
+}
+
+const getCwdBase = async function (cwdOpt) {
+  const cwd = await safeGetCwd(cwdOpt)
+  if (cwd === '') {
+    return []
+  }
+
+  return [{ name: 'cwd', base: cwd }]
+}

package/lib/utils/cwd.js

@@ -0,0 +1,23 @@
+import { normalize } from 'path'
+import process from 'process'
+
+import { pathExists } from 'path-exists'
+
+// Like `process.cwd()` but safer when current directory is wrong
+export const safeGetCwd = async function (cwdOpt) {
+  try {
+    const cwd = getCwdValue(cwdOpt)
+
+    if (!(await pathExists(cwd))) {
+      return ''
+    }
+
+    return cwd
+  } catch {
+    return ''
+  }
+}
+
+const getCwdValue = function (cwdOpt = process.cwd()) {
+  return normalize(cwdOpt)
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@netlify/cache-utils",
-  "version": "4.1.5-rc",
+  "version": "4.1.6-rc",
   "description": "Utility for caching files in Netlify Build",
   "type": "module",
   "exports": "./lib/main.js",