@jsenv/filesystem 3.1.0 → 3.2.2

package/README.md

@@ -64,7 +64,7 @@ import { registerDirectoryLifecycle } from "@jsenv/filesystem"
 
 const directoryContentDescription = {}
 const unregister = registerDirectoryLifecycle("file:///directory/", {
-  watchDescription: {
+  watchPatterns: {
     "./**/*": true,
     "./node_modules/": false,
   },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jsenv/filesystem",
-  "version": "3.1.0",
+  "version": "3.2.2",
   "description": "Collection of functions to interact with filesystem in Node.js",
   "license": "MIT",
   "author": {

package/src/fileSystemRootUrl.js

@@ -1,2 +1,2 @@
 export const fileSystemRootUrl =
-  process.platform === "win32" ? `file///${process.cwd()[0]}:/` : "file:///"
+  process.platform === "win32" ? `file:///${process.cwd()[0]}:/` : "file:///"

package/src/internal/createWatcher.js

@@ -1,4 +1,4 @@
-import { watch, openSync, closeSync } from "fs"
+import { watch, openSync, closeSync } from "node:fs"
 
 const isWindows = process.platform === "win32"
 

package/src/internal/guard_second_call.js

@@ -0,0 +1,20 @@
+export const guardTooFastSecondCall = (
+  callback,
+  cooldownBetweenFileEvents = 40,
+) => {
+  const previousCallMsMap = new Map()
+  return (fileEvent) => {
+    const { relativeUrl } = fileEvent
+    const previousCallMs = previousCallMsMap.get(relativeUrl)
+    const nowMs = Date.now()
+    if (previousCallMs) {
+      const msEllapsed = nowMs - previousCallMs
+      if (msEllapsed < cooldownBetweenFileEvents) {
+        previousCallMsMap.delete(relativeUrl)
+        return
+      }
+    }
+    previousCallMsMap.set(relativeUrl, nowMs)
+    callback(fileEvent)
+  }
+}

package/src/internal/read_entry_info.js

@@ -0,0 +1,20 @@
+import { statSync } from "node:fs"
+
+import { statsToType } from "./statsToType.js"
+
+export const readEntryInfo = (url) => {
+  try {
+    const stats = statSync(new URL(url))
+    const type = statsToType(stats)
+    return {
+      type,
+      atimeMs: stats.atimeMs,
+      mtimeMs: stats.mtimeMs,
+    }
+  } catch (e) {
+    if (e.code === "ENOENT") {
+      return null
+    }
+    throw e
+  }
+}

package/src/registerDirectoryLifecycle.js

@@ -1,17 +1,16 @@
-/* eslint-disable import/max-dependencies */
-import { readdirSync } from "fs"
+import { readdirSync, statSync } from "node:fs"
 import {
   normalizeStructuredMetaMap,
   urlCanContainsMetaMatching,
   urlToMeta,
 } from "@jsenv/url-meta"
+
+import { assertAndNormalizeDirectoryUrl } from "./assertAndNormalizeDirectoryUrl.js"
+import { statsToType } from "./internal/statsToType.js"
+import { guardTooFastSecondCall } from "./internal/guard_second_call.js"
 import { replaceBackSlashesWithSlashes } from "./internal/replaceBackSlashesWithSlashes.js"
-import { entryToTypeOrNull } from "./internal/entryToTypeOrNull.js"
 import { createWatcher } from "./internal/createWatcher.js"
 import { trackRessources } from "./internal/trackRessources.js"
-import { ensureUrlTrailingSlash } from "./internal/ensureUrlTrailingSlash.js"
-import { resolveUrl } from "./resolveUrl.js"
-import { assertAndNormalizeFileUrl } from "./assertAndNormalizeFileUrl.js"
 import { urlToFileSystemPath } from "./urlToFileSystemPath.js"
 import { urlToRelativeUrl } from "./urlToRelativeUrl.js"
 
@@ -22,18 +21,25 @@ const fsWatchSupportsRecursive = !isLinux
 export const registerDirectoryLifecycle = (
   source,
   {
+    debug = false,
     added,
     updated,
     removed,
-    watchDescription = {
+    watchPatterns = {
       "./**/*": true,
     },
     notifyExistent = false,
     keepProcessAlive = true,
     recursive = false,
+    // filesystem might dispatch more events than expected
+    // Code can use "cooldownBetweenFileEvents" to prevent that
+    // BUT it is UNADVISED to rely on this as explained later (search for "is lying" in this file)
+    // For this reason"cooldownBetweenFileEvents" should be reserved to scenarios
+    // like unit tests
+    cooldownBetweenFileEvents = 0,
   },
 ) => {
-  const sourceUrl = ensureUrlTrailingSlash(assertAndNormalizeFileUrl(source))
+  const sourceUrl = assertAndNormalizeDirectoryUrl(source)
   if (!undefinedOrFunction(added)) {
     throw new TypeError(`added must be a function or undefined, got ${added}`)
   }
@@ -47,34 +53,70 @@ export const registerDirectoryLifecycle = (
       `removed must be a function or undefined, got ${removed}`,
     )
   }
+  if (cooldownBetweenFileEvents) {
+    if (added) {
+      added = guardTooFastSecondCall(added, cooldownBetweenFileEvents)
+    }
+    if (updated) {
+      updated = guardTooFastSecondCall(updated, cooldownBetweenFileEvents)
+    }
+    if (removed) {
+      removed = guardTooFastSecondCall(removed, cooldownBetweenFileEvents)
+    }
+  }
 
   const structuredMetaMap = normalizeStructuredMetaMap(
-    { watch: watchDescription },
+    { watch: watchPatterns },
     sourceUrl,
   )
-  const entryShouldBeWatched = ({ relativeUrl, type }) => {
-    const entryUrl = resolveUrl(relativeUrl, sourceUrl)
-
+  const getWatchPatternValue = ({ url, type }) => {
     if (type === "directory") {
-      const canContainEntryToWatch = urlCanContainsMetaMatching({
-        url: `${entryUrl}/`,
+      let firstMeta = false
+      urlCanContainsMetaMatching({
+        url: `${url}/`,
         structuredMetaMap,
-        predicate: ({ watch }) => watch,
+        predicate: ({ watch }) => {
+          if (watch) {
+            firstMeta = watch
+          }
+          return watch
+        },
       })
-      return canContainEntryToWatch
+      return firstMeta
     }
-
-    const entryMeta = urlToMeta({
-      url: entryUrl,
+    const filesystemEntryMeta = urlToMeta({
+      url,
       structuredMetaMap,
     })
-
-    return entryMeta.watch
+    return filesystemEntryMeta.watch
   }
-
   const tracker = trackRessources()
-
-  const contentMap = new Map()
+  const infoMap = new Map()
+  const readEntryInfo = (url) => {
+    try {
+      const relativeUrl = urlToRelativeUrl(url, source)
+      const previousInfo = infoMap.get(relativeUrl)
+      const stats = statSync(new URL(url))
+      const type = statsToType(stats)
+      const patternValue = previousInfo
+        ? previousInfo.patternValue
+        : getWatchPatternValue({ url, type })
+      return {
+        previousInfo,
+        url,
+        relativeUrl,
+        type,
+        atimeMs: stats.atimeMs,
+        mtimeMs: stats.mtimeMs,
+        patternValue,
+      }
+    } catch (e) {
+      if (e.code === "ENOENT") {
+        return null
+      }
+      throw e
+    }
+  }
 
   const handleDirectoryEvent = ({
     directoryRelativeUrl,
@@ -84,98 +126,103 @@ export const registerDirectoryLifecycle = (
     if (filename) {
       if (directoryRelativeUrl) {
         handleChange(`${directoryRelativeUrl}/${filename}`)
-      } else {
-        handleChange(`${filename}`)
+        return
+      }
+      handleChange(`${filename}`)
+      return
+    }
+    if (eventType === "rename") {
+      if (!removed && !added) {
+        return
       }
-    } else if ((removed || added) && eventType === "rename") {
       // we might receive `rename` without filename
       // in that case we try to find ourselves which file was removed.
-
-      let relativeUrlCandidateArray = Array.from(contentMap.keys())
-
+      let relativeUrlCandidateArray = Array.from(infoMap.keys())
       if (recursive && !fsWatchSupportsRecursive) {
         relativeUrlCandidateArray = relativeUrlCandidateArray.filter(
           (relativeUrlCandidate) => {
             if (!directoryRelativeUrl) {
               // ensure entry is top level
-              if (relativeUrlCandidate.includes("/")) return false
+              if (relativeUrlCandidate.includes("/")) {
+                return false
+              }
               return true
             }
-
             // entry not inside this directory
-            if (!relativeUrlCandidate.startsWith(directoryRelativeUrl))
+            if (!relativeUrlCandidate.startsWith(directoryRelativeUrl)) {
               return false
-
+            }
             const afterDirectory = relativeUrlCandidate.slice(
               directoryRelativeUrl.length + 1,
             )
             // deep inside this directory
-            if (afterDirectory.includes("/")) return false
-
+            if (afterDirectory.includes("/")) {
+              return false
+            }
             return true
           },
         )
       }
-
       const removedEntryRelativeUrl = relativeUrlCandidateArray.find(
         (relativeUrlCandidate) => {
-          const entryUrl = resolveUrl(relativeUrlCandidate, sourceUrl)
-          const type = entryToTypeOrNull(entryUrl)
-          return type === null
+          try {
+            statSync(new URL(relativeUrlCandidate, sourceUrl))
+            return false
+          } catch (e) {
+            if (e.code === "ENOENT") {
+              return true
+            }
+            throw e
+          }
         },
       )
-
       if (removedEntryRelativeUrl) {
-        handleEntryLost({
-          relativeUrl: removedEntryRelativeUrl,
-          type: contentMap.get(removedEntryRelativeUrl),
-        })
+        handleEntryLost(infoMap.get(removedEntryRelativeUrl))
       }
     }
   }
 
   const handleChange = (relativeUrl) => {
-    const entryUrl = resolveUrl(relativeUrl, sourceUrl)
-    const previousType = contentMap.get(relativeUrl)
-    const type = entryToTypeOrNull(entryUrl)
-
-    if (!entryShouldBeWatched({ relativeUrl, type })) {
+    const entryUrl = new URL(relativeUrl, sourceUrl).href
+    const entryInfo = readEntryInfo(entryUrl)
+    if (!entryInfo) {
+      const previousEntryInfo = infoMap.get(relativeUrl)
+      if (!previousEntryInfo) {
+        // on MacOS it's possible to receive a "rename" event for
+        // a file that does not exists...
+        return
+      }
+      if (debug) {
+        console.debug(`"${relativeUrl}" removed`)
+      }
+      handleEntryLost(previousEntryInfo)
       return
     }
-
-    // it's something new
-    if (!previousType) {
-      if (type !== null) {
-        handleEntryFound({ relativeUrl, type, existent: false })
+    const { previousInfo } = entryInfo
+    if (!previousInfo) {
+      if (debug) {
+        console.debug(`"${relativeUrl}" added`)
       }
+      handleEntryFound(entryInfo)
       return
     }
-
-    // it existed but now it's not here anymore
-    if (type === null) {
-      handleEntryLost({ relativeUrl, type: previousType })
+    if (entryInfo.type !== previousInfo.type) {
+      // it existed and was replaced by something else
+      // we don't handle this as an update. We rather say the ressource
+      // is lost and something else is found (call removed() then added())
+      handleEntryLost(previousInfo)
+      handleEntryFound(entryInfo)
       return
     }
-
-    // it existed and was replaced by something else
-    // we don't handle this as an update. We rather say the ressource
-    // is lost and something else is found (call removed() then added())
-    if (previousType !== type) {
-      handleEntryLost({ relativeUrl, type: previousType })
-      handleEntryFound({ relativeUrl, type })
+    if (entryInfo.type === "directory") {
+      // a directory cannot really be updated in way that matters for us
+      // filesystem is trying to tell us the directory content have changed
+      // but we don't care about that
+      // we'll already be notified about what has changed
       return
     }
-
-    // a directory cannot really be updated in way that matters for us
-    // filesystem is trying to tell us the directory content have changed
-    // but we don't care about that
-    // we'll already be notified about what has changed
-    if (type === "directory") {
-      return
-    }
-
     // something has changed at this relativeUrl (the file existed and was not deleted)
-    // it's possible to get there and there is no real update
+    // it's possible to get there without a real update
     // (file content is the same and file mtime is the same).
     // In short filesystem is sometimes "lying"
     // Not trying to guard against that because:
@@ -183,76 +230,95 @@ export const registerDirectoryLifecycle = (
     // - it happens very rarely
     // - it's not really a concern in practice
     // - filesystem did not send an event out of nowhere:
-    // something occured but we don't know what with the information we have.
-    if (updated) {
-      updated({ relativeUrl, type })
+    //   something occured but we don't know exactly what
+    // maybe we should exclude some stuff as done in
+    // https://github.com/paulmillr/chokidar/blob/b2c4f249b6cfa98c703f0066fb4a56ccd83128b5/lib/nodefs-handler.js#L366
+    if (debug) {
+      console.debug(`"${relativeUrl}" modified`)
     }
+    handleEntryUpdated(entryInfo)
   }
-
-  const handleEntryFound = ({ relativeUrl, type, existent }) => {
-    if (!entryShouldBeWatched({ relativeUrl, type })) {
-      return
-    }
-
-    contentMap.set(relativeUrl, type)
-
-    const entryUrl = resolveUrl(relativeUrl, sourceUrl)
-
-    if (type === "directory") {
-      visitDirectory({
-        directoryUrl: `${entryUrl}/`,
-        entryFound: (entry) => {
-          handleEntryFound({
-            relativeUrl: `${relativeUrl}/${entry.relativeUrl}`,
-            type: entry.type,
-            existent,
-          })
-        },
-      })
-    }
-
-    if (added) {
-      if (existent) {
-        if (notifyExistent) {
-          added({ relativeUrl, type, existent: true })
+  const handleEntryFound = (entryInfo, { notify = true } = {}) => {
+    infoMap.set(entryInfo.relativeUrl, entryInfo)
+    if (entryInfo.type === "directory") {
+      const directoryUrl = `${entryInfo.url}/`
+      readdirSync(new URL(directoryUrl)).forEach((entryName) => {
+        const childEntryUrl = new URL(entryName, directoryUrl).href
+        const childEntryInfo = readEntryInfo(childEntryUrl)
+        if (childEntryInfo && childEntryInfo.patternValue) {
+          handleEntryFound(childEntryInfo, { notify })
         }
-      } else {
-        added({ relativeUrl, type })
+      })
+      // we must watch manually every directory we find
+      if (!fsWatchSupportsRecursive) {
+        const watcher = createWatcher(urlToFileSystemPath(entryInfo.url), {
+          persistent: keepProcessAlive,
+        })
+        tracker.registerCleanupCallback(() => {
+          watcher.close()
+        })
+        watcher.on("change", (eventType, filename) => {
+          handleDirectoryEvent({
+            directoryRelativeUrl: entryInfo.relativeUrl,
+            filename: filename ? replaceBackSlashesWithSlashes(filename) : "",
+            eventType,
+          })
+        })
       }
     }
-
-    // we must watch manually every directory we find
-    if (!fsWatchSupportsRecursive && type === "directory") {
-      const watcher = createWatcher(urlToFileSystemPath(entryUrl), {
-        persistent: keepProcessAlive,
-      })
-      tracker.registerCleanupCallback(() => {
-        watcher.close()
+    if (added && entryInfo.patternValue && notify) {
+      added({
+        relativeUrl: entryInfo.relativeUrl,
+        type: entryInfo.type,
+        patternValue: entryInfo.patternValue,
+        mtime: entryInfo.mtimeMs,
       })
-      watcher.on("change", (eventType, filename) => {
-        handleDirectoryEvent({
-          directoryRelativeUrl: relativeUrl,
-          filename: filename ? replaceBackSlashesWithSlashes(filename) : "",
-          eventType,
-        })
+    }
+  }
+  const handleEntryLost = (entryInfo) => {
+    infoMap.delete(entryInfo.relativeUrl)
+    if (removed && entryInfo.patternValue) {
+      removed({
+        relativeUrl: entryInfo.relativeUrl,
+        type: entryInfo.type,
+        patternValue: entryInfo.patternValue,
+        mtime: entryInfo.mtimeMs,
       })
     }
   }
-
-  const handleEntryLost = ({ relativeUrl, type }) => {
-    contentMap.delete(relativeUrl)
-    if (removed) {
-      removed({ relativeUrl, type })
+  const handleEntryUpdated = (entryInfo) => {
+    infoMap.set(entryInfo.relativeUrl, entryInfo)
+    if (updated && entryInfo.patternValue) {
+      updated({
+        relativeUrl: entryInfo.relativeUrl,
+        type: entryInfo.type,
+        patternValue: entryInfo.patternValue,
+        mtime: entryInfo.mtimeMs,
+        previousMtime: entryInfo.previousInfo.mtimeMs,
+      })
     }
   }
 
-  visitDirectory({
-    directoryUrl: sourceUrl,
-    entryFound: ({ relativeUrl, type }) => {
-      handleEntryFound({ relativeUrl, type, existent: true })
-    },
+  readdirSync(new URL(sourceUrl)).forEach((entry) => {
+    const entryUrl = new URL(entry, sourceUrl).href
+    const entryInfo = readEntryInfo(entryUrl)
+    if (entryInfo && entryInfo.patternValue) {
+      handleEntryFound(entryInfo, {
+        notify: notifyExistent,
+      })
+    }
   })
-
+  if (debug) {
+    const relativeUrls = Array.from(infoMap.keys())
+    if (relativeUrls.length === 0) {
+      console.debug(`No file found`)
+    } else {
+      console.debug(
+        `${relativeUrls.length} file found: 
+${relativeUrls.join("\n")}`,
+      )
+    }
+  }
   const watcher = createWatcher(urlToFileSystemPath(sourceUrl), {
     recursive: recursive && fsWatchSupportsRecursive,
     persistent: keepProcessAlive,
@@ -270,24 +336,8 @@ export const registerDirectoryLifecycle = (
   return tracker.cleanup
 }
 
-const undefinedOrFunction = (value) =>
-  typeof value === "undefined" || typeof value === "function"
-
-const visitDirectory = ({ directoryUrl, entryFound }) => {
-  const directoryPath = urlToFileSystemPath(directoryUrl)
-  readdirSync(directoryPath).forEach((entry) => {
-    const entryUrl = resolveUrl(entry, directoryUrl)
-    const type = entryToTypeOrNull(entryUrl)
-    if (type === null) {
-      return
-    }
-
-    const relativeUrl = urlToRelativeUrl(entryUrl, directoryUrl)
-    entryFound({
-      relativeUrl,
-      type,
-    })
-  })
+const undefinedOrFunction = (value) => {
+  return typeof value === "undefined" || typeof value === "function"
 }
 
 const fileSystemPathToDirectoryRelativeUrlAndFilename = (path) => {

package/src/registerFileLifecycle.js

@@ -1,5 +1,8 @@
-import { dirname, basename } from "path"
-import { entryToTypeOrNull } from "./internal/entryToTypeOrNull.js"
+import { statSync } from "node:fs"
+import { dirname, basename } from "node:path"
+
+import { guardTooFastSecondCall } from "./internal/guard_second_call.js"
+import { statsToType } from "./internal/statsToType.js"
 import { createWatcher } from "./internal/createWatcher.js"
 import { trackRessources } from "./internal/trackRessources.js"
 import { assertAndNormalizeFileUrl } from "./assertAndNormalizeFileUrl.js"
@@ -7,7 +10,14 @@ import { urlToFileSystemPath } from "./urlToFileSystemPath.js"
 
 export const registerFileLifecycle = (
   source,
-  { added, updated, removed, notifyExistent = false, keepProcessAlive = true },
+  {
+    added,
+    updated,
+    removed,
+    notifyExistent = false,
+    keepProcessAlive = true,
+    cooldownBetweenFileEvents = 0,
+  },
 ) => {
   const sourceUrl = assertAndNormalizeFileUrl(source)
   if (!undefinedOrFunction(added)) {
@@ -23,6 +33,17 @@ export const registerFileLifecycle = (
       `removed must be a function or undefined, got ${removed}`,
     )
   }
+  if (cooldownBetweenFileEvents) {
+    if (added) {
+      added = guardTooFastSecondCall(added, cooldownBetweenFileEvents)
+    }
+    if (updated) {
+      updated = guardTooFastSecondCall(updated, cooldownBetweenFileEvents)
+    }
+    if (removed) {
+      removed = guardTooFastSecondCall(removed, cooldownBetweenFileEvents)
+    }
+  }
 
   const tracker = trackRessources()
 
@@ -89,6 +110,18 @@ export const registerFileLifecycle = (
   return tracker.cleanup
 }
 
+const entryToTypeOrNull = (url) => {
+  try {
+    const stats = statSync(new URL(url))
+    return statsToType(stats)
+  } catch (e) {
+    if (e.code === "ENOENT") {
+      return null
+    }
+    throw e
+  }
+}
+
 const undefinedOrFunction = (value) =>
   typeof value === "undefined" || typeof value === "function"
 

package/src/internal/entryToTypeOrNull.js

@@ -1,18 +0,0 @@
-import { statSync } from "node:fs"
-
-import { urlToFileSystemPath } from "../urlToFileSystemPath.js"
-
-import { statsToType } from "./statsToType.js"
-
-export const entryToTypeOrNull = (url) => {
-  const path = urlToFileSystemPath(url)
-  try {
-    const stats = statSync(path)
-    return statsToType(stats)
-  } catch (e) {
-    if (e.code === "ENOENT") {
-      return null
-    }
-    throw e
-  }
-}