imdone-core 1.54.10 → 2.0.1

package/lib/mixins/repo-fs-store.js

@@ -1,53 +1,48 @@
-'use strict'
-
-var _path = require('path'),
-  _cloneDeep = require('lodash.clonedeep'),
-  _isFunction = require('lodash.isfunction'),
-  _isUndefined = require('lodash.isundefined'),
-  _assign = require('lodash.assign'),
-  _remove = require('lodash.remove'),
-  _union = require('lodash.union'),
-  async = require('async'),
-  checksum = require('checksum'),
-  tools = require('../tools'),
-  constants = require('../constants'),
-  log = require('debug')('imdone-mixins:repo-fs-store'),
-  File = require('../file'),
-  Config = require('../config'),
-  languages = require('../languages'),
-  isBinaryFile = require('isbinaryfile'),
-  eol = require('eol'),
-  _eol = String(eol.auto),
-  readdirp = require('readdirp'),
-  _ignore = require('ignore'),
-  fastSort = require('fast-sort/dist/sort.js'),
-  Repository = require('../repository'),
-  buildMigrateConfig = require('../migrate-config'),
-  { loadYAML, dumpYAML } = require('../adapters/yaml')
-
-
-module.exports = mixin
+import _path from 'path'
+import _cloneDeep from 'lodash.clonedeep'
+import _isFunction from 'lodash.isfunction'
+import _isUndefined from 'lodash.isundefined'
+import _assign from 'lodash.assign'
+import _remove from 'lodash.remove'
+import _union from 'lodash.union'
+import { eachLimit } from 'async'
+import { computeChecksum } from '../checksum.js'
+import { constants } from '../constants.js'
+import debug from 'debug'
+const log = debug('imdone-mixins:repo-fs-store.js')
+import { File } from '../file.js'
+import { Config } from '../config.js'
+import { languages } from '../languages.js'
+import { isBinaryFile } from 'isbinaryfile'
+import eol from 'eol'
+const _eol = String(eol.auto)
+import readdirp from 'readdirp'
+import _ignore from 'ignore'
+import { sort } from 'fast-sort'
+import { buildMigrateConfig } from '../migrate-config.js'
+import { loadYAML, dumpYAML } from '../adapters/yaml.js';
+import realFs, { readFileSync } from 'fs'
+import { preparePathForWriting, readFile, writeFile, stat, lstat, unlink } from '../adapters/file-gateway.js'
 
 const {
-  CONFIG_DIR,
   CONFIG_FILE,
   CONFIG_FILE_YML,
-  DEFAULT_CONFIG,
   ASYNC_LIMIT,
-  DEFAULT_IGNORE_DIRS,
+  DEFAULT_IGNORE_PATHS,
   DEFAULT_IGNORE_EXTS,
   IGNORE_FILE,
   SORT_FILE,
   ERRORS,
 } = constants
 
-function mixin(repo, fs = require('fs')) {
-  repo.checksum = checksum
+const { DEFAULT_CONFIG } = Config
+
+export function fileSystemStoreMixin(repo, fs = realFs) {
   repo.allFiles = []
 
   function getIgnorePatterns() {
     var ignoreFile = _path.join(repo.path, IGNORE_FILE)
-    let patterns = DEFAULT_IGNORE_DIRS
+    let patterns = DEFAULT_IGNORE_PATHS
     if (fs.existsSync(ignoreFile))
       patterns = patterns + _eol + fs.readFileSync(ignoreFile).toString()
 
@@ -61,28 +56,13 @@ function mixin(repo, fs = require('fs')) {
     return !dirent.isSymbolicLink() && !IG.ignores(path)
   }
 
-  function getAllFilePaths(root, cb) {
-    readdirp
-      .promise(root, {
-        lstat: true,
-        fileFilter: shouldIncludeFile,
-      })
-      .then((entries) => cb(null, entries))
-      .catch((error) => cb(error, []))
-  }
-
-  function ls(root, cb) {
-    getAllFilePaths(root, function (error, files) {
-      if (error) {
-        console.error(`Error reading dir: ${root} `, error)
-      }
-      cb(
-        null,
-        files
-          .filter((res) => repo.shouldInclude(res.path))
-          .map((res) => res.fullPath)
-      )
-    })
+  repo.getAllFilePaths = async function (root) {
+    const entries = []
+    for await (const entry of readdirp(root, { lstat: true, fileFilter: shouldIncludeFile })) {
+      entries.push(entry)
+    }
+    return entries.filter((res) => repo.shouldInclude(res.path))
+      .map((res) => res.fullPath)
   }
 
   repo.addFilePath = async function (path) {
@@ -104,38 +84,32 @@ function mixin(repo, fs = require('fs')) {
     return repo.allFiles
   }
 
-  const _init = repo.init
   repo.migrateConfig = buildMigrateConfig(repo)
-  repo.init = function (cb) {
-    _init.call(repo, (err) => {
-      if (err && repo.destroyed) return cb(err)
-      cb = tools.cb(cb)
-      fs.stat(repo.path, function (err, stat) {
-        if (err || !stat.isDirectory()) {
-          var error = new Error(
-            `Path must be an existing directory on the file system: ${repo.path}`
-          )
-          repo.emit('error', error)
-          return cb(error)
-        }
+  // READY Refactor init to use async/await
+  // #esm-migration #important #urgent
+  // <!--
+  // order:-270
+  // -->
+  repo.init = async function () {
+    const stats = await stat(repo.path)
+    if (!stats.isDirectory()) {
+      var error = new Error(
+        `Path must be an existing directory on the file system: ${repo.path}`
+      )
+      repo.emit('error', error)
+      throw error
+    }
 
-        repo.loadConfig((err) => {
-          if (err) {
-            repo.emit('error', err)
-            return cb(err)
-          }
-          repo.migrateConfig()
-          repo.saveConfig()
-          repo.emit('config.loaded')
-          repo.createListeners()
-          repo.readFiles((err, files) => {
-            if (err && repo.destroyed) return cb(err)
-            repo.emit('initialized', { ok: true, lists: repo.getTasksByList() })
-            cb(err, files)
-          })
-        })
-      })
-    })
+    await repo.loadConfig()
+    repo.migrateConfig()
+    await repo.saveConfig()
+    repo.emit('config.loaded')
+    repo.createListeners()
+    
+    const files = await repo.readFiles()
+    const lists = repo.getTasksByList()
+    repo.emit('fs-store-initialized', { ok: true, lists })
+    return files
   }
 
   repo.loadIgnore = function () {
@@ -143,79 +117,62 @@ function mixin(repo, fs = require('fs')) {
     repo.setIgnores(patterns)
   }
 
-  repo.fileOK = function (file, includeDirs, cb) {
+  repo.fileOK = async function (file, includeDirs = false) {
     if (File.isFile(file)) file = file.path
-    if (_isFunction(includeDirs)) {
-      cb = includeDirs
-      includeDirs = false
-    }
-    cb = tools.cb(cb)
-    try {
-      if (!file || !this.shouldInclude(file)) return cb(null, false)
-    } catch (e) {
-      cb(e)
-    }
+    if (!file || !this.shouldInclude(file)) return false
+
     var fullPath = repo.getFullPath(file)
-    fs.lstat(fullPath, function (err, stat) {
-      if (err) {
-        console.log(err)
-        return cb(null, false)
+    try {
+      const stats = await lstat(fullPath)
+      
+      if (!stats) return false
+      
+      if (/\.\.(\/|\\)/.test(file) || (!includeDirs && stats.isDirectory()))
+        return false
+          
+      const content = await readFile(fullPath)
+      
+      if (await isBinaryFile(content, stats.size) ) {
+        return false
       }
-      if (/\.\.(\/|\\)/.test(file) || (!includeDirs && stat.isDirectory()))
-        return cb(null, false)
-      if (stat.isFile()) {
-        isBinaryFile(fullPath, function (err, result) {
-          if (err || result) return cb(null, false)
-          cb(null, stat)
-        })
-      } else if (includeDirs && stat.isDirectory()) {
-        cb(null, stat)
-      } else cb(null, false)
-    })
+      
+      return stats
+    } catch (err) {
+      return false
+    }
   }
 
-  repo.saveSort = function (sort, cb) {
-    cb = tools.cb(cb)
-    fs.writeFile(repo.getFullPath(SORT_FILE), JSON.stringify(sort), cb)
+  // READY Refactor saveSort to use async/await
+  // #esm-migration #important
+  // TODO readSort and saveSort are no longer used
+  repo.saveSort = async function (sort) {
+    writeFile(repo.getFullPath(SORT_FILE), JSON.stringify(sort), 'utf8')
   }
 
-  repo.readSort = function (cb) {
-    fs.readFile(repo.getFullPath(SORT_FILE), (err, data) => {
-      if (err) return cb(err)
-      repo.sort = JSON.parse(data)
-      cb(null, repo.sort)
-    })
+  repo.readSort = async function () {
+    const data = readFile(repo.getFullPath(SORT_FILE))
+    repo.sort = JSON.parse(data)
   }
 
-  repo.saveConfig = function (cb) {
-    repo.savingConfig = true
-    cb = tools.cb(cb)
-    var _cb = function (err, resp) {
-      repo.savingConfig = false
-      cb(err, resp)
-    }
-    var dir = repo.getFullPath(CONFIG_DIR)
-    var file = repo.getFullPath(CONFIG_FILE_YML)
-    fs.exists(dir, function (exists) {
+  repo.saveConfig = async function () {
+    try {
+      repo.savingConfig = true
+      const file = repo.getFullPath(CONFIG_FILE_YML)
       const data = { ...repo.getConfig() }
       delete data.cardActionsFunction
       delete data.boardActionsFunction
       delete data.cardPropertiesFunction
       delete data.path
       if (data.lists) data.lists.forEach((list) => delete list.tasks)
-      var yaml = dumpYAML(data)
-      if (exists) {
-        fs.writeFile(file, yaml, _cb)
-      } else {
-        fs.mkdir(dir, function (err) {
-          if (err) return _cb(err)
-          fs.writeFile(file, yaml, _cb)
-        })
-      }
-    })
+      const yaml = dumpYAML(data)
+      await preparePathForWriting(file)
+      await writeFile(file, yaml)
+    } finally {
+      repo.savingConfig = false
+    }
   }
 
-  repo.updateConfig = function (loadedConfig, cb) {
+  repo.updateConfig = async function (loadedConfig) {
     var baseConfig = _cloneDeep(DEFAULT_CONFIG)
     var include_lists = baseConfig.code.include_lists
     const currentConfig = repo.config
@@ -232,38 +189,42 @@ function mixin(repo, fs = require('fs')) {
       _assign({}, baseConfig, repo.config, loadedConfig, { path: repo.path })
     )
 
-    repo.migrateTasksByConfig(currentConfig, newConfig, (err) => {
-      if (err) return cb(err)
-      repo.config = newConfig
-      repo.config.code.include_lists = include_lists
-      var _languages = _cloneDeep(languages)
-      repo.languages = repo.config.languages
-        ? _assign(_languages, repo.config.languages)
-        : _languages
-      cb(null, repo.config)
-    })
+    await repo.migrateTasksByConfig(currentConfig, newConfig)
+    repo.config = newConfig
+    repo.config.code.include_lists = include_lists
+    var _languages = _cloneDeep(languages)
+    repo.languages = repo.config.languages
+      ? _assign(_languages, repo.config.languages)
+      : _languages
+    return repo.config
   }
 
-  repo.loadConfig = function (cb) {
+  // READY Refactor loadConfig to use async/await
+  // #esm-migration
+  // <!--
+  // order:-255
+  // -->
+  repo.loadConfig = async function (cb) {
     // BACKLOG If a config is bad move it to config.json.bak and save a new one with defaults gh:2 id:9 +enhancement +standup +chore
     // <!--
     // order:-670
     // -->
-    cb = tools.cb(cb)
     repo.loadIgnore()
-    // new
-    const configData =  (!repo.config || repo.config.dirty) ? this.getYamlConfig() : repo.config || {}
-    repo.updateConfig(configData, cb)
+    const configData =  (!repo.config || repo.config.dirty) 
+    ? await this.getYamlConfig() 
+    : repo.config || {}
+    
+    return await repo.updateConfig(configData)
   }
 
   repo.getConfigFile = function () {
     return repo.getFullPath(CONFIG_FILE)
   }
 
-  repo.getYamlConfig = function () {
+  repo.getYamlConfig = async function () {
     const filePath = repo.getFullPath(CONFIG_FILE_YML)
     if (!fs.existsSync(filePath)) return
-    const configData = fs.readFileSync(filePath, 'utf-8')
+    const configData = await readFile(filePath, 'utf-8')
     let configLike = {}
     try {
       configLike = loadYAML(configData.toString())
@@ -273,151 +234,125 @@ function mixin(repo, fs = require('fs')) {
     return configLike
   }
 
-  var _deleteTasks = repo.deleteTasks
-  repo.deleteTasks = function (tasks, cb) {
-    _deleteTasks.call(repo, tasks, function (err) {
-      if (err) return cb(err)
-      repo.emit('tasks.updated', tasks)
-      cb()
-    })
-  }
-
-  repo.writeFile = function (file, emitFileUpdate, cb) {
-    if (_isFunction(emitFileUpdate)) {
-      cb = emitFileUpdate
-      emitFileUpdate = false
-    }
-    cb = tools.cb(cb)
-
-    if (!File.isFile(file)) return cb(new Error(ERRORS.NOT_A_FILE))
-    if (file.deleted) return cb(null, file)
+  // READY Refactor writeFile to use async/await
+  // #esm-migration #important
+  // <!--
+  // order:-280
+  // -->
+  repo.writeFile = async (file, emitFileUpdate) => {
+    if (!File.isFile(file)) throw new Error(ERRORS.NOT_A_FILE)
+    if (file.deleted) return file
     
-    var filePath = repo.getFullPath(file)
+    if (/\.\.(\/|\\)/.test(file.path)) throw new Error('Unable to write file:' + file.path)
 
-    if (!/\.\.(\/|\\)/.test(file.path)) {
-      var write = function () {
-        var oldChecksum = file.checksum
-        file.checksum = checksum(file.getContentForFile())
-        fs.writeFile(
-          filePath,
-          file.getContentForFile(),
-          'utf8',
-          function (err) {
-            if (err) {
-              file.checksum = oldChecksum
-              return cb([new Error('Unable to write file:' + filePath), err])
-            }
-
-            fs.stat(filePath, function (err, stats) {
-              if (err)
-                return cb([new Error('Unable to stat file:' + filePath), err])
-
-              file.setModifiedTime(stats.mtime)
-              file.modified = false
-              repo.addFilePath(filePath)
-              if (emitFileUpdate) repo.emit('file.saved', file)
-              cb(null, file)
-            })
-          }
-        )
-      }
+    const filePath = repo.getFullPath(file)
+    
+    await preparePathForWriting(filePath)
 
-      var dirName = _path.dirname(filePath)
-      if (fs.existsSync(dirName)) {
-        return write()
-      }
+    const oldChecksum = file.checksum
+    file.checksum = computeChecksum(file.getContentForFile())
+        
+    try {
+      await writeFile(filePath, file.getContentForFile(), 'utf8')
+    } catch (err) {
+      file.checksum = oldChecksum
+      throw new Error('Unable to write file:' + filePath, {cause: err})
+    }
 
-      tools.mkdirp(fs, dirName, function (err) {
-        if (err) return cb(err)
-        write()
-      })
-    } else return cb(new Error('Unable to write file:' + file.path), file)
+    try {
+      const stats = await stat(filePath)
+      file.setModifiedTime(stats.mtime)
+      file.modified = false
+      repo.addFilePath(filePath)
+      if (emitFileUpdate) repo.emit('file.saved', file)
+    } catch (err) {
+      throw new Error('Unable to stat file:' + filePath, {cause: err})
+    }
+  
+    return file      
   }
 
-  repo.getFilesInPath = function (includeDirs, cb) {
-    cb = tools.cb(cb)
-    ls(repo.path, function (err, allPaths) {
-      if (err) return cb(err)
-      var files = [],
-        processed = 0
-      log('allPaths=', allPaths)
-      if (allPaths.length === 0) return cb(null, [])
-      async.eachLimit(
-        allPaths,
-        ASYNC_LIMIT,
-        function (path, _cb) {
-          if (!path) return _cb()
-          path = repo.getRelativePath(path)
-          repo.fileOK(path, includeDirs, function (err, stat) {
-            // stack-3 repo-fs-store.js:250:14
-            processed++
-            if (err) return _cb(err)
-            repo.emit('file.processed', {
-              file: path,
-              ok: stat !== false,
-              total: allPaths.length,
-              processed: processed + 1,
-              repoId: repo.getId(),
-            })
-
-            if (stat) {
-              log('%s is ok %j', path, stat)
-              var file = new File({
-                repoId: repo.getId(),
-                filePath: path,
-                modifiedTime: stat.mtime,
-                createdTime: stat.birthtime,
-                languages: repo.languages,
-                project: repo.project,
-              })
-              file.isDir = stat.isDirectory()
-              files.push(file)
-            }
-            log('err=%j', err, null)
-            log('stat=%j', stat, null)
-            log('processed=%d allPaths.length=%d', processed, allPaths.length)
-            _cb() // stack-1 repo-fs-store.js:270:11
+  // READY Refactor getFilesInPath to use async/await
+  // #esm-migration #important #urgent
+  // <!--
+  // order:-20
+  // -->
+  repo.getFilesInPath = async function (includeDirs) {
+    const allPaths = await repo.getAllFilePaths(repo.path)
+    const files = []
+    let processed = 0
+    log('allPaths=', allPaths)
+    if (allPaths.length === 0) return files
+    await eachLimit(
+      allPaths,
+      ASYNC_LIMIT,
+      async (path) => {
+        if (!path) return
+        path = repo.getRelativePath(path)
+        const stat = await repo.fileOK(path, includeDirs)
+        processed++
+        repo.emit('file.processed', {
+          file: path,
+          ok: stat !== false,
+          total: allPaths.length,
+          processed: processed + 1,
+          repoId: repo.getId(),
+        })
+
+        if (stat) {
+          log('%s is ok %j', path, stat)
+          var file = new File({
+            repoId: repo.getId(),
+            filePath: path,
+            modifiedTime: stat.mtime,
+            createdTime: stat.birthtime,
+            languages: repo.languages,
+            project: repo.project,
           })
-        },
-        function (err) {
-          cb(err, fastSort(files).asc(u => u.path))
+          file.isDir = stat.isDirectory()
+          files.push(file)
         }
-      )
-    })
+        log('stat=%j', stat, null)
+        log('processed=%d allPaths.length=%d', processed, allPaths.length)
+      })
+      return sort(files).asc(u => u.path)
   }
 
-  repo.readFileContent = function (file, cb) {
-    cb = tools.cb(cb)
-    if (!File.isFile(file)) return cb(new Error(ERRORS.NOT_A_FILE))
+  // READY Refactor readFileContent to use async/await
+  // #esm-migration #important #urgent
+  // <!--
+  // order:-40
+  // -->
+  repo.readFileContent = async function (file) {
+    if (!File.isFile(file)) throw new Error(ERRORS.NOT_A_FILE)
     var filePath = repo.getFullPath(file)
 
-    fs.lstat(filePath, function (err, stats) {
-      if (err) return cb([new Error('Unable to stat file:' + filePath), err])
-      if (!stats.isFile()) return cb(null, file)
-      fs.readFile(filePath, 'utf8', function (err, data) {
-        if (err) return cb([new Error('Unable to read file:' + filePath), err])
+    const stats = await stat(filePath)
+    if (!stats.isFile()) return file
+    const data = await readFile(filePath, 'utf8')
 
-        file
-          .setContentFromFile(data)
-          .setModifiedTime(stats.mtime)
-          .setCreatedTime(stats.birthtime)
+    file
+      .setContentFromFile(data)
+      .setModifiedTime(stats.mtime)
+      .setCreatedTime(stats.birthtime)
 
-        file.lineCount = eol.split(data).length
-        cb(null, file)
-      })
-    })
+    file.lineCount = eol.split(data).length
+    return file
   }
 
-  repo.deleteFile = function (path, cb) {
-    cb = tools.cb(cb)
-    var file = File.isFile(path) ? path : repo.getFile(path)
-    if (!_isUndefined(file)) {
-      fs.unlink(repo.getFullPath(file), function (err) {
-        if (err) return cb('Unable to delete:' + path + ' : ' + err.toString())
-        repo.removeFile(file)
-        cb(null, file)
-      })
-    } else cb('Unable to delete:' + path)
+  // READY Refactor deleteFile to use async/await
+  // #esm-migration #important
+  repo.deleteFile = async function (path) {
+    const file = File.isFile(path) ? path : repo.getFile(path)
+    if (!file) throw new Error('Unable to delete:' + path)
+
+    try {
+      await unlink(repo.getFullPath(file))
+      repo.removeFile(file)
+    } catch (err) {
+      throw new Error('Unable to delete file:' + path, {cause: err})
+    }
+
   }
 
   return repo