@ocfl/ocfl-fs 0.2.0 → 0.2.2

package/lib/store.js

@@ -1,9 +1,10 @@
 //@ts-check
 const path = require('path');
-const fs = require('fs');
-const { mkdir, copyFile, rename, unlink } = fs.promises;
+const fs = require('node:fs');
+const { Readable, Writable } = require('node:stream');
 
 const { OcflStore, OcflConstants } = require('@ocfl/ocfl');
+const { opendir } = require('./utils.js');
 const { INVENTORY_NAME } = OcflConstants;
 
 /**
@@ -37,8 +38,8 @@ class OcflFsStore extends OcflStore {
     this.fs = config?.fs ?? fs;
   }
 
-  async stat(filePath) { 
-    return this.fs.promises.stat(filePath);    
+  async stat(filePath) {
+    return this.fs.promises.stat(filePath);
   }
 
   async createReadStream(filePath, options) {
@@ -46,11 +47,20 @@ class OcflFsStore extends OcflStore {
     return this.fs.createReadStream(filePath, options);
   }
 
-  async createWriteStream(filePath, options) { 
+  async createWriteStream(filePath, options) {
     await this.fs.promises.mkdir(path.dirname(filePath), { recursive: true });
     return this.fs.createWriteStream(filePath, options);
   }
 
+  async createReadable(filePath, options) {
+    return Readable.toWeb(this.fs.createReadStream(filePath, options));
+  }
+
+  async createWritable(filePath, options) {
+    await this.fs.promises.mkdir(path.dirname(filePath), { recursive: true });
+    return Writable.toWeb(this.fs.createWriteStream(filePath, options));
+  }
+
   async readFile(filePath, options) {
     return this.fs.promises.readFile(filePath, options);
     //if (err.code !== 'ENOENT') throw validation.createError(34, invPath);
@@ -67,11 +77,54 @@ class OcflFsStore extends OcflStore {
   }
 
   async opendir(filePath, options) {
-    return this.fs.promises.opendir(filePath, options); 
+    return this.fs.promises.opendir(filePath, options);
   }
 
   async readdir(filePath, options) {
-    return this.fs.promises.readdir(filePath, options); 
+    return this.fs.promises.readdir(filePath, options);
+  }
+
+  /**
+   * @param {string} dirPath 
+   * @param {OpenDirOptions} [options]
+   * @return {Promise<AsyncIterableIterator<{name: string, path: string, size: number, lastModified: Date}>> }
+   */
+  async list(dirPath, { encoding = 'utf8', bufferSize = 32, recursive = false } = {}) {
+    const {opendir, stat} = this.fs.promises;
+    const maxBuffer = bufferSize || 1;
+    async function* generator() {
+      /** @type Promise<{ name: string, path: string, size: number, lastModified: Date}>[] */
+      let buffer = [];
+      let dirp;
+      const dirQueue = [opendir(dirPath, { encoding, bufferSize })];
+      while (dirp = dirQueue.shift()) {
+        const dir = await dirp;
+        for await (const de of dir) {
+          const cdp = path.join(dir.path, de.name);
+          if (recursive && de.isDirectory()) {
+            dirQueue.push(opendir(cdp, { encoding, bufferSize }));
+          } else {
+            let f = stat(cdp).then(stats => ({
+              name: de.name,
+              path: path.relative(dirPath, cdp),
+              size: stats.size,
+              lastModified: stats.mtime
+            }));
+            if (buffer.length < maxBuffer) {
+              buffer.push(f);
+            } else {
+              yield buffer.shift();
+              buffer.push(f);
+            }
+          }
+        }
+      }
+      let file;
+      while (file = buffer.shift()) {
+        yield file;
+      }
+    }
+    return generator();
   }
 
   async mkdir(filePath, options = { recursive: true }) {

package/lib/types.d.ts

@@ -1,3 +1,5 @@
-type TestType = {
-  name: string;
-};
+interface OpenDirOptions {
+  encoding?: BufferEncoding | undefined;
+  bufferSize?: number | undefined;
+  recursive?: boolean | undefined;
+}

package/lib/utils.js

@@ -0,0 +1,62 @@
+const fs = require('node:fs/promises');
+const path = require('node:path');
+
+/** 
+ * Wrapper for fs.opendir that reimplements the recursion and 
+ * returns just the file without the directory itself
+ * fs.opendir has a bug that gives wrong result when recursive option is used
+ * @param {string} dirPath
+ * @param {OpenDirOptions} [options]
+ */
+async function* opendir(dirPath, { encoding = 'utf8', bufferSize = 32, recursive = false } = {}) {
+  /** @type Promise<{ name: string, path: string, size: number, lastModified: Date}>[] */
+  let buffer = [];
+  let dirp;
+  const maxBuffer = bufferSize || 1;
+  const dirQueue = [fs.opendir(dirPath, { encoding, bufferSize })];
+  while (dirp = dirQueue.shift()) {
+    const dir = await dirp;
+    for await (const de of dir) {
+      const cdp = path.join(dir.path, de.name);
+      if (recursive && de.isDirectory()) {
+        dirQueue.push(fs.opendir(cdp));
+      } else {
+        let f = fs.stat(cdp).then(stats => ({
+          name: de.name,
+          path: path.relative(dirPath, cdp),
+          size: stats.size,
+          lastModified: stats.mtime
+        }));
+        if (buffer.length < maxBuffer) {
+          buffer.push(f);
+        } else {
+          yield buffer.shift();
+          buffer.push(f);
+        }
+      }
+    }
+  }
+  let file;
+  while (file = buffer.shift()) {
+    yield file;
+  }
+}
+
+// (async function() {
+  // var a = opendir('/home/alvinsw/.ssh', {recursive: true})
+  // var b = 'a';
+  // console.log(await a);
+
+  // let c = 0;
+  // for await (const f of await opendir('/home/alvinsw/uqlang', {recursive: true})) {
+  //   c++;
+  // }
+  // console.log(c);
+  // c=0;
+  // for await (const f of await fs.opendir('/home/alvinsw/uqlang', {recursive: true})) {
+  //   c++;
+  // }
+  // console.log(c);
+// })();
+
+module.exports = { opendir }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ocfl/ocfl-fs",
-  "version": "0.2.0",
+  "version": "0.2.2",
   "description": "Oxford Common File Layout (OCFL) JS client library for filesystem storage",
   "main": "index.js",
   "exports": {
@@ -37,14 +37,15 @@
   "license": "GPL-3.0-or-later",
   "devDependencies": {
     "@types/fs-extra": "^9.0.13",
-    "clean-jsdoc-theme": "^3.3.4",
-    "jsdoc": "^3.6.10",
-    "mocha": "^10.0.0",
+    "@types/node": "^18",
+    "clean-jsdoc-theme": "^4.3.0",
+    "jsdoc": "^4.0.3",
+    "mocha": "^10.7.3",
     "typedoc": "^0.22.15",
     "@ocfl/ocfl-tests": "*"
   },
   "dependencies": {
-    "@ocfl/ocfl": "0.2.0"
+    "@ocfl/ocfl": "0.2.2"
   },
   "files": [
     "index.js",